diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index ff57d89c802d77..582db25ff6d667 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -17,22 +17,77 @@ #include +#include +#include +#include +#include + #include "benchmark_arrow_validation.hpp" #include "benchmark_bit_pack.hpp" +#include "benchmark_bits.hpp" +#include "benchmark_block_bloom_filter.hpp" #include "benchmark_column_array_view.hpp" #include "benchmark_column_array_view_distance.hpp" +#include "benchmark_column_view.hpp" +#include "benchmark_damerau_levenshtein.hpp" #include "benchmark_fastunion.hpp" #include "benchmark_fmod.hpp" #include "benchmark_hll_merge.hpp" +#include "benchmark_hybrid_set.hpp" +#include "benchmark_pdep_unpack.hpp" +#include "benchmark_string.hpp" +#include "benchmark_string_replace.hpp" #include "benchmark_zone_map_index.hpp" #include "binary_cast_benchmark.hpp" +#include "common/config.h" #include "core/block/block.h" -#include "vec/columns/column_string.h" -#include "vec/data_types/data_type.h" -#include "vec/data_types/data_type_string.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_string.h" +#include "parquet/benchmark_parquet_decoder.hpp" +#include "parquet/benchmark_parquet_reader.hpp" +#include "runtime/exec_env.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" +#include "runtime/thread_context.h" + +// benchmark_binary_plain_page_v2.hpp must be included LAST: it transitively pulls AWS SDK +// headers (via storage/cache/page_cache.h) whose symbols shadow types used by the benchmark +// headers above (notably binary_cast_benchmark.hpp). Keeping it last avoids the clash without +// disabling any benchmark. (Do not let clang-format reorder it above the others.) +#include "benchmark_binary_plain_page_v2.hpp" namespace doris { // change if need +static bool init_benchmark_config(const char* executable) { + std::vector candidates; + if (const char* doris_home = std::getenv("DORIS_HOME"); doris_home != nullptr) { + candidates.emplace_back(std::filesystem::path(doris_home) / "conf" / "be.conf"); + } + candidates.emplace_back(std::filesystem::current_path() / "conf" / "be.conf"); + const auto executable_dir = std::filesystem::absolute(executable).parent_path(); + candidates.emplace_back(executable_dir / ".." / ".." / "conf" / "be.conf"); + candidates.emplace_back(executable_dir / ".." / ".." / ".." / "conf" / "be.conf"); + for (const auto& candidate : candidates) { + if (!std::filesystem::exists(candidate)) { + continue; + } + if (std::getenv("DORIS_HOME") == nullptr) { + const auto inferred_home = candidate.parent_path().parent_path().string(); + // be.conf contains paths relative to DORIS_HOME. Keep standalone benchmark launches + // equivalent to build/test scripts after locating the same repository config. + if (::setenv("DORIS_HOME", inferred_home.c_str(), 0) != 0) { + return false; + } + } + // Mutable config storage is populated by config::init. Reader benchmarks must use the + // production defaults because zero-initialized safety limits reject every valid footer. + return config::init(candidate.c_str(), false); + } + std::cerr << "Unable to find conf/be.conf for benchmark initialization\n"; + return false; +} + static void Example1(benchmark::State& state) { // init. dont time it. state.PauseTiming(); @@ -55,4 +110,28 @@ static void Example1(benchmark::State& state) { BENCHMARK(Example1); } // namespace doris -BENCHMARK_MAIN(); +// Custom main: benchmarks that touch DataPage allocation require a Doris +// ThreadContext + mem tracker, otherwise the allocator throws E-7412. Mirrors +// the minimal subset of be/test/testutil/run_all_tests.cpp::main. +int main(int argc, char** argv) { + if (!doris::init_benchmark_config(argv[0])) { + return 1; + } + doris::config::enable_bmi2_optimizations = true; + + SCOPED_INIT_THREAD_CONTEXT(); + doris::ExecEnv::GetInstance()->init_mem_tracker(); + doris::thread_context()->thread_mem_tracker_mgr->init(); + auto bench_tracker = doris::MemTrackerLimiter::create_shared( + doris::MemTrackerLimiter::Type::GLOBAL, "BE-BENCH"); + doris::thread_context()->thread_mem_tracker_mgr->attach_limiter_tracker(bench_tracker); + doris::ExecEnv::set_tracking_memory(false); + + ::benchmark::Initialize(&argc, argv); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + return 0; +} diff --git a/be/benchmark/parquet/AGENTS.md b/be/benchmark/parquet/AGENTS.md new file mode 100644 index 00000000000000..a8667fb43cdf5f --- /dev/null +++ b/be/benchmark/parquet/AGENTS.md @@ -0,0 +1,296 @@ +# Parquet microbenchmark guide for agents + +This file applies to `be/benchmark/parquet/`. Read it before changing, running, or interpreting +these benchmarks. The current suite is a local benchmark foundation, not the complete Parquet +benchmark system described in the design document. + +## What exists today + +The benchmark binary registers two groups: + +- `ParquetDecoder`: native page decoder benchmarks using in-memory encoded pages. +- `ParquetReader`: local-file benchmarks that call the format V2 Parquet reader directly. + +The relevant files are: + +- `benchmark_parquet_decoder.hpp`: deterministic page construction and decoder registration. +- `benchmark_parquet_reader.hpp`: deterministic local Parquet fixtures and reader registration. +- `parquet_benchmark_scenarios.h`: scenario definitions and the selected matrix. +- `README.md`: short human-oriented build and invocation examples. +- `be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp`: matrix invariants. + +Do not describe this suite as end-to-end SQL, `FileScannerV2`, remote I/O, V1/V2 comparison, or a +cross-engine benchmark. The reader benchmark starts at `format::parquet::ParquetReader` and does +not include FE planning, scanner scheduling, `TableReader`, client latency, or Runtime Profile +collection. + +## Build and list cases + +Performance results must come from a Release build: + +```shell +./build.sh --benchmark -j128 +``` + +List all Parquet cases and verify the expected registration counts: + +```shell +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -E '^Parquet(Decoder|Reader)/' + +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -c '^ParquetDecoder/' # currently 152 + +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -c '^ParquetReader/' # currently 137 +``` + +When running the binary directly from `be/build_RELEASE/bin`, make sure the JVM and third-party +libraries are discoverable. Prefer the installed `be/output/lib/benchmark_test` produced by +`build.sh` because the normal Doris environment already configures its dependencies. + +## Run smoke verification + +A smoke run proves that every registered case initializes and completes. It does not produce a +stable performance baseline: + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetDecoder/' \ + --benchmark_min_time=0.001s \ + --benchmark_out=parquet-decoder-smoke.json \ + --benchmark_out_format=json + +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/' \ + --benchmark_min_time=0.001s \ + --benchmark_out=parquet-reader-smoke.json \ + --benchmark_out_format=json +``` + +Reject a smoke run if the process is non-zero, the expected number of JSON results is absent, or +any result contains `error_occurred`. Also run the scenario matrix unit test after changing the +matrix. Never use the 1 ms smoke measurements to claim a speedup or regression. + +## Run a performance comparison + +Compare the same named cases on the same host, compiler, build flags, CPU set, NUMA node, and power +policy. Use the same fixture directory and cache state for both revisions. A suitable local command +is: + +```shell +taskset -c 8 be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/predicate_scan/plain/null_50/alternating/sel_10/' \ + --benchmark_min_time=1s \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=parquet-reader.json \ + --benchmark_out_format=json +``` + +Use at least three untimed warmups before collecting local warm-cache results. Run revisions in an +ABBA order when comparing two commits. Do not compare results from different CPUs, cache +topologies, compiler versions, or build types. + +The suite does not currently control the OS page cache. Repeated reader cases are normally +warm-cache measurements. Do not label them cold-NVMe results. Do not clear a shared machine's page +cache to manufacture a cold run. + +## Current scenario matrix + +`ParquetDecoder` contains 19 encoding/type pairs. Each pair is run at 1%, 10%, 50%, and 100% +selection with clustered and alternating selection ranges, for 152 registered cases. + +| Encoding | Physical types | +|---|---| +| PLAIN | INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | +| Dictionary | INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | +| BYTE_STREAM_SPLIT | FLOAT, DOUBLE, FIXED_LEN_BYTE_ARRAY | +| DELTA_BINARY_PACKED | INT32, INT64 | +| DELTA_LENGTH_BYTE_ARRAY | BYTE_ARRAY | +| DELTA_BYTE_ARRAY | BYTE_ARRAY | + +`ParquetReader` deliberately uses a single-variable matrix rather than a Cartesian product. After +deduplication it contains 137 cases covering: + +- operations: open-to-first-block, full scan, predicate scan, limit 1, and limit 1000; +- file encodings: PLAIN, dictionary, BYTE_STREAM_SPLIT, and DELTA_BINARY_PACKED; +- null ratios: 0%, 1%, 10%, 50%, and 90%; +- null shapes: clustered and alternating; +- predicate selectivities: 0%, 1%, 10%, 50%, 90%, and 100%; +- projection shapes: predicate-only and predicate plus one lazy payload column; +- schema widths: 4, 32, 128, and 512 columns; +- predicate position: first or last column. + +Except for the axis being varied, reader cases inherit the baseline: nullable INT32, PLAIN, +alternating 10% nulls, 10% selectivity, 32 columns, predicate at column zero, and predicate plus +payload projection. + +## How decoder data is generated + +Decoder pages are constructed in memory before the timed loop. There is no Parquet file, Python +generator, random seed, or manifest involved. + +- Every page contains 65,536 logical values. +- Integer values use `(row * 17) % 1000003`. +- Floating-point values use `(row % 1009) * 0.25 - 100.0`. +- Binary values are fixed at 16 bytes. Their first bytes contain `row % 1009` and the remaining + bytes are deterministic filler. +- PLAIN fixed-width values are copied in physical byte order. PLAIN BYTE_ARRAY values use a + four-byte length followed by payload bytes. +- BYTE_STREAM_SPLIT pages transpose the byte lanes of the fixed-width PLAIN representation. +- DELTA pages are produced with the Arrow Parquet DELTA encoders. +- Dictionary pages contain 256 deterministic entries. IDs repeat from 0 through 255 and are + encoded with an eight-bit RLE/bit-packed stream. +- Clustered selection is one continuous range. Alternating selection spreads selected rows evenly + and intentionally produces many one-row physical ranges. + +Page generation, selection construction, decoder creation, dictionary setup, and `set_data` are +outside the timed decode call. The sinks consume decoder callbacks and prevent compiler removal, +but they do not build a Doris `Column`. Consequently these cases isolate decoder traversal and +selection cost; they do not measure definition-level decoding, nullable reconstruction, type +conversion, or full column materialization. + +## How reader Parquet files are generated + +Reader fixtures are generated lazily by C++ code using Arrow builders and the Arrow Parquet writer. +They are stored under: + +```text +${TMPDIR:-/tmp}/doris_parquet_reader_benchmark/ +``` + +Fixture contents and writer settings are: + +- 16,384 rows and 4 row groups of 4,096 rows each; +- all columns are nullable INT32 and reuse the same deterministic Arrow array; +- every non-null value is `row % 100`; +- the predicate is `value < selectivity_percent`, so the threshold maps directly to the intended + non-null selectivity; +- alternating nulls use a 101-row period and `(row * 37) % 101`, avoiding direct correlation with + the 100-value predicate period; +- clustered nulls use contiguous null prefixes inside each 1,024-row cluster; +- Parquet format version 2.6, DataPage V2, no compression, and statistics disabled; +- dictionary is explicitly enabled only for dictionary fixtures; other fixtures disable dictionary + and request their named encoding; +- after writing, every column chunk in every row group is checked through footer metadata to ensure + that the writer did not silently choose another encoding. + +The fixture name depends only on axes that change file contents. Therefore predicate thresholds, +projection mode, and operation can reuse the same file. Existing fixtures are footer-validated +before reuse. Delete only the dedicated `doris_parquet_reader_benchmark` temporary directory when +fixture rules change; never remove a broad temporary or workspace directory. + +Fixture creation and footer verification happen before benchmark iterations. Reader initialization +and `close()` are excluded from steady-state full/predicate/LIMIT timing. The +`open_to_first_block` case intentionally includes reader construction, metadata loading, `open()`, +and the first `get_block()` call. + +## Interpret the result correctly + +Google Benchmark JSON contains `real_time`, `cpu_time`, repetitions, and custom counters. Use: + +- `cpu_time` for decoder CPU efficiency; +- both `real_time` and `cpu_time` for reader cases, because latency and CPU cost answer different + questions; +- median across repetitions as the primary result; +- p95 and a confidence interval for automated gates; never select the fastest repetition. + +Custom counters mean: + +- `raw_rows`: logical source rows considered by the case; +- `selected_rows`: rows returned after the predicate or selection; +- `ns/raw_row`: normalized source-row cost and the primary comparison across selectivities; +- `ns/selected_row`: cost per surviving row; it naturally rises as selectivity falls; +- `selection_ranges`: decoder physical ranges; compare alternating with clustered at the same + selectivity to expose per-range overhead; +- `fixture_bytes` or `encoded_bytes`: fixture/page size, not peak memory; +- `items_per_second`: selected rows per second, so it is not directly comparable across different + selectivities; +- `bytes_per_second`: a logical benchmark counter. In reader cases it is estimated from projected + INT32 values, not measured storage-device bytes or compressed bytes. + +Useful comparisons are: + +- clustered versus alternating at equal selectivity: sparse-range/cursor overhead; +- null 0% versus 1/10/50/90% at equal selection: definition-level and nullable reconstruction + overhead in the reader; +- predicate-only versus predicate-projected: lazy materialization benefit; +- width 4/32/128/512 with predicate first/last: schema and metadata traversal overhead; +- PLAIN versus dictionary/BYTE_STREAM_SPLIT/DELTA at the same reader baseline: encoding path cost; +- open-to-first-block and LIMIT cases separately from steady-state full scans. + +Before attributing a regression, confirm that output row counts are identical, the benchmark name +and fixture encoding match, and repetition variability is acceptable. A coefficient of variation +above 3% is inconclusive and should be rerun. Correlate CPU regressions with `perf stat` counters +such as cycles, instructions, branch misses, and cache misses when possible. + +## Coverage audit and required follow-up + +The current matrix is not comprehensive. Preserve this distinction in PR descriptions and reports. + +### P0: complete the local benchmark phase + +1. Add the design's `matrix.yaml`, deterministic corpus generator, `manifest.json`, checksum, and + standalone corpus verifier. The current runtime-generated files cannot be shared unchanged with + V1, StarRocks, or DuckDB. +2. Add correctness oracles. Benchmarks must validate consumed counts and representative checksums + outside the timed region before their performance samples are trusted. +3. Add reader-level INT64, FLOAT, DOUBLE, BYTE_ARRAY/string, FIXED_LEN_BYTE_ARRAY, DATE, + TIMESTAMP, and DECIMAL cases. Today only nullable INT32 reaches the complete reader path. +4. Extend decoder coverage with 0% and 90% selection, definition levels/null reconstruction, + dictionary conversion, and real Doris `Column` materialization. The current decoder sink does + not cover those costs or report decoded bytes per second. +5. Add the representative nullable sparse corpus requested by the design: 32 INT64 columns, 128 + row groups, and enough rows to exercise many pages. The current 16K-row/four-row-group fixture + is a smoke-sized workload. +6. Add dictionary footprint sweeps based on actual bytes relative to L1/L2 cache, value-width + sweeps, cardinality sweeps, and dictionary-to-PLAIN fallback/mixed-page files. The current + dictionary is small and fixed. +7. Add a Doris V1/V2 SQL runner over the same verified corpus, correctness comparison, Runtime + Profile collection, normalized result JSON, comparison report, and the first stable local + baseline. The current 1 ms run only proves executability. + +### P1: broaden Parquet format and execution behavior + +1. Vary compression codecs: uncompressed, Snappy, ZSTD, LZ4, Brotli, and Gzip where supported. +2. Cover DataPage V1 and V2, page size, row-group size/count, page boundaries, small files, and + large files that exceed cache capacity. +3. Add required/non-nullable columns, correlated and anti-correlated null/predicate distributions, + random deterministic runs, long null runs, and selections that cross page boundaries. +4. Add Boolean, INT96 compatibility, logical annotations, timestamp units/time zones, decimal + physical representations, short/long strings, skew, and empty values. +5. Add nested ARRAY/MAP/STRUCT and repeated definition/repetition-level cases. +6. Add row-group statistics, page index, Bloom filter, dictionary filtering, all-filtered batches, + missing-column/default-value handling, schema mapping, and schema-evolution cases. Keep pruning + benchmarks separate from decode benchmarks so skipped work is not misattributed to a faster + decoder. +7. Add predicate LIMIT 1/1000 cases. Current LIMIT cases only vary reader batch size without a + predicate. +8. Add peak tracked memory, allocation counts, and an explicit warm/cold-cache methodology. + +### Later phases + +Remote S3/FileCache/RTT workloads, prefetch metrics, cross-engine adapters, nightly/weekly jobs, and +release regression gates belong to later phases. They require isolated infrastructure and must not +be simulated by silently changing the local reader benchmark. + +## Current validation record + +At commit `16e05dd5c71`, a Release build completed and the matrix unit test passed 4/4. A 1 ms smoke +run executed 152 decoder and 137 reader cases with zero benchmark errors. This is an execution +record only. It is not a reviewed performance baseline because repetitions, host isolation, +warmups, cache control, `perf` data, variance, and before/after comparison were not collected. + +## Rules for extending the suite + +- Keep deterministic data construction and validation outside timed regions. +- Verify actual footer encodings; never trust writer configuration alone. +- Change one primary axis at a time and add only meaningful second-order interactions. +- Add or update matrix invariant tests before registering new dimensions. +- Keep correctness/error-injection tests separate from throughput benchmarks. +- Do not commit large generated corpora. Commit generation rules and manifests; keep only small + correctness fixtures in the repository when necessary. +- Record the exact commit, compiler, build type, host topology, command, repetitions, cache state, + and raw JSON for any claimed performance result. +- Never claim a performance improvement from a smoke run or from incomparable machines. diff --git a/be/benchmark/parquet/README.md b/be/benchmark/parquet/README.md new file mode 100644 index 00000000000000..40c79bc28bea5f --- /dev/null +++ b/be/benchmark/parquet/README.md @@ -0,0 +1,63 @@ +# Parquet reader microbenchmarks + +These benchmarks separate native page decoding from the complete local-file reader path. They use +deterministic data and verify the physical encoding recorded in each generated Parquet footer before +running a measurement. + +Agents and maintainers must read [AGENTS.md](AGENTS.md) for the exact timing boundaries, synthetic +data rules, result interpretation, current coverage limitations, and prioritized follow-up work. + +## Build + +Build the Release benchmark binary from the repository root: + +```shell +./build.sh --benchmark -j128 +``` + +List only the Parquet cases: + +```shell +be/output/lib/benchmark_test --benchmark_list_tests | grep '^Parquet' +``` + +## Decoder cases + +`ParquetDecoder` measures the native decoder with data generation and encoder setup outside the +timed region. It covers PLAIN, dictionary, byte-stream-split, and DELTA encodings across their +supported fixed-width and binary physical types. Sparse selections are provided as both one +clustered range and many alternating ranges. + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetDecoder/plain/int64/sel_10/alternating$' \ + --benchmark_min_time=0.1s +``` + +## Local reader cases + +`ParquetReader` measures local open-to-first-block, full scan, predicate scan, and LIMIT-shaped +reads. The matrix covers: + +- PLAIN, dictionary, byte-stream-split, and DELTA binary-packed files; +- NULL ratios of 0%, 1%, 10%, 50%, and 90%, with clustered and alternating placement; +- predicate selectivities of 0%, 1%, 10%, 50%, 90%, and 100%; +- predicate-only and predicate-plus-lazy-projected reads; +- schemas with 4, 32, 128, and 512 columns, with the predicate first or last. + +Fixtures are created lazily under the system temporary directory in +`doris_parquet_reader_benchmark`. Generation, footer validation, and reader setup are excluded from +steady-state scan timings. `open_to_first_block` intentionally includes reader initialization, +footer loading, open, and the first `get_block` call. + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/predicate_scan/plain/null_50/alternating/sel_10/' \ + --benchmark_min_time=0.1s \ + --benchmark_out=parquet-reader.json \ + --benchmark_out_format=json +``` + +Every result reports throughput plus `raw_rows`, `selected_rows`, `fixture_bytes`, `ns/raw_row`, +and (when at least one row survives) `ns/selected_row`. Keep CPU frequency, build type, compiler, +machine placement, and benchmark filters fixed when comparing two commits. diff --git a/be/benchmark/parquet/benchmark_parquet_decoder.hpp b/be/benchmark/parquet/benchmark_parquet_decoder.hpp new file mode 100644 index 00000000000000..25fb05d1449e53 --- /dev/null +++ b/be/benchmark/parquet/benchmark_parquet_decoder.hpp @@ -0,0 +1,479 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "parquet_benchmark_scenarios.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::parquet_benchmark { +namespace detail { + +constexpr size_t DECODER_ROWS = 1UL << 16; +constexpr size_t DICTIONARY_ENTRIES = 256; +constexpr size_t FIXED_BINARY_WIDTH = 16; + +struct EncodedPage { + std::vector data; + std::vector dictionary; + size_t dictionary_entries = 0; + size_t value_width = 0; + bool binary = false; +}; + +inline tparquet::Type::type physical_type(ValueType value_type) { + switch (value_type) { + case ValueType::INT32: + return tparquet::Type::INT32; + case ValueType::INT64: + return tparquet::Type::INT64; + case ValueType::FLOAT: + return tparquet::Type::FLOAT; + case ValueType::DOUBLE: + return tparquet::Type::DOUBLE; + case ValueType::BYTE_ARRAY: + return tparquet::Type::BYTE_ARRAY; + case ValueType::FIXED_LEN_BYTE_ARRAY: + return tparquet::Type::FIXED_LEN_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark value type"); +} + +inline tparquet::Encoding::type parquet_encoding(Encoding encoding) { + switch (encoding) { + case Encoding::PLAIN: + return tparquet::Encoding::PLAIN; + case Encoding::DICTIONARY: + return tparquet::Encoding::RLE_DICTIONARY; + case Encoding::BYTE_STREAM_SPLIT: + return tparquet::Encoding::BYTE_STREAM_SPLIT; + case Encoding::DELTA_BINARY_PACKED: + return tparquet::Encoding::DELTA_BINARY_PACKED; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case Encoding::DELTA_BYTE_ARRAY: + return tparquet::Encoding::DELTA_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark encoding"); +} + +inline std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type type, + int type_length = -1) { + auto node = type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY + ? ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, type, + ::parquet::ConvertedType::NONE, type_length) + : ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +template +std::vector fixed_values(size_t rows) { + std::vector values(rows); + for (size_t row = 0; row < rows; ++row) { + if constexpr (std::is_floating_point_v) { + values[row] = static_cast((row % 1009) * 0.25 - 100.0); + } else { + values[row] = static_cast((row * 17) % 1000003); + } + } + return values; +} + +inline std::vector binary_values(size_t rows, size_t width = FIXED_BINARY_WIDTH) { + std::vector values; + values.reserve(rows); + for (size_t row = 0; row < rows; ++row) { + std::string value(width, 'a'); + const uint64_t id = row % 1009; + memcpy(value.data(), &id, std::min(width, sizeof(id))); + values.push_back(std::move(value)); + } + return values; +} + +inline std::vector encode_plain_binary(const std::vector& values) { + size_t bytes = 0; + for (const auto& value : values) { + bytes += sizeof(uint32_t) + value.size(); + } + std::vector encoded; + encoded.reserve(bytes); + for (const auto& value : values) { + const auto length = static_cast(value.size()); + const auto* length_bytes = reinterpret_cast(&length); + encoded.insert(encoded.end(), length_bytes, length_bytes + sizeof(length)); + encoded.insert(encoded.end(), value.begin(), value.end()); + } + return encoded; +} + +template +std::vector encode_plain_fixed(const std::vector& values) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + return encoded; +} + +inline std::vector encode_fixed_binary(const std::vector& values) { + std::vector encoded; + encoded.reserve(values.size() * FIXED_BINARY_WIDTH); + for (const auto& value : values) { + encoded.insert(encoded.end(), value.begin(), value.end()); + } + return encoded; +} + +inline std::vector encode_byte_stream_split(const std::vector& plain, + size_t value_width) { + const size_t rows = plain.size() / value_width; + std::vector encoded(plain.size()); + for (size_t row = 0; row < rows; ++row) { + for (size_t byte = 0; byte < value_width; ++byte) { + encoded[byte * rows + row] = plain[row * value_width + byte]; + } + } + return encoded; +} + +template +std::vector encode_delta_fixed(const std::vector& values) { + auto desc = descriptor(ParquetType::type_num); + auto encoder = ::parquet::MakeTypedEncoder( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, desc.get()); + encoder->Put(values.data(), static_cast(values.size())); + const auto buffer = encoder->FlushValues(); + return {buffer->data(), buffer->data() + buffer->size()}; +} + +inline std::vector encode_delta_binary(const std::vector& values, + ::parquet::Encoding::type encoding) { + std::vector<::parquet::ByteArray> arrays; + arrays.reserve(values.size()); + for (const auto& value : values) { + arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto desc = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = + ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, desc.get()); + encoder->Put(arrays.data(), static_cast(arrays.size())); + const auto buffer = encoder->FlushValues(); + return {buffer->data(), buffer->data() + buffer->size()}; +} + +inline std::vector encode_dictionary_indices(size_t rows) { + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 8); + for (size_t row = 0; row < rows; ++row) { + encoder.Put(static_cast(row % DICTIONARY_ENTRIES)); + } + for (size_t padding = rows; padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector result(encoded_ids.size() + 1); + result[0] = 8; + memcpy(result.data() + 1, encoded_ids.data(), encoded_ids.size()); + return result; +} + +inline EncodedPage dictionary_page(ValueType value_type) { + EncodedPage page; + page.data = encode_dictionary_indices(DECODER_ROWS); + page.dictionary_entries = DICTIONARY_ENTRIES; + page.binary = value_type == ValueType::BYTE_ARRAY; + switch (value_type) { + case ValueType::INT32: + page.value_width = sizeof(int32_t); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::INT64: + page.value_width = sizeof(int64_t); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::FLOAT: + page.value_width = sizeof(float); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::DOUBLE: + page.value_width = sizeof(double); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::BYTE_ARRAY: + page.value_width = FIXED_BINARY_WIDTH; + page.dictionary = encode_plain_binary(binary_values(DICTIONARY_ENTRIES)); + break; + case ValueType::FIXED_LEN_BYTE_ARRAY: + page.value_width = FIXED_BINARY_WIDTH; + page.dictionary = encode_fixed_binary(binary_values(DICTIONARY_ENTRIES)); + break; + } + return page; +} + +inline EncodedPage encoded_page(const DecoderScenario& scenario) { + if (scenario.encoding == Encoding::DICTIONARY) { + return dictionary_page(scenario.value_type); + } + + EncodedPage page; + page.binary = scenario.value_type == ValueType::BYTE_ARRAY; + switch (scenario.value_type) { + case ValueType::INT32: { + auto values = fixed_values(DECODER_ROWS); + page.value_width = sizeof(int32_t); + page.data = scenario.encoding == Encoding::DELTA_BINARY_PACKED + ? encode_delta_fixed<::parquet::Int32Type>(values) + : encode_plain_fixed(values); + break; + } + case ValueType::INT64: { + auto values = fixed_values(DECODER_ROWS); + page.value_width = sizeof(int64_t); + page.data = scenario.encoding == Encoding::DELTA_BINARY_PACKED + ? encode_delta_fixed<::parquet::Int64Type>(values) + : encode_plain_fixed(values); + break; + } + case ValueType::FLOAT: { + auto plain = encode_plain_fixed(fixed_values(DECODER_ROWS)); + page.value_width = sizeof(float); + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + case ValueType::DOUBLE: { + auto plain = encode_plain_fixed(fixed_values(DECODER_ROWS)); + page.value_width = sizeof(double); + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + case ValueType::BYTE_ARRAY: { + auto values = binary_values(DECODER_ROWS); + page.value_width = FIXED_BINARY_WIDTH; + if (scenario.encoding == Encoding::DELTA_LENGTH_BYTE_ARRAY) { + page.data = encode_delta_binary(values, ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY); + } else if (scenario.encoding == Encoding::DELTA_BYTE_ARRAY) { + page.data = encode_delta_binary(values, ::parquet::Encoding::DELTA_BYTE_ARRAY); + } else { + page.data = encode_plain_binary(values); + } + break; + } + case ValueType::FIXED_LEN_BYTE_ARRAY: { + auto plain = encode_fixed_binary(binary_values(DECODER_ROWS)); + page.value_width = FIXED_BINARY_WIDTH; + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + } + return page; +} + +class FixedSink final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + benchmark::DoNotOptimize(values); + benchmark::DoNotOptimize(num_values); + benchmark::DoNotOptimize(value_width); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; +}; + +class BinarySink final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + benchmark::DoNotOptimize(values); + benchmark::DoNotOptimize(num_values); + consumed += num_values; + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + benchmark::DoNotOptimize(encoded_data); + benchmark::DoNotOptimize(payload_offsets); + benchmark::DoNotOptimize(value_offsets); + auto value_spans_data = value_spans.data(); + benchmark::DoNotOptimize(value_spans_data); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; +}; + +class DictionarySink final : public ParquetDictionaryValueConsumer { +public: + DictionarySink(const uint8_t* dictionary, size_t value_width) + : _dictionary(dictionary), _value_width(value_width) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + auto* value = _dictionary + indices[row] * _value_width; + benchmark::DoNotOptimize(value); + } + consumed += num_values; + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + auto* value = _dictionary + index * _value_width; + benchmark::DoNotOptimize(value); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; + +private: + const uint8_t* _dictionary; + const size_t _value_width; +}; + +inline ParquetSelection native_selection(const SelectionPlan& plan) { + ParquetSelection selection { + .total_values = plan.total_rows, .selected_values = plan.selected_rows, .ranges = {}}; + selection.ranges.reserve(plan.ranges.size()); + for (const auto& range : plan.ranges) { + selection.ranges.push_back({.first = range.first, .count = range.count}); + } + return selection; +} + +inline void run_decoder(benchmark::State& state, DecoderScenario scenario, int selectivity, + Pattern pattern) { + auto page = encoded_page(scenario); + const auto plan = make_selection_plan(DECODER_ROWS, selectivity, pattern); + const auto selection = native_selection(plan); + std::unique_ptr decoder; + auto status = format::parquet::native::Decoder::get_decoder( + physical_type(scenario.value_type), parquet_encoding(scenario.encoding), decoder); + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + return; + } + decoder->set_type_length(static_cast(page.value_width)); + decoder->set_expected_values(DECODER_ROWS); + + std::vector dictionary_for_sink; + if (scenario.encoding == Encoding::DICTIONARY) { + dictionary_for_sink = page.dictionary; + auto dictionary = make_unique_buffer(page.dictionary.size()); + memcpy(dictionary.get(), page.dictionary.data(), page.dictionary.size()); + status = decoder->set_dict(dictionary, static_cast(page.dictionary.size()), + page.dictionary_entries); + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + return; + } + } + + Slice encoded(page.data.data(), page.data.size()); + FixedSink fixed_sink; + BinarySink binary_sink; + DictionarySink dictionary_sink(dictionary_for_sink.data(), page.value_width); + for (auto _ : state) { + state.PauseTiming(); + status = decoder->set_data(&encoded); + fixed_sink.consumed = 0; + binary_sink.consumed = 0; + dictionary_sink.consumed = 0; + state.ResumeTiming(); + if (scenario.encoding == Encoding::DICTIONARY) { + status = decoder->decode_selected_dictionary_values(selection, dictionary_sink); + } else if (page.binary) { + status = decoder->decode_selected_binary_values(selection, binary_sink); + } else { + status = decoder->decode_selected_fixed_values(selection, fixed_sink); + } + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + break; + } + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(static_cast(state.iterations()) * + static_cast(plan.selected_rows)); + state.SetBytesProcessed(static_cast(state.iterations()) * + static_cast(page.data.size())); + state.counters["raw_rows"] = static_cast(DECODER_ROWS); + state.counters["selected_rows"] = static_cast(plan.selected_rows); + state.counters["selection_ranges"] = static_cast(plan.ranges.size()); + state.counters["encoded_bytes"] = static_cast(page.data.size()); + state.counters["ns/raw_row"] = benchmark::Counter( + static_cast(DECODER_ROWS), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + if (plan.selected_rows > 0) { + state.counters["ns/selected_row"] = benchmark::Counter( + static_cast(plan.selected_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + } +} + +inline bool register_decoder_benchmarks() { + for (const auto& scenario : decoder_scenarios()) { + for (const int selectivity : {1, 10, 50, 100}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + const std::string name = "ParquetDecoder/" + to_string(scenario.encoding) + "/" + + to_string(scenario.value_type) + "/sel_" + + std::to_string(selectivity) + "/" + to_string(pattern); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + run_decoder(state, scenario, selectivity, pattern); + })->Unit(benchmark::kNanosecond); + } + } + } + return true; +} + +inline const bool DECODER_BENCHMARKS_REGISTERED = register_decoder_benchmarks(); + +} // namespace detail +} // namespace doris::parquet_benchmark diff --git a/be/benchmark/parquet/benchmark_parquet_reader.hpp b/be/benchmark/parquet/benchmark_parquet_reader.hpp new file mode 100644 index 00000000000000..ae3e14c0f66ad9 --- /dev/null +++ b/be/benchmark/parquet/benchmark_parquet_reader.hpp @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_reader.h" +#include "gen_cpp/Types_types.h" +#include "io/io_common.h" +#include "parquet_benchmark_scenarios.h" +#include "runtime/runtime_state.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" + +namespace doris::parquet_benchmark { +namespace reader_detail { + +constexpr size_t READER_ROWS = 1UL << 14; +constexpr size_t READER_ROW_GROUP_ROWS = 1UL << 12; + +inline void throw_if_error(const Status& status) { + if (!status.ok()) { + throw std::runtime_error(status.to_string()); + } +} + +inline bool is_null_row(size_t row, int null_percent, Pattern pattern) { + if (null_percent <= 0) { + return false; + } + if (pattern == Pattern::ALTERNATING) { + constexpr size_t NULL_PERIOD = 101; + const size_t null_slots = (static_cast(null_percent) * NULL_PERIOD + 99) / 100; + return (row * 37) % NULL_PERIOD < null_slots; + } + constexpr size_t CLUSTER_ROWS = 1024; + return row % CLUSTER_ROWS < CLUSTER_ROWS * static_cast(null_percent) / 100; +} + +inline std::shared_ptr build_int32_array(int null_percent, Pattern pattern) { + arrow::Int32Builder builder; + PARQUET_THROW_NOT_OK(builder.Reserve(READER_ROWS)); + for (size_t row = 0; row < READER_ROWS; ++row) { + if (is_null_row(row, null_percent, pattern)) { + PARQUET_THROW_NOT_OK(builder.AppendNull()); + } else { + PARQUET_THROW_NOT_OK(builder.Append(static_cast(row % 100))); + } + } + return builder.Finish().ValueOrDie(); +} + +inline ::parquet::Encoding::type file_encoding(Encoding encoding) { + switch (encoding) { + case Encoding::PLAIN: + return ::parquet::Encoding::PLAIN; + case Encoding::BYTE_STREAM_SPLIT: + return ::parquet::Encoding::BYTE_STREAM_SPLIT; + case Encoding::DELTA_BINARY_PACKED: + return ::parquet::Encoding::DELTA_BINARY_PACKED; + case Encoding::DICTIONARY: + return ::parquet::Encoding::RLE_DICTIONARY; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case Encoding::DELTA_BYTE_ARRAY: + return ::parquet::Encoding::DELTA_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark encoding"); +} + +inline std::string fixture_name(const ReaderScenario& scenario) { + return "v2_" + to_string(scenario.encoding) + "_null" + std::to_string(scenario.null_percent) + + "_" + to_string(scenario.null_pattern) + "_w" + std::to_string(scenario.schema_width) + + "_p" + std::to_string(scenario.predicate_position) + ".parquet"; +} + +inline void verify_fixture_encoding(const std::filesystem::path& path, + const ReaderScenario& scenario) { + auto reader = ::parquet::ParquetFileReader::OpenFile(path.string(), false); + const auto metadata = reader->metadata(); + if (metadata->num_rows() != static_cast(READER_ROWS) || + metadata->num_columns() != scenario.schema_width) { + throw std::runtime_error("Parquet benchmark fixture has unexpected shape: " + + path.string()); + } + const auto expected = file_encoding(scenario.encoding); + for (int row_group = 0; row_group < metadata->num_row_groups(); ++row_group) { + for (int column = 0; column < metadata->num_columns(); ++column) { + const auto encodings = metadata->RowGroup(row_group)->ColumnChunk(column)->encodings(); + if (std::ranges::find(encodings, expected) == encodings.end()) { + throw std::runtime_error("Parquet benchmark fixture did not use " + + to_string(scenario.encoding) + + " encoding: " + path.string()); + } + } + } +} + +inline std::filesystem::path ensure_fixture(const ReaderScenario& scenario) { + static std::mutex fixture_mutex; + const auto directory = + std::filesystem::temp_directory_path() / "doris_parquet_reader_benchmark"; + const auto path = directory / fixture_name(scenario); + std::lock_guard guard(fixture_mutex); + if (std::filesystem::exists(path)) { + verify_fixture_encoding(path, scenario); + return path; + } + + std::filesystem::create_directories(directory); + const auto temporary_path = path.string() + ".tmp"; + std::filesystem::remove(temporary_path); + const auto values = build_int32_array(scenario.null_percent, scenario.null_pattern); + std::vector> fields; + std::vector> columns; + fields.reserve(scenario.schema_width); + columns.reserve(scenario.schema_width); + for (int column = 0; column < scenario.schema_width; ++column) { + fields.push_back(arrow::field("c" + std::to_string(column), arrow::int32(), true)); + columns.push_back(std::make_shared(values)); + } + const auto table = arrow::Table::Make(arrow::schema(std::move(fields)), std::move(columns)); + + const auto output_result = arrow::io::FileOutputStream::Open(temporary_path); + if (!output_result.ok()) { + throw std::runtime_error(output_result.status().ToString()); + } + const auto output = *output_result; + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + properties.disable_statistics(); + if (scenario.encoding == Encoding::DICTIONARY) { + properties.enable_dictionary(); + } else { + properties.disable_dictionary(); + properties.encoding(file_encoding(scenario.encoding)); + } + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, + READER_ROW_GROUP_ROWS, properties.build())); + PARQUET_THROW_NOT_OK(output->Close()); + std::filesystem::rename(temporary_path, path); + verify_fixture_encoding(path, scenario); + return path; +} + +class Int32LessThanExpr final : public VExpr { +public: + Int32LessThanExpr(int column_id, int32_t upper_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _upper_bound(upper_bound) {} + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& values = assert_cast(nullable.get_nested_column()); + const auto& nulls = nullable.get_null_map_data(); + auto result = ColumnUInt8::create(count, 0); + auto& matches = result->get_data(); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + matches[row] = !nulls[input_row] && values.get_element(input_row) < _upper_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int column_id, + uint8_t* matches) const override { + DORIS_CHECK(column_id == _column_id); + DORIS_CHECK(value_width == sizeof(int32_t)); + const auto* typed_values = reinterpret_cast(values); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= typed_values[row] < _upper_bound; + } + return Status::OK(); + } + + bool can_evaluate_zonemap_filter() const override { return true; } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(_column_id); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + const auto upper_bound = Field::create_field(_upper_bound); + return zone_map->min_value >= upper_bound ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + const auto upper_bound = Field::create_field(_upper_bound); + return std::ranges::any_of(dictionary->values, + [&](const Field& value) { return value < upper_bound; }) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _upper_bound; + const std::string _expr_name = "ParquetBenchmarkInt32LessThan"; +}; + +inline VExprContextSPtr make_predicate(int column_position, int selectivity_percent) { + auto context = VExprContext::create_shared( + std::make_shared(column_position, selectivity_percent)); + context->_prepared = true; + context->_opened = true; + return context; +} + +inline Block make_block(const std::vector& schema) { + Block block; + for (const auto& column : schema) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +struct ReaderSession { + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + std::unique_ptr reader; + std::vector schema; + std::shared_ptr request; +}; + +inline std::unique_ptr open_reader(const std::filesystem::path& path, + const ReaderScenario& scenario) { + auto session = std::make_unique(); + auto properties = std::make_shared(); + properties->system_type = TFileType::FILE_LOCAL; + auto description = std::make_unique(); + description->path = path.string(); + description->file_size = static_cast(std::filesystem::file_size(path)); + description->range_start_offset = 0; + description->range_size = -1; + session->reader = std::make_unique(properties, description, + nullptr, nullptr); + throw_if_error(session->reader->init(&session->runtime_state)); + throw_if_error(session->reader->get_schema(&session->schema)); + + session->request = std::make_shared(); + format::FileScanRequestBuilder request_builder(session->request.get()); + if (scenario.operation == ReaderOperation::FULL_SCAN) { + for (int column = 0; column < scenario.schema_width; ++column) { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(column))); + } + } else if (scenario.operation == ReaderOperation::PREDICATE_SCAN) { + const auto predicate_id = format::LocalColumnId(scenario.predicate_position); + throw_if_error(request_builder.add_predicate_column(predicate_id)); + if (scenario.projection == Projection::PREDICATE_ONLY) { + session->request->predicate_only_columns.push_back(predicate_id); + } else { + const int payload = scenario.predicate_position == 0 ? scenario.schema_width - 1 : 0; + throw_if_error( + request_builder.add_non_predicate_column(format::LocalColumnId(payload))); + } + const auto predicate_position = session->request->local_positions.at(predicate_id).value(); + session->request->conjuncts.push_back( + make_predicate(static_cast(predicate_position), scenario.selectivity_percent)); + } else { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(0))); + if (scenario.schema_width > 1) { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(1))); + } + } + + if (scenario.operation == ReaderOperation::LIMIT_1) { + session->reader->set_batch_size(1); + } else if (scenario.operation == ReaderOperation::LIMIT_1000) { + session->reader->set_batch_size(1000); + } + throw_if_error(session->reader->open(session->request)); + return session; +} + +inline size_t scan_reader(ReaderSession* session, const ReaderScenario& scenario) { + size_t output_rows = 0; + bool eof = false; + const size_t limit = scenario.operation == ReaderOperation::LIMIT_1 ? 1 + : scenario.operation == ReaderOperation::LIMIT_1000 ? 1000 + : 0; + while (!eof) { + auto block = make_block(session->schema); + size_t rows = 0; + throw_if_error(session->reader->get_block(&block, &rows, &eof)); + output_rows += rows; + benchmark::DoNotOptimize(block); + if (scenario.operation == ReaderOperation::OPEN_TO_FIRST_BLOCK) { + break; + } + if (limit != 0 && output_rows >= limit) { + break; + } + } + return output_rows; +} + +inline size_t raw_rows_per_iteration(const ReaderScenario& scenario) { + if (scenario.operation == ReaderOperation::LIMIT_1) { + return 1; + } + if (scenario.operation == ReaderOperation::LIMIT_1000) { + return 1000; + } + if (scenario.operation == ReaderOperation::OPEN_TO_FIRST_BLOCK) { + return format::parquet::ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; + } + return READER_ROWS; +} + +inline int projected_columns(const ReaderScenario& scenario) { + if (scenario.operation == ReaderOperation::FULL_SCAN) { + return scenario.schema_width; + } + if (scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.projection == Projection::PREDICATE_ONLY) { + return 1; + } + return std::min(2, scenario.schema_width); +} + +inline void run_reader(benchmark::State& state, ReaderScenario scenario) { + std::filesystem::path fixture; + try { + fixture = ensure_fixture(scenario); + size_t selected_rows = 0; + for (auto _ : state) { + if (scenario.operation != ReaderOperation::OPEN_TO_FIRST_BLOCK) { + state.PauseTiming(); + } + auto session = open_reader(fixture, scenario); + if (scenario.operation != ReaderOperation::OPEN_TO_FIRST_BLOCK) { + state.ResumeTiming(); + } + selected_rows = scan_reader(session.get(), scenario); + state.PauseTiming(); + throw_if_error(session->reader->close()); + state.ResumeTiming(); + benchmark::ClobberMemory(); + } + + const auto raw_rows = raw_rows_per_iteration(scenario); + state.SetItemsProcessed(static_cast(state.iterations() * selected_rows)); + state.SetBytesProcessed(static_cast( + state.iterations() * raw_rows * projected_columns(scenario) * sizeof(int32_t))); + state.counters["raw_rows"] = static_cast(raw_rows); + state.counters["selected_rows"] = static_cast(selected_rows); + state.counters["fixture_bytes"] = static_cast(std::filesystem::file_size(fixture)); + state.counters["ns/raw_row"] = benchmark::Counter( + static_cast(raw_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + if (selected_rows > 0) { + state.counters["ns/selected_row"] = benchmark::Counter( + static_cast(selected_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + } + } catch (const std::exception& error) { + state.SkipWithError(error.what()); + } +} + +inline bool register_reader_benchmarks() { + for (const auto& scenario : reader_scenarios()) { + std::string name = + "ParquetReader/" + to_string(scenario.operation) + "/" + + to_string(scenario.encoding) + "/null_" + std::to_string(scenario.null_percent) + + "/" + to_string(scenario.null_pattern) + "/sel_" + + std::to_string(scenario.selectivity_percent) + "/" + + to_string(scenario.projection) + "/width_" + std::to_string(scenario.schema_width) + + "/predicate_" + std::to_string(scenario.predicate_position); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + run_reader(state, scenario); + })->Unit(benchmark::kNanosecond); + } + return true; +} + +inline const bool READER_BENCHMARKS_REGISTERED = register_reader_benchmarks(); + +} // namespace reader_detail +} // namespace doris::parquet_benchmark diff --git a/be/benchmark/parquet/parquet_benchmark_scenarios.h b/be/benchmark/parquet/parquet_benchmark_scenarios.h new file mode 100644 index 00000000000000..a01c955c6a2aed --- /dev/null +++ b/be/benchmark/parquet/parquet_benchmark_scenarios.h @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::parquet_benchmark { + +enum class Encoding { + PLAIN, + DICTIONARY, + BYTE_STREAM_SPLIT, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY +}; +enum class ValueType { INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY }; +enum class Pattern { CLUSTERED, ALTERNATING }; +enum class Projection { PREDICATE_ONLY, PREDICATE_PROJECTED }; +enum class ReaderOperation { OPEN_TO_FIRST_BLOCK, FULL_SCAN, PREDICATE_SCAN, LIMIT_1, LIMIT_1000 }; + +struct DecoderScenario { + Encoding encoding; + ValueType value_type; +}; + +struct ReaderScenario { + ReaderOperation operation; + Encoding encoding; + int null_percent; + Pattern null_pattern; + int selectivity_percent; + Projection projection; + int schema_width; + int predicate_position; +}; + +struct SelectionRange { + size_t first; + size_t count; +}; + +struct SelectionPlan { + size_t total_rows = 0; + size_t selected_rows = 0; + std::vector ranges; +}; + +inline std::vector decoder_scenarios() { + return { + {Encoding::PLAIN, ValueType::INT32}, + {Encoding::PLAIN, ValueType::INT64}, + {Encoding::PLAIN, ValueType::FLOAT}, + {Encoding::PLAIN, ValueType::DOUBLE}, + {Encoding::PLAIN, ValueType::BYTE_ARRAY}, + {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::INT32}, + {Encoding::DICTIONARY, ValueType::INT64}, + {Encoding::DICTIONARY, ValueType::FLOAT}, + {Encoding::DICTIONARY, ValueType::DOUBLE}, + {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, + {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + }; +} + +inline std::vector reader_scenarios() { + std::vector scenarios; + std::set> seen; + const auto add = [&](ReaderScenario scenario) { + const auto key = std::make_tuple(scenario.operation, scenario.encoding, + scenario.null_percent, scenario.null_pattern, + scenario.selectivity_percent, scenario.projection, + scenario.schema_width, scenario.predicate_position); + if (seen.insert(key).second) { + scenarios.push_back(scenario); + } + }; + + const ReaderScenario baseline {.operation = ReaderOperation::FULL_SCAN, + .encoding = Encoding::PLAIN, + .null_percent = 10, + .null_pattern = Pattern::ALTERNATING, + .selectivity_percent = 10, + .projection = Projection::PREDICATE_PROJECTED, + .schema_width = 32, + .predicate_position = 0}; + for (const auto operation : + {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, + ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { + auto scenario = baseline; + scenario.operation = operation; + add(scenario); + } + for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, + Encoding::DELTA_BINARY_PACKED}) { + auto scenario = baseline; + scenario.encoding = encoding; + add(scenario); + scenario.operation = ReaderOperation::PREDICATE_SCAN; + add(scenario); + } + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.encoding = encoding; + scenario.selectivity_percent = selectivity; + scenario.projection = projection; + add(scenario); + } + } + } + for (const int width : {4, 32, 128, 512}) { + for (const int predicate_position : {0, width - 1}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.schema_width = width; + scenario.predicate_position = predicate_position; + add(scenario); + } + } + for (const int null_percent : {0, 1, 10, 50, 90}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.null_percent = null_percent; + scenario.null_pattern = pattern; + scenario.selectivity_percent = selectivity; + scenario.projection = projection; + add(scenario); + } + } + } + } + return scenarios; +} + +inline SelectionPlan make_selection_plan(size_t total_rows, int selectivity_percent, + Pattern pattern) { + SelectionPlan plan {.total_rows = total_rows, .selected_rows = 0, .ranges = {}}; + if (total_rows == 0 || selectivity_percent <= 0) { + return plan; + } + if (selectivity_percent >= 100) { + plan.selected_rows = total_rows; + plan.ranges.push_back({.first = 0, .count = total_rows}); + return plan; + } + plan.selected_rows = total_rows * static_cast(selectivity_percent) / 100; + if (plan.selected_rows == 0) { + plan.selected_rows = 1; + } + if (pattern == Pattern::CLUSTERED) { + plan.ranges.push_back({.first = 0, .count = plan.selected_rows}); + return plan; + } + + // Evenly spaced rows deliberately maximize the number of physical ranges. This is the + // adversarial sparse shape that exposes per-run decoder and cursor overhead. + for (size_t selected = 0; selected < plan.selected_rows; ++selected) { + const size_t row = selected * total_rows / plan.selected_rows; + if (!plan.ranges.empty() && plan.ranges.back().first + plan.ranges.back().count == row) { + ++plan.ranges.back().count; + } else { + plan.ranges.push_back({.first = row, .count = 1}); + } + } + return plan; +} + +inline std::string to_string(Encoding value) { + switch (value) { + case Encoding::PLAIN: + return "plain"; + case Encoding::DICTIONARY: + return "dictionary"; + case Encoding::BYTE_STREAM_SPLIT: + return "byte_stream_split"; + case Encoding::DELTA_BINARY_PACKED: + return "delta_binary_packed"; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return "delta_length_byte_array"; + case Encoding::DELTA_BYTE_ARRAY: + return "delta_byte_array"; + } + return "unknown"; +} + +inline std::string to_string(ValueType value) { + switch (value) { + case ValueType::INT32: + return "int32"; + case ValueType::INT64: + return "int64"; + case ValueType::FLOAT: + return "float"; + case ValueType::DOUBLE: + return "double"; + case ValueType::BYTE_ARRAY: + return "byte_array"; + case ValueType::FIXED_LEN_BYTE_ARRAY: + return "fixed_len_byte_array"; + } + return "unknown"; +} + +inline std::string to_string(Pattern value) { + return value == Pattern::CLUSTERED ? "clustered" : "alternating"; +} + +inline std::string to_string(Projection value) { + return value == Projection::PREDICATE_ONLY ? "predicate_only" : "predicate_projected"; +} + +inline std::string to_string(ReaderOperation value) { + switch (value) { + case ReaderOperation::OPEN_TO_FIRST_BLOCK: + return "open_to_first_block"; + case ReaderOperation::FULL_SCAN: + return "full_scan"; + case ReaderOperation::PREDICATE_SCAN: + return "predicate_scan"; + case ReaderOperation::LIMIT_1: + return "limit_1"; + case ReaderOperation::LIMIT_1000: + return "limit_1000"; + } + return "unknown"; +} + +} // namespace doris::parquet_benchmark diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index 9547ff1a7ca3da..3b727285e64613 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -262,6 +262,14 @@ class IColumn : public COW { "Method insert_many_continuous_binary_data is not supported for " + get_name()); } + // OFFSET_ONLY readers still need true string lengths; keeping this virtual preserves that + // invariant when decoders only hold an IColumn reference and intentionally skip payloads. + virtual void insert_offsets_from_lengths(const uint32_t* lengths, size_t num) { + throw doris::Exception( + ErrorCode::NOT_IMPLEMENTED_ERROR, + "Method insert_offsets_from_lengths is not supported for " + get_name()); + } + virtual void insert_many_strings(const StringRef* strings, size_t num) { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "Method insert_many_strings is not supported for " + get_name()); @@ -682,6 +690,12 @@ class IColumn : public COW { */ String dump_structure() const; + Status column_self_check() const { + // branch-4.1 predates the recursive debug validators used by master format_v2; keeping + // this check side-effect free preserves the branch's established column semantics. + return Status::OK(); + } + // only used in agg value replace for column which is not variable length, eg.BlockReader::_copy_value_data // usage: self_column.replace_column_data(other_column, other_column's row index, self_column's row index) virtual void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) = 0; diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 8972a4ca210ceb..7b6317e297f18c 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -277,6 +278,79 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + template + void insert_many_parquet_plain_byte_arrays(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num, + const std::vector& value_spans) { + if (UNLIKELY(num == 0)) { + return; + } + const size_t old_chars_size = chars.size(); + const size_t bytes = value_offsets[num]; + check_chars_length(old_chars_size + bytes, offsets.size() + num); + chars.resize(old_chars_size + bytes); + + // Source payloads may be contiguous for decoder-owned buffers even though PLAIN normally + // separates them with length prefixes. Coalesce whenever the published layout permits it; + // the fallback remains one direct source-to-column copy without a StringRef staging array. + size_t covered_values = 0; + for (const auto& span : value_spans) { + DORIS_CHECK_EQ(span.first, covered_values); + DORIS_CHECK_LE(span.first + span.count, num); + size_t run_first = span.first; + const size_t span_end = span.first + span.count; + while (run_first < span_end) { + size_t run_end = run_first + 1; + while (run_end < span_end && + payload_offsets[run_end] == payload_offsets[run_first] + + value_offsets[run_end] - + value_offsets[run_first]) { + ++run_end; + } + const size_t run_bytes = value_offsets[run_end] - value_offsets[run_first]; + if (run_bytes != 0) { + memcpy(chars.data() + old_chars_size + value_offsets[run_first], + encoded_data + payload_offsets[run_first], run_bytes); + } + run_first = run_end; + } + covered_values = span_end; + } + DORIS_CHECK_EQ(covered_values, num); + + const size_t old_rows = offsets.size(); + const auto tail_offset = offsets.back(); + offsets.resize(old_rows + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_rows + row] = tail_offset + value_offsets[row + 1]; + } + sanity_check_simple(); + } + + // Insert `num` string entries with real length information but no actual + // character data. The `lengths` array provides the byte length of each + // string. Offsets are built with correct cumulative sizes so that + // size_at(i) returns the true string length. The chars buffer is extended + // with zero-filled padding to maintain the invariant chars.size() == offsets.back(). + // Used by OFFSET_ONLY reading mode where actual string content is not needed + // but length information must be preserved (e.g., for length() function). + void insert_offsets_from_lengths(const uint32_t* lengths, size_t num) override { + if (UNLIKELY(num == 0)) { + return; + } + const auto old_rows = offsets.size(); + // Build cumulative offsets from lengths + offsets.resize(old_rows + num); + auto* offsets_ptr = &offsets[old_rows]; + size_t running_offset = offsets[old_rows - 1]; + for (size_t i = 0; i < num; ++i) { + running_offset += lengths[i]; + offsets_ptr[i] = static_cast(running_offset); + } + chars.resize(offsets[old_rows + num - 1]); + } + void insert_many_strings(const StringRef* strings, size_t num) override { size_t new_size = 0; for (size_t i = 0; i < num; i++) { @@ -300,6 +374,35 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + void insert_many_fixed_length_data(const char* data, size_t value_length, size_t num) { + if (num == 0) { + return; + } + if (value_length > std::numeric_limits::max() / num) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + const size_t old_chars_size = chars.size(); + const size_t old_offsets_size = offsets.size(); + const size_t bytes = value_length * num; + if (bytes > std::numeric_limits::max() - old_chars_size || + num > std::numeric_limits::max() - old_offsets_size) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + check_chars_length(old_chars_size + bytes, old_offsets_size + num); + chars.resize(old_chars_size + bytes); + if (bytes != 0) { + memcpy(chars.data() + old_chars_size, data, bytes); + } + offsets.resize(old_offsets_size + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_offsets_size + row] = + static_cast(old_chars_size + (row + 1) * value_length); + } + sanity_check_simple(); + } + template void insert_many_strings_fixed_length(const StringRef* strings, size_t num) { size_t new_size = 0; diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 5b372cce23bd65..787c73a8fa1a30 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -31,10 +31,13 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_string.h" +#include "util/unaligned.h" enum { DIVISOR_FOR_SECOND = 1, @@ -49,22 +52,6 @@ static const int64_t micro_to_nano_second = 1000; namespace { -#pragma pack(1) -struct DecodedInt96Timestamp { - int64_t nanos_of_day; - int32_t julian_day; - - int64_t to_timestamp_micros() const { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_IN_DAY = 86400000000; - static constexpr int64_t NANOS_PER_MICROSECOND = 1000; - return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + - nanos_of_day / NANOS_PER_MICROSECOND; - } -}; -#pragma pack() -static_assert(sizeof(DecodedInt96Timestamp) == 12); - Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -106,9 +93,9 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, return Status::OK(); } -void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, - int64_t timestamp_micros, - const cctz::time_zone& timezone) { +Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros, + const cctz::time_zone& timezone) { static constexpr int64_t MICROS_PER_SECOND = 1000000; int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; @@ -121,19 +108,91 @@ void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, DateV2Value datetime_value; datetime_value.from_unixtime(epoch_seconds, timezone); datetime_value.set_microsecond(static_cast(micros_of_second)); + if (!datetime_value.is_valid_date()) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}", + timestamp_micros); + } data.push_back(datetime_value); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = _context.timezone == nullptr ? utc_timezone : *_context.timezone; + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + status = _context.physical_type == ParquetPhysicalType::INT96 || + _context.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(_data, timestamp_micros); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectDateTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATETIMEV2"); + } +}; + } // namespace // NOLINTBEGIN(readability-function-size) @@ -561,7 +620,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); static const auto utc_timezone = cctz::utc_time_zone(); const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; for (int64_t row = 0; row < view.row_count; ++row) { @@ -569,8 +628,19 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - append_datetimev2_from_utc_epoch_micros(data, values[row].to_timestamp_micros(), - timezone); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -583,24 +653,63 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - const int64_t timestamp_micros = decoded_timestamp_micros(view, values[row]); - if (view.timestamp_is_adjusted_to_utc) { - append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); - } else { - auto st = append_datetimev2_from_epoch_micros(data, timestamp_micros); - if (!st.ok()) { - if (decoded_column_view_can_null_on_conversion_failure(view)) { - decoded_column_view_insert_null_on_conversion_failure(column, view, row); - continue; - } - data.resize(old_size); - return st; + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = view.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; } + data.resize(old_size); + return status; } } return Status::OK(); } +Status DataTypeDateTimeV2SerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateTimeV2ParquetConsumer consumer(column, context); + RejectDateTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("DATETIMEV2 expects Parquet INT64 or INT96"); + } + DateTimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateTimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectDateTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index e089d789ccee7d..c3411ed4249843 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -90,6 +90,11 @@ class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe* value) { + DORIS_CHECK(value != nullptr); + const int64_t day_number = static_cast(encoded_date) + date_threshold; + if (day_number < 0 || !value->get_date_from_daynr(static_cast(day_number))) { + return Status::DataQualityError("Parquet DATE value is out of range"); + } + return Status::OK(); +} + +class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + explicit DateV2ParquetConsumer(IColumn& column, ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + DateV2Value value; + const auto status = decode_parquet_date( + unaligned_load(values + row * sizeof(int32_t)), &value); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = DateV2Value(); + continue; + } + _data.resize(old_size); + return status; + } + _data[old_size + row] = value; + } + return Status::OK(); + } + +private: + ColumnDateV2::Container& _data; + ParquetMaterializationState* _state; +}; + +class RejectDateV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATEV2"); + } +}; + +} // namespace Status DataTypeDateV2SerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, @@ -142,6 +194,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { @@ -149,12 +202,56 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, continue; } DateV2Value date_v2; - date_v2.get_date_from_daynr(values[row] + date_threshold); + const auto status = decode_parquet_date(values[row], &date_v2); + if (!status.ok()) { + // Decoded values back both metadata conversion and native materialization. Preserve + // strict errors while allowing nullable non-strict scans to mark only the bad row. + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } data.push_back(date_v2); } return Status::OK(); } +Status DataTypeDateV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateV2ParquetConsumer consumer(column); + RejectDateV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT32 || + context.logical_type != ParquetLogicalType::DATE) { + return Status::NotSupported("DATEV2 expects Parquet DATE stored as INT32"); + } + DateV2ParquetConsumer consumer(column, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, &state); + RejectDateV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h index bc02b61b520193..39d44efcfa0652 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datev2_serde.h @@ -88,6 +88,11 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe #include +#include +#include #include #include "arrow/type.h" @@ -35,6 +37,7 @@ #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" #include "exprs/function/cast/cast_to_decimal.h" @@ -74,8 +77,13 @@ NativeType decode_big_endian_signed_integer(const uint8_t* data, int length) { template bool decoded_decimal_value_fits(const typename PrimitiveTypeTraits::CppType::NativeType& value, UInt32 precision) { - return value >= min_decimal_value(precision).value && - value <= max_decimal_value(precision).value; + if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; + } else { + return value >= min_decimal_value(precision).value && + value <= max_decimal_value(precision).value; + } } template @@ -84,6 +92,9 @@ bool decoded_decimal_int_value_fits(Int128 value, UInt32 precision) { if constexpr (std::is_same_v) { const auto wide_value = wide::Int256(value); return decoded_decimal_value_fits(wide_value, precision); + } else if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; } else { return value >= static_cast(min_decimal_value(precision).value) && value <= static_cast(max_decimal_value(precision).value); @@ -178,6 +189,214 @@ Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& vie return Status::OK(); } +template +wide::Int256 parquet_decimal_limit(UInt32 precision) { + if constexpr (T == TYPE_DECIMALV2) { + return wide::Int256(DataTypeDecimal::get_max_digits_number(precision)); + } else { + return wide::Int256(max_decimal_value(precision).value); + } +} + +template +Status scale_parquet_decimal(wide::Int256 value, int32_t source_scale, int32_t target_scale, + UInt32 target_precision, wide::Int256* result) { + DORIS_CHECK(result != nullptr); + const auto limit = parquet_decimal_limit(target_precision); + if (source_scale > target_scale) { + const int64_t scale_delta = static_cast(source_scale) - target_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // The precision bound above guarantees that narrowing the positive scale delta keeps + // the multiplier lookup in range. + const auto divisor = + decimal_scale_multiplier(static_cast(scale_delta)); + // Scale-down must be exact. Truncating before the target precision check silently + // changes source values and makes plain and dictionary decoding disagree with casts. + if (value % divisor != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + value /= divisor; + } + } else if (source_scale < target_scale) { + const int64_t scale_delta = static_cast(target_scale) - source_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // Keep the same checked narrowing invariant for scale-up as for exact scale-down. + const auto multiplier = + decimal_scale_multiplier(static_cast(scale_delta)); + if (value > limit / multiplier || value < -limit / multiplier) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + value *= multiplier; + } + } + if (value < -limit || value > limit) { + return Status::DataQualityError("Parquet decimal value is out of range"); + } + *result = value; + return Status::OK(); +} + +template +class DecimalParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + using FieldType = typename PrimitiveTypeTraits::CppType; + using NativeType = typename FieldType::NativeType; + + DecimalParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + UInt32 target_precision, int32_t target_scale, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast&>(column).get_data()), + _context(context), + _target_precision(target_precision), + _target_scale(target_scale), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_integers(values, num_values); + } + if (_context.physical_type == ParquetPhysicalType::INT64) { + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_integers(values, num_values); + } + if (_context.physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return Status::NotSupported("Unsupported Parquet physical type {} for decimal SerDe", + static_cast(_context.physical_type)); + } + DORIS_CHECK_EQ(value_width, static_cast(_context.type_length)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = + append_binary_value(values + row * value_width, value_width, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = append_binary_value(reinterpret_cast(values[row].data), + values[row].size, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const size_t length = value_offsets[row + 1] - value_offsets[row]; + auto status = append_binary_value( + reinterpret_cast(encoded_data + payload_offsets[row]), length, + old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + template + Status append_integers(const uint8_t* values, size_t num_values) { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + constexpr int32_t SOURCE_DIGITS = std::numeric_limits::digits10 + 1; + if (_context.decimal_scale == _target_scale && + SOURCE_DIGITS <= static_cast(_target_precision)) { + // The complete physical domain fits the target at the same scale, so narrowing cannot + // precede a failure check and the hot same-scale path needs no wide arithmetic. + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = + unaligned_load(values + row * sizeof(SourceType)); + _data[old_size + row] = FieldType {NativeType(source_value)}; + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = unaligned_load(values + row * sizeof(SourceType)); + auto status = append_wide_value(wide::Int256(source_value), old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status append_binary_value(const uint8_t* value, size_t length, size_t output_row) { + if (UNLIKELY(length > sizeof(wide::Int256))) { + return Status::DataQualityError("Parquet decimal binary value is too wide: {}", length); + } + return append_wide_value( + decode_big_endian_signed_integer(value, cast_set(length)), + output_row); + } + + Status append_wide_value(wide::Int256 value, size_t output_row) { + wide::Int256 scaled_value; + RETURN_IF_ERROR(scale_parquet_decimal(value, _context.decimal_scale, _target_scale, + _target_precision, &scaled_value)); + // Narrow only after scaling and target-precision validation. In particular, an INT64 or a + // sign-extended binary value must never wrap through Decimal32 before it can be rejected. + _data[output_row] = FieldType {static_cast(scaled_value)}; + return Status::OK(); + } + + typename ColumnDecimal::Container& _data; + const ParquetDecodeContext& _context; + UInt32 _target_precision; + int32_t _target_scale; + ParquetMaterializationState* _state; +}; + } // namespace template @@ -529,6 +748,49 @@ Status DataTypeDecimalSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeDecimalSerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DecimalParquetConsumer dictionary_consumer(*state.typed_dictionary, context, + cast_set(precision), scale, &state); + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, dictionary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeDecimalSerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 9f8fd58799b753..bc79b22c417ea0 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -110,6 +110,11 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { int64_t end, const cctz::time_zone& ctz) const override; Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index af6ae9d7f43a72..abd3b082df74bf 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include #include @@ -33,6 +35,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/packed_int128.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -50,6 +53,23 @@ namespace doris { namespace { +float parquet_half_to_float(uint16_t half) { + const uint32_t sign = (half & 0x8000U) << 16; + const uint32_t exponent = (half & 0x7C00U) >> 10; + const uint32_t mantissa = half & 0x03FFU; + if (exponent == 0) { + if (mantissa == 0) { + return std::bit_cast(sign); + } + const float value = std::ldexp(static_cast(mantissa), -24); + return sign == 0 ? value : -value; + } + if (exponent == 0x1FU) { + return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); + } + return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); +} + template const NativeType* decoded_values_as(const DecodedColumnView& view) { return reinterpret_cast(view.values); @@ -77,6 +97,43 @@ bool decoded_number_value_fits(SourceType value) { } } +template +constexpr bool parquet_number_conversion_always_fits() { + if constexpr (std::is_floating_point_v) { + return true; + } else if constexpr (!std::is_integral_v || !std::is_integral_v || + std::is_same_v) { + return false; + } else if constexpr (std::is_signed_v == std::is_signed_v) { + return sizeof(DorisCppType) >= sizeof(SourceType); + } else if constexpr (std::is_signed_v) { + return sizeof(DorisCppType) > sizeof(SourceType); + } else { + return false; + } +} + +template +bool parquet_logical_integer_carrier_fits(SourceType value) { + if constexpr (sizeof(LogicalType) < sizeof(SourceType)) { + // Parquet INT(bitWidth) annotations constrain the physical INT32/INT64 carrier. Validate + // before narrowing so malformed values cannot wrap into an apparently valid logical value. + if constexpr (std::is_signed_v) { + const auto widened = static_cast(value); + if constexpr (std::is_signed_v) { + return widened >= static_cast(std::numeric_limits::lowest()) && + widened <= static_cast(std::numeric_limits::max()); + } + return widened >= 0 && + static_cast(widened) <= + static_cast(std::numeric_limits::max()); + } + const auto widened = static_cast(value); + return widened <= static_cast(std::numeric_limits::max()); + } + return true; +} + template Status read_number_decoded_values(IColumn& column, const DecodedColumnView& view) { if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { @@ -121,7 +178,21 @@ Status read_logical_integer_decoded_values_as(IColumn& column, const DecodedColu data.push_back(DorisCppType()); continue; } - const auto logical_value = static_cast(values[row]); + const auto physical_value = values[row]; + // Predicate decoding must match permissive materialization: annotated narrow integers use + // their declared bit width unless strict metadata validation explicitly requests rejection. + if (view.enable_strict_mode && + !parquet_logical_integer_carrier_fits(physical_value)) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Decoded logical integer carrier is out of range for {} at row {}", + column.get_name(), row); + } + const auto logical_value = static_cast(physical_value); if (!decoded_number_value_fits(logical_value)) { if (decoded_column_view_can_null_on_conversion_failure(view)) { decoded_column_view_insert_null_on_conversion_failure(column, view, row); @@ -178,6 +249,226 @@ Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& vie } } +template +Status append_parquet_number(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (std::is_same_v) { + // Identical fixed-width physical/logical types need no validation or conversion. Parquet + // PLAIN values and Doris POD columns share the byte representation on supported targets, + // so preserve one dense vector-at-a-time memcpy instead of converting value by value. + memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); + return Status::OK(); + } + if constexpr (parquet_number_conversion_always_fits()) { + // A widening conversion cannot fail, so keeping range checks in the row loop only blocks + // auto-vectorization. Input can be unaligned at a Parquet page boundary; load explicitly. + for (size_t row = 0; row < num_values; ++row) { + data[old_size + row] = static_cast( + unaligned_load(values + row * sizeof(SourceType))); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load(values + row * sizeof(SourceType)); + if (!decoded_number_value_fits(value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet value is out of range at row {}", row); + } + data[old_size + row] = static_cast(value); + } + return Status::OK(); +} + +template +Status append_parquet_logical_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (parquet_number_conversion_always_fits() && + sizeof(LogicalType) >= sizeof(SourceType)) { + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = + unaligned_load(values + row * sizeof(SourceType)); + data[old_size + row] = + static_cast(static_cast(physical_value)); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); + // Permissive scans preserve the long-standing bit-width interpretation of annotated + // carriers; strict scans still reject malformed physical values before narrowing. + if ((state == nullptr || state->enable_strict_mode) && + !parquet_logical_integer_carrier_fits(physical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Parquet logical integer carrier is out of range at row {}", row); + } + const auto logical_value = static_cast(physical_value); + if (!decoded_number_value_fits(logical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet logical integer is out of range at row {}", + row); + } + data[old_size + row] = static_cast(logical_value); + } + return Status::OK(); +} + +template +Status append_parquet_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + if (context.logical_integer_bit_width <= 0) { + return append_parquet_number(data, values, num_values, context, + state); + } + if (context.logical_integer_is_signed) { + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers( + data, values, num_values, state); + case 16: + return append_parquet_logical_integers( + data, values, num_values, state); + case 32: + return append_parquet_logical_integers( + data, values, num_values, state); + case 64: + return append_parquet_logical_integers( + data, values, num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } + } + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers(data, values, + num_values, state); + case 16: + return append_parquet_logical_integers(data, values, + num_values, state); + case 32: + return append_parquet_logical_integers(data, values, + num_values, state); + case 64: + return append_parquet_logical_integers(data, values, + num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } +} + +template +class NumberParquetConsumer final : public ParquetFixedValueConsumer { +public: + using DorisCppType = typename PrimitiveTypeTraits::CppType; + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + + NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return consume_impl(values, num_values, value_width); + } + + Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) override { + // Each selected PLAIN range is already contiguous in the encoded page. Append those spans + // directly so sparse reads never build an intermediate selected-values array. + for (const auto& range : ranges) { + RETURN_IF_ERROR( + consume_impl(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } + +private: + Status consume_impl(const uint8_t* values, size_t num_values, size_t value_width) { + if (_context.logical_float16) { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY); + DORIS_CHECK_EQ(value_width, sizeof(uint16_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const float value = parquet_half_to_float( + unaligned_load(values + row * sizeof(uint16_t))); + if (!decoded_number_value_fits(value)) { + if (_state != nullptr && _state->can_insert_null_on_conversion_failure()) { + _data[old_size + row] = DorisCppType(); + DORIS_CHECK(_state->mark_conversion_failure(old_size + row)); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet FLOAT16 value is out of range at row {}", row); + } + _data[old_size + row] = static_cast(value); + } + return Status::OK(); + } + switch (_context.physical_type) { + case ParquetPhysicalType::BOOLEAN: + DORIS_CHECK_EQ(value_width, sizeof(uint8_t)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::INT32: + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::INT64: + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::FLOAT: + DORIS_CHECK_EQ(value_width, sizeof(float)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::DOUBLE: + DORIS_CHECK_EQ(value_width, sizeof(double)); + return append_parquet_number(_data, values, num_values, _context, + _state); + default: + return Status::NotSupported("Unsupported Parquet physical type {} for numeric SerDe", + static_cast(_context.physical_type)); + } + } + + PaddedPODArray& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectParquetBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as a number"); + } +}; + } // namespace // Basic structure of the type map. template @@ -329,6 +620,52 @@ Status DataTypeNumberSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeNumberSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_parquet_dictionary(column, source, context); + } else { + NumberParquetConsumer consumer(column, context); + RejectParquetBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); + } +} + +template +Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_column_from_parquet(column, source, context, num_values, state); + } else { + NumberParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + NumberParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectParquetBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); + } +} + template Status DataTypeNumberSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 16c4ccd48e5c8a..f2cf7d26200f34 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -120,6 +120,11 @@ class DataTypeNumberSerDe : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index da5db282337883..47e2c2ab983ae8 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -52,6 +52,7 @@ #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/data_type_timestamptz_serde.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/field.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -710,6 +711,18 @@ Status DataTypeSerDe::read_column_from_decoded_values(IColumn& column, get_name())); } +Status DataTypeSerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + return Status::NotSupported("read_column_from_parquet is not supported for {}", get_name()); +} + +Status DataTypeSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + return Status::NotSupported("read_parquet_dictionary is not supported for {}", get_name()); +} + Status DataTypeSerDe::read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const { return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index 2c32504984a237..70ae8e4758b4fa 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -94,6 +94,9 @@ struct CastParameters; class DataTypeSerDe; using DataTypeSerDeSPtr = std::shared_ptr; using DataTypeSerDeSPtrs = std::vector; +class ParquetDecodeSource; +struct ParquetDecodeContext; +struct ParquetMaterializationState; /// Info that represents a scalar or array field in a decomposed view. /// It allows to recreate field with different number @@ -503,6 +506,18 @@ class DataTypeSerDe { // the Doris-type-specific materialization into IColumn. virtual Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const; + + // Read encoded Parquet values directly into the destination Doris column. ColumnReader owns + // levels/null/filter handling; the source owns only encoding-stream state; the target SerDe + // owns all physical/logical type interpretation and materialization. + virtual Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const; + // Decode one dictionary page into the selected Doris type without consuming data-page + // indices. Dictionary filters use this to keep type interpretation in SerDe instead of + // exposing dictionary bytes or decoder-owned strings to ColumnReader. + virtual Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const; virtual Status read_field_from_decoded_value(const IDataType& data_type, Field* field, const DecodedColumnView& view) const; diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 80cf74634f0a16..1461c1d8746470 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -17,14 +17,18 @@ #include "core/data_type_serde/data_type_string_serde.h" +#include #include #include +#include #include "common/config.h" #include "core/column/column_string.h" +#include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -57,6 +61,65 @@ Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view return Status::OK(); } +template +class StringParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + explicit StringParquetConsumer(IColumn& column) : _column(assert_cast(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_fixed_length_data(static_cast(nullptr), + size_t(), size_t()); + }) { + // FIXED_LEN_BYTE_ARRAY is already a dense byte span. Copy it once and synthesize + // offsets; StringRef batches add a second row loop and hundreds of tiny memcpy calls. + _column.insert_many_fixed_length_data(reinterpret_cast(values), + value_width, num_values); + } else { + static constexpr size_t BATCH_SIZE = 256; + std::array refs; + size_t offset = 0; + while (offset < num_values) { + const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); + for (size_t row = 0; row < batch_size; ++row) { + refs[row] = StringRef( + reinterpret_cast(values + (offset + row) * value_width), + value_width); + } + _column.insert_many_strings(refs.data(), batch_size); + offset += batch_size; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + _column.insert_many_strings(values, num_values); + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_parquet_plain_byte_arrays( + encoded_data, payload_offsets, value_offsets, num_values, + value_spans); + }) { + _column.insert_many_parquet_plain_byte_arrays(encoded_data, payload_offsets, + value_offsets, num_values, value_spans); + return Status::OK(); + } + return ParquetBinaryValueConsumer::consume_plain_byte_array( + encoded_data, payload_offsets, value_offsets, num_values, value_spans); + } + +private: + ColumnType& _column; +}; + } // namespace namespace { @@ -497,6 +560,57 @@ Status DataTypeStringSerDeBase::read_column_from_decoded_values( return read_string_decoded_values(column, view); } +template +Status DataTypeStringSerDeBase::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + StringParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeStringSerDeBase::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + auto& indices = assert_cast(column).get_data(); + const size_t old_size = indices.size(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + StringParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type {} for string SerDe", + static_cast(context.physical_type)); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeStringSerDeBase::write_column_to_orc( const std::string& timezone, const IColumn& column, const NullMap* null_map, diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index 6c64b1276c3afb..6389b6be92546e 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -206,6 +206,11 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index c40e671793c848..52d387eda5ca6d 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -17,13 +17,17 @@ #include "core/data_type_serde/data_type_time_serde.h" +#include + #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" +#include "util/unaligned.h" namespace doris { namespace { @@ -51,6 +55,74 @@ TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64 abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); } +class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + int64_t raw_value; + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + raw_value = unaligned_load(values + row * sizeof(int32_t)); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + raw_value = unaligned_load(values + row * sizeof(int64_t)); + } + + int64_t units_per_day; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + units_per_day = 86400000; + } else if (_context.time_unit == ParquetTimeUnit::MICROS) { + units_per_day = 86400000000; + } else { + DORIS_CHECK(_context.time_unit == ParquetTimeUnit::NANOS); + units_per_day = 86400000000000; + } + // Validate the declared carrier before rescaling: truncating nanoseconds first could + // turn a value at or beyond 24:00:00 into an apparently valid TIMEV2 value. + if (raw_value < 0 || raw_value >= units_per_day) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = TimeValue::TimeType(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet TIME value {} is outside the one-day domain", raw_value); + } + int64_t micros = raw_value; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + micros /= 1000; + } + // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields + // and immediately recombining them is an identity operation with several divisions. + _data[old_size + row] = static_cast(micros); + } + return Status::OK(); + } + +private: + ColumnTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMEV2"); + } +}; + } // namespace Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, @@ -193,6 +265,41 @@ Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeTimeV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + TimeV2ParquetConsumer consumer(column, context); + RejectTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if ((context.physical_type != ParquetPhysicalType::INT32 && + context.physical_type != ParquetPhysicalType::INT64) || + context.logical_type != ParquetLogicalType::TIME) { + return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); + } + TimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnType& int_col, ColumnNullable& target_col) const { diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h index c3b8ab41a0fdd5..c7f3e15383f54f 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.h +++ b/be/src/core/data_type_serde/data_type_time_serde.h @@ -70,6 +70,11 @@ class DataTypeTimeV2SerDe : public DataTypeNumberSerDe(micros_of_second)); + if (!timestamp_tz.is_valid_date()) { + return Status::DataQualityError( + "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } data.push_back(timestamp_tz); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class TimestampTzParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimestampTzParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnTimeStampTz::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimestampTzBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMPTZ"); + } +}; + } // namespace // The implementation of these functions mainly refers to data_type_datetimev2_serde.cpp @@ -308,14 +355,27 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, values[row].to_timestamp_micros()); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -326,11 +386,57 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, decoded_timestamp_micros(view, values[row])); + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } +Status DataTypeTimeStampTzSerDe::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + TimestampTzParquetConsumer consumer(column, context); + RejectTimestampTzBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeStampTzSerDe::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("TIMESTAMPTZ expects Parquet INT64 or INT96"); + } + TimestampTzParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimestampTzParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimestampTzBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { return CastToString::from_timestamptz(field.get(), 6); } diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.h b/be/src/core/data_type_serde/data_type_timestamptz_serde.h index 23d57f57fc8dac..6777459909a090 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.h @@ -75,6 +75,11 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(reinterpret_cast(values + row * value_width), + value_width); + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(values[row].data, values[row].size); + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return Status::OK(); + } + +private: + ColumnVarbinary& _column; +}; + +} // namespace + +Status DataTypeVarbinarySerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + VarbinaryParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +Status DataTypeVarbinarySerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + VarbinaryParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type for VARBINARY"); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} void DataTypeVarbinarySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& arena, int32_t col_id, int64_t row_num, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index a3ce30f6f7dac8..cf4d33a99de11e 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -82,6 +82,12 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe { "read_column_from_arrow with type " + column.get_name()); } + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h new file mode 100644 index 00000000000000..e9762385689d07 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/string_ref.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { + +// These enums deliberately do not expose parquet thrift classes to the core type system. The +// format reader translates the thrift metadata once when it creates a column reader. +enum class ParquetPhysicalType { + BOOLEAN, + INT32, + INT64, + INT96, + FLOAT, + DOUBLE, + BYTE_ARRAY, + FIXED_LEN_BYTE_ARRAY, +}; + +enum class ParquetValueEncoding { + PLAIN, + DICTIONARY, + RLE, + BIT_PACKED, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, + BYTE_STREAM_SPLIT, +}; + +enum class ParquetTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +enum class ParquetLogicalType { + NONE, + STRING, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + INTEGER, + UUID, + FLOAT16, +}; + +// Immutable metadata required to turn one Parquet physical value into the selected Doris type. +// Encoding describes how the value source is read; logical annotations describe its meaning. +struct ParquetDecodeContext { + ParquetPhysicalType physical_type = ParquetPhysicalType::INT32; + ParquetValueEncoding encoding = ParquetValueEncoding::PLAIN; + ParquetLogicalType logical_type = ParquetLogicalType::NONE; + ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; + + int32_t type_length = -1; + int32_t decimal_precision = -1; + int32_t decimal_scale = -1; + int32_t logical_integer_bit_width = -1; + bool logical_integer_is_signed = true; + bool timestamp_is_adjusted_to_utc = false; + bool logical_float16 = false; + bool logical_uuid = false; + bool dictionary_index_only = false; + + const cctz::time_zone* timezone = nullptr; +}; + +struct ParquetSelectionRange { + size_t first = 0; + size_t count = 0; +}; + +// A decoder may produce multiple contiguous spans for one request (for example delta encodings). +// Consumers are invoked per span, never per value, keeping virtual dispatch out of the row loop. +class ParquetFixedValueConsumer { +public: + virtual ~ParquetFixedValueConsumer() = default; + virtual Status consume(const uint8_t* values, size_t num_values, size_t value_width) = 0; + virtual Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) { + for (const auto& range : ranges) { + RETURN_IF_ERROR(consume(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } +}; + +class ParquetBinaryValueConsumer { +public: + virtual ~ParquetBinaryValueConsumer() = default; + virtual Status consume(const StringRef* values, size_t num_values) = 0; + + // PLAIN BYTE_ARRAY decoders already have to parse every length prefix. Publish the parsed + // source and destination offsets so string columns do not rebuild an equally large StringRef + // array and rescan all lengths before copying. Spans are expressed in output coordinates and + // preserve adjacent surviving runs for consumers that can amortize range setup. + virtual Status consume_plain_byte_array(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) { + std::vector values; + values.reserve(num_values); + for (size_t row = 0; row < num_values; ++row) { + values.emplace_back(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return consume(values.data(), values.size()); + } +}; + +// Dictionary decoders publish validated IDs without knowing the destination Doris type. A +// cache-resident dictionary can therefore fuse RLE decode with target-column gathering, while a +// large sparse dictionary can still choose a compact ID buffer before random dictionary access. +class ParquetDictionaryValueConsumer { +public: + virtual ~ParquetDictionaryValueConsumer() = default; + virtual Status consume_indices(const uint32_t* indices, size_t num_values) = 0; + virtual Status consume_repeated(uint32_t index, size_t num_values) { + std::vector indices(num_values, index); + return consume_indices(indices.data(), indices.size()); + } +}; + +// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs do +// not occupy the encoded value stream, so the native ColumnReader merges the logical selection +// with definition levels before constructing this plan. The SerDe sees only selected non-NULL +// payload, while the reader restores selected NULL slots after the compact decode. Ranges are +// sorted, disjoint, and expressed in the physical value stream's coordinate space. +struct ParquetSelection { + size_t total_values = 0; + size_t selected_values = 0; + std::vector ranges; +}; + +// Encoding decoders implement this interface. They own encoded-stream cursors and dictionary +// storage, but they never know the destination Doris column type. DataTypeSerDe owns the consumer +// and therefore the physical/logical-to-Doris conversion. +class ParquetDecodeSource { +public: + virtual ~ParquetDecodeSource() = default; + + virtual Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) = 0; + virtual Status decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) = 0; + virtual Status skip_values(size_t num_values) = 0; + + // Batch-level sparse decode. The default implementation preserves every encoding's cursor + // semantics while moving SerDe dispatch and consumer construction out of the selection-run + // loop. Decoders with cheap random access or batch decode override these methods to remove the + // remaining per-range virtual calls as well. + virtual Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_fixed_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + virtual Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_binary_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + + virtual bool has_dictionary() const { return false; } + virtual uint64_t dictionary_generation() const { return 0; } + virtual size_t dictionary_size() const { return 0; } + virtual Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return Status::NotSupported("Parquet dictionary is not supported by this decoder"); + } + virtual Status decode_dictionary_indices(size_t num_values, std::vector* indices) { + return Status::NotSupported("Parquet dictionary indices are not supported by this decoder"); + } + virtual Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) { + DORIS_CHECK(indices != nullptr); + indices->clear(); + indices->reserve(selection.selected_values); + std::vector range_indices; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_dictionary_indices(range.count, &range_indices)); + indices->insert(indices->end(), range_indices.begin(), range_indices.end()); + cursor = range.first + range.count; + } + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(indices->size(), selection.selected_values); + return Status::OK(); + } + virtual Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_dictionary_indices(num_values, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_selected_dictionary_indices(selection, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const { + return false; + } +}; + +enum class ParquetDictionaryMaterializationStrategy : uint8_t { DIRECT, INDICES }; + +// Dictionary values are materialized once into the selected Doris type. The state belongs to a +// column reader rather than DataTypeSerDe because a SerDe instance can be shared by many files. +struct ParquetMaterializationState { + MutableColumnPtr typed_dictionary; + std::vector dictionary_indices; + ParquetSelection selection; + uint64_t dictionary_generation = std::numeric_limits::max(); + bool enable_strict_mode = false; + IColumn::Filter* conversion_failure_null_map = nullptr; + IColumn::Filter dictionary_conversion_failures; + bool capturing_dictionary_conversion_failures = false; + bool dictionary_has_conversion_failures = false; + size_t dictionary_failure_scan_rows = 0; + ParquetDictionaryMaterializationStrategy dictionary_materialization_strategy = + ParquetDictionaryMaterializationStrategy::INDICES; + + void reset_dictionary() { + typed_dictionary.reset(); + dictionary_indices.clear(); + dictionary_conversion_failures.clear(); + capturing_dictionary_conversion_failures = false; + dictionary_has_conversion_failures = false; + dictionary_failure_scan_rows = 0; + dictionary_generation = std::numeric_limits::max(); + } + + bool can_insert_null_on_conversion_failure() const { + return conversion_failure_null_map != nullptr && + (!enable_strict_mode || capturing_dictionary_conversion_failures); + } + + bool mark_conversion_failure(size_t output_row) { + if (!can_insert_null_on_conversion_failure()) { + return false; + } + DORIS_CHECK_LT(output_row, conversion_failure_null_map->size()); + (*conversion_failure_null_map)[output_row] = 1; + if (capturing_dictionary_conversion_failures) { + dictionary_has_conversion_failures = true; + } + return true; + } + + IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { + auto* output_null_map = conversion_failure_null_map; + dictionary_conversion_failures.resize_fill(dictionary_size, 0); + dictionary_has_conversion_failures = false; + conversion_failure_null_map = &dictionary_conversion_failures; + capturing_dictionary_conversion_failures = true; + return output_null_map; + } + + void end_dictionary_conversion(IColumn::Filter* output_null_map) { + conversion_failure_null_map = output_null_map; + capturing_dictionary_conversion_failures = false; + } + + Status materialize_dictionary(IColumn& column) { + const size_t old_size = column.size(); + dictionary_failure_scan_rows = 0; + if (UNLIKELY(dictionary_has_conversion_failures)) { + const bool insert_failure_as_null = can_insert_null_on_conversion_failure(); + if (!insert_failure_as_null) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] == 0) { + continue; + } + // A malformed dictionary entry is irrelevant until a selected row references + // it; failing while building the dictionary would reject valid pages. + return Status::DataQualityError( + "Parquet dictionary entry {} cannot be converted to the target type", + dictionary_id); + } + } + } + column.insert_indices_from(*typed_dictionary, dictionary_indices.data(), + dictionary_indices.data() + dictionary_indices.size()); + if (UNLIKELY(dictionary_has_conversion_failures && + can_insert_null_on_conversion_failure())) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] != 0) { + mark_conversion_failure(old_size + row); + } + } + } + return Status::OK(); + } + + Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, size_t num_values) { + DORIS_CHECK(typed_dictionary); + if (UNLIKELY(dictionary_has_conversion_failures) || + source.prefer_dictionary_index_materialization(typed_dictionary->byte_size())) { + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::INDICES; + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &dictionary_indices)); + DORIS_CHECK_EQ(dictionary_indices.size(), num_values); + return materialize_dictionary(column); + } + + class ColumnConsumer final : public ParquetDictionaryValueConsumer { + public: + ColumnConsumer(IColumn& destination, const IColumn& dictionary) + : _destination(destination), _dictionary(dictionary) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + _destination.insert_indices_from(_dictionary, indices, indices + num_values); + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + _destination.insert_many_from(_dictionary, index, num_values); + return Status::OK(); + } + + private: + IColumn& _destination; + const IColumn& _dictionary; + } consumer(column, *typed_dictionary); + + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::DIRECT; + const size_t old_size = column.size(); + const Status status = source.decode_dictionary_values(num_values, consumer); + if (!status.ok()) { + // Streaming direct gather may discover a corrupt late ID after earlier valid runs; + // restore the same all-or-nothing column invariant as the index-buffer path. + column.resize(old_size); + } + return status; + } +}; + +} // namespace doris diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h new file mode 100644 index 00000000000000..ba2aa686ad4272 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { + +#pragma pack(1) +struct ParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(ParquetInt96Timestamp) == 12); + +inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL; +inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL; + +inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet timestamp is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } + return Status::OK(); +} + +inline Status parquet_timestamp_micros(ParquetTimeUnit unit, int64_t value, int64_t* result) { + if (unit == ParquetTimeUnit::MILLIS) { + // Validate in the source unit before scaling; signed overflow could otherwise turn an + // invalid file value into an in-range timestamp that survives later range checks. + if (value > std::numeric_limits::max() / 1000 || + value < std::numeric_limits::min() / 1000) { + return Status::DataQualityError("Parquet timestamp overflows microseconds"); + } + *result = value * 1000; + } else if (unit == ParquetTimeUnit::NANOS) { + *result = value / 1000; + // Epoch-relative negative timestamps must round toward the preceding representable + // microsecond; C++ division truncates toward zero and would move pre-epoch values forward. + if (value < 0 && value % 1000 != 0) { + --*result; + } + } else { + *result = value; + } + return validate_parquet_timestamp_micros(*result); +} + +inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value, int64_t* result) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000LL; + static constexpr int64_t NANOS_IN_DAY = 86400000000000LL; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + // INT96 nanos is a time-of-day, not a signed duration. Validate it before widened day + // arithmetic so corrupt input cannot wrap into a plausible Doris timestamp. + if (value.nanos_of_day < 0 || value.nanos_of_day >= NANOS_IN_DAY) { + return Status::DataQualityError("Invalid Parquet INT96 nanos-of-day: {}", + value.nanos_of_day); + } + const __int128 days = static_cast(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + const __int128 timestamp_micros = + days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND; + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet INT96 timestamp is outside the Doris 0001-9999 range"); + } + *result = static_cast(timestamp_micros); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index ae2b84cc98e72e..f3b209ac2dd540 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -73,17 +73,34 @@ bool ScanLocalState::should_run_serial() const { return _parent->cast()._should_run_serial; } -Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, - int& arrived_rf_num) { +Status ScanLocalStateBase::update_late_arrival_runtime_filter( + RuntimeState* state, int applied_rf_num, int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts) { // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads LockGuard lock(_conjuncts_lock); + arrived_conjuncts.clear(); + size_t conjuncts_before = _conjuncts.size(); RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter(state, _parent->row_descriptor(), arrived_rf_num, _conjuncts)); + if (_conjuncts.size() > conjuncts_before) { + VExprContextSPtrs appended(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + _late_arrival_conjunct_batches.emplace_back(arrived_rf_num, std::move(appended)); + } if (state->enable_adjust_conjunct_order_by_cost()) { std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { return a->execute_cost() < b->execute_cost(); }); }; + for (const auto& [batch_arrived_rf_num, batch] : _late_arrival_conjunct_batches) { + if (batch_arrived_rf_num <= applied_rf_num) { + continue; + } + for (const auto& conjunct : batch) { + VExprContextSPtr cloned; + RETURN_IF_ERROR(conjunct->clone(state, cloned)); + arrived_conjuncts.push_back(std::move(cloned)); + } + } return Status::OK(); } @@ -1011,6 +1028,12 @@ TPushAggOp::type ScanLocalState::get_push_down_agg_type() { return _parent->cast()._push_down_agg_type; } +template +const std::optional>& ScanLocalState::get_push_down_count_slot_ids() + const { + return _parent->cast()._push_down_count_slot_ids; +} + template int64_t ScanLocalState::limit_per_scanner() { return _parent->cast()._limit_per_scanner; @@ -1164,6 +1187,9 @@ Status ScanOperatorX::init(const TPlanNode& tnode, RuntimeState* } else { _push_down_agg_type = TPushAggOp::type::NONE; } + if (tnode.__isset.push_down_count_slot_ids) { + _push_down_count_slot_ids = tnode.push_down_count_slot_ids; + } if (tnode.__isset.topn_filter_source_node_ids) { _topn_filter_source_node_ids = tnode.topn_filter_source_node_ids; diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index 931917482736a6..a9a1f3f90f1dd9 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -18,8 +18,11 @@ #pragma once #include +#include #include #include +#include +#include #include "common/status.h" #include "common/thread_safety_annotations.h" @@ -73,6 +76,7 @@ class ScanLocalStateBase : public PipelineXLocalState<> { virtual void set_scan_ranges(RuntimeState* state, const std::vector& scan_ranges) = 0; virtual TPushAggOp::type get_push_down_agg_type() = 0; + virtual const std::optional>& get_push_down_count_slot_ids() const = 0; // If scan operator is serial operator(like topn), its real parallelism is 1. // Otherwise, its real parallelism is query_parallel_instance_num. @@ -87,11 +91,13 @@ class ScanLocalStateBase : public PipelineXLocalState<> { [[nodiscard]] std::string get_name() { return _parent->get_name(); } - Status update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num); + uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } - Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts); + Status update_late_arrival_runtime_filter(RuntimeState* state, int applied_rf_num, + int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts); - uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } + Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts); protected: friend class ScannerContext; @@ -128,6 +134,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { AnnotatedMutex _conjuncts_lock; RuntimeFilterConsumerHelper _helper; + // Preserve append identity independently of the cost-sorted operator snapshot. Every scanner + // needs the exact RF contexts added since its own applied count. + std::vector> _late_arrival_conjunct_batches; // magic number as seed to generate hash value for condition cache uint64_t _condition_cache_digest = 0; @@ -231,6 +240,7 @@ class ScanLocalState : public ScanLocalStateBase { const std::vector& scan_ranges) override {} TPushAggOp::type get_push_down_agg_type() override; + const std::optional>& get_push_down_count_slot_ids() const override; std::vector execution_dependencies() override { if (_filter_dependencies.empty()) { @@ -409,6 +419,16 @@ class ScanOperatorX : public OperatorX { TPushAggOp::type _push_down_agg_type; + // Semantic arguments of a pushed-down COUNT. This is deliberately optional because absence + // and an empty list have different meanings during a BE-first rolling upgrade: + // + // - nullopt: an old FE did not send the field, so the new BE must use the normal scan; + // - empty: the new FE explicitly planned COUNT(*)/COUNT(1); + // - non-empty: the new FE explicitly planned COUNT(col). + // + // Treating nullopt as empty would silently reinterpret an old plan as COUNT(*). + std::optional> _push_down_count_slot_ids; + // Record the value of the aggregate function 'count' from doris's be int64_t _push_down_count = -1; const int _parallel_tasks = 0; diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index b215212b6d861b..1c48b9285d9eda 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,10 +87,18 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; + // The presence bit is part of the mapping contract: an explicit empty mapping must remain + // authoritative after access-path pruning instead of enabling current-name fallback. + column->has_name_mapping = schema_column->has_name_mapping; + // Initial defaults describe the logical value of fields absent from older files. Nested + // access-path pruning must retain them just like it retains rename metadata. + column->initial_default_value = schema_column->initial_default_value; + column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( - const format::ColumnDefinition* schema_column, const std::string& child_path) { + const format::ColumnDefinition* schema_column, const std::string& child_path, + bool prefer_exact_name_match) { if (schema_column == nullptr) { return nullptr; } @@ -103,15 +111,31 @@ const format::ColumnDefinition* find_schema_child_by_path( }); return child_it == schema_column->children.end() ? nullptr : &*child_it; } - const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { - if (to_lower(child.name) == to_lower(child_path)) { - return true; - } + if (!prefer_exact_name_match) { + const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + if (to_lower(child.name) == to_lower(child_path)) { + return true; + } + return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(child_path); + }); + }); + return child_it == schema_column->children.end() ? nullptr : &*child_it; + } + // Iceberg can reuse a historical name for a newly added sibling. Current names therefore + // have precedence across the entire struct; an earlier alias must not steal that access path. + const auto exact_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + return to_lower(child.name) == to_lower(child_path); + }); + if (exact_it != schema_column->children.end()) { + return &*exact_it; + } + const auto alias_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { return to_lower(alias) == to_lower(child_path); }); }); - return child_it == schema_column->children.end() ? nullptr : &*child_it; + return alias_it == schema_column->children.end() ? nullptr : &*alias_it; } int32_t schema_field_id(const format::ColumnDefinition* schema_column) { @@ -169,7 +193,8 @@ void insert_access_path(AccessPathNode* root, const std::vector& pa Status build_nested_children_from_access_node(format::ColumnDefinition* column, const DataTypePtr& type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match); // Expand a full complex-column projection into table-schema children when the table format provides // an external/current schema. Without this, `SELECT complex_col` or `SELECT *` leaves @@ -186,7 +211,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, // wrapper here: ColumnMapper and TableReader treat MAP children as [key, value]. Status build_all_nested_children_from_schema(format::ColumnDefinition* column, const DataTypePtr& type, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); const auto nested_type = remove_nullable(type); @@ -197,14 +223,16 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, const auto& struct_type = assert_cast(*nested_type); for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) { const auto field_name = struct_type.get_element_name(field_idx); - const auto* schema_child = find_schema_child_by_path(schema_column, field_name); + const auto* schema_child = + find_schema_child_by_path(schema_column, field_name, prefer_exact_name_match); auto* child = find_or_add_child( column, schema_field_id_or(schema_child, cast_set(field_idx)), schema_field_name_or(schema_child, field_name), struct_type.get_element(field_idx)); inherit_schema_metadata(child, schema_child); RETURN_IF_ERROR(build_nested_children_from_access_node( - child, child->type, project_all, path + "." + child->name, schema_child)); + child, child->type, project_all, path + "." + child->name, schema_child, + prefer_exact_name_match)); } return Status::OK(); } @@ -217,7 +245,7 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, array_type.get_nested_type()); inherit_schema_metadata(child, element_schema); return build_nested_children_from_access_node(child, child->type, project_all, path + ".*", - element_schema); + element_schema, prefer_exact_name_match); } case TYPE_MAP: { const auto& map_type = assert_cast(*nested_type); @@ -231,12 +259,14 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, map_type.get_key_type()); inherit_schema_metadata(key_child, key_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - key_child, key_child->type, project_all, path + ".KEYS", key_schema)); + key_child, key_child->type, project_all, path + ".KEYS", key_schema, + prefer_exact_name_match)); auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", map_type.get_value_type()); inherit_schema_metadata(value_child, value_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - value_child, value_child->type, project_all, path + ".VALUES", value_schema)); + value_child, value_child->type, project_all, path + ".VALUES", value_schema, + prefer_exact_name_match)); return Status::OK(); } default: @@ -247,7 +277,8 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, Status build_struct_children_from_access_node(format::ColumnDefinition* column, const DataTypeStruct& struct_type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); for (const auto& [child_path, child_node] : node.children) { // Struct children are resolved by name or schema field id. We do not treat a numeric @@ -262,7 +293,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, // Prefer the table/schema ColumnDefinition because it carries field ids and aliases. // Fallback to the struct type name only for formats without external schema metadata. - const auto* schema_child = find_schema_child_by_path(schema_column, child_path); + const auto* schema_child = + find_schema_child_by_path(schema_column, child_path, prefer_exact_name_match); int32_t field_id = schema_field_id(schema_child); std::string field_name = schema_child == nullptr ? child_path : schema_child->name; DataTypePtr field_type = schema_child == nullptr ? nullptr : schema_child->type; @@ -288,7 +320,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, auto* child = find_or_add_child(column, field_id, field_name, field_type); inherit_schema_metadata(child, schema_child); RETURN_IF_ERROR(build_nested_children_from_access_node( - child, child->type, child_node, path + "." + child_path, schema_child)); + child, child->type, child_node, path + "." + child_path, schema_child, + prefer_exact_name_match)); } return Status::OK(); } @@ -296,7 +329,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, Status build_map_children_from_access_node(format::ColumnDefinition* column, const DataTypeMap& map_type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); AccessPathNode key_node; AccessPathNode value_node; @@ -368,14 +402,16 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column, map_type.get_key_type()); inherit_schema_metadata(key_child, key_schema); RETURN_IF_ERROR(build_nested_children_from_access_node(key_child, key_child->type, key_node, - path + ".KEYS", key_schema)); + path + ".KEYS", key_schema, + prefer_exact_name_match)); } if (need_value) { auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", map_type.get_value_type()); inherit_schema_metadata(value_child, value_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - value_child, value_child->type, value_node, path + ".VALUES", value_schema)); + value_child, value_child->type, value_node, path + ".VALUES", value_schema, + prefer_exact_name_match)); } return Status::OK(); } @@ -383,18 +419,20 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column, Status build_nested_children_from_access_node(format::ColumnDefinition* column, const DataTypePtr& type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); if (node.project_all || node.children.empty()) { - return build_all_nested_children_from_schema(column, type, path, schema_column); + return build_all_nested_children_from_schema(column, type, path, schema_column, + prefer_exact_name_match); } const auto nested_type = remove_nullable(type); switch (nested_type->get_primitive_type()) { case TYPE_STRUCT: return build_struct_children_from_access_node( - column, assert_cast(*nested_type), node, path, - schema_column); + column, assert_cast(*nested_type), node, path, schema_column, + prefer_exact_name_match); case TYPE_ARRAY: { if (node.children.size() != 1 || !node.children.contains("*")) { return Status::NotSupported( @@ -409,11 +447,13 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, array_type.get_nested_type()); inherit_schema_metadata(child, element_schema); return build_nested_children_from_access_node(child, child->type, node.children.at("*"), - path + ".*", element_schema); + path + ".*", element_schema, + prefer_exact_name_match); } case TYPE_MAP: return build_map_children_from_access_node( - column, assert_cast(*nested_type), node, path, schema_column); + column, assert_cast(*nested_type), node, path, schema_column, + prefer_exact_name_match); default: return Status::NotSupported("AccessPathParser does not support access path {} for slot {}", path, column->name); @@ -424,7 +464,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, const std::vector& access_paths, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); if (is_scanner_materialized_virtual_column(column->name)) { return Status::OK(); @@ -465,15 +506,17 @@ Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, } // Recursively build nested children for the column based on the AccessPathNode tree. return build_nested_children_from_access_node(column, column->type, root, column->name, - schema_column); + schema_column, prefer_exact_name_match); } Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, const SlotDescriptor* slot_desc, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); DORIS_CHECK(slot_desc != nullptr); - return build_nested_children(column, slot_desc->all_access_paths(), schema_column); + return build_nested_children(column, slot_desc->all_access_paths(), schema_column, + prefer_exact_name_match); } } // namespace doris diff --git a/be/src/exec/scan/access_path_parser.h b/be/src/exec/scan/access_path_parser.h index 1aa4c5b89d492a..0be785a33906e8 100644 --- a/be/src/exec/scan/access_path_parser.h +++ b/be/src/exec/scan/access_path_parser.h @@ -31,11 +31,13 @@ class AccessPathParser { public: static Status build_nested_children(format::ColumnDefinition* column, const SlotDescriptor* slot_desc, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match = true); static Status build_nested_children(format::ColumnDefinition* column, const std::vector& access_paths, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match = true); }; } // namespace doris diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 2117bc3873b6ff..5c7beb3e87e268 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -68,6 +68,7 @@ #include "format/table/hive_reader.h" #include "format/table/hudi_jni_reader.h" #include "format/table/hudi_reader.h" +#include "format/table/iceberg_position_delete_sys_table_reader.h" #include "format/table/iceberg_reader.h" #include "format/table/iceberg_sys_table_jni_reader.h" #include "format/table/lakesoul_jni_reader.h" @@ -99,6 +100,20 @@ namespace doris { #include "common/compile_check_begin.h" using namespace ErrorCode; +namespace { +constexpr int kIcebergPositionDeleteContent = 1; +constexpr int kIcebergDeletionVectorContent = 3; + +bool is_iceberg_position_deletes_sys_table(const TFileRangeDesc& range) { + return range.__isset.table_format_params && + range.table_format_params.table_format_type == "iceberg" && + range.table_format_params.__isset.iceberg_params && + range.table_format_params.iceberg_params.__isset.content && + (range.table_format_params.iceberg_params.content == kIcebergPositionDeleteContent || + range.table_format_params.iceberg_params.content == kIcebergDeletionVectorContent); +} +} // namespace + const std::string FileScanner::FileReadBytesProfile = "FileReadBytes"; const std::string FileScanner::FileReadTimeProfile = "FileReadTime"; @@ -1060,6 +1075,7 @@ Status FileScanner::_get_next_reader() { // create reader for specific format Status init_status = Status::OK(); TFileFormatType::type format_type = _get_current_format_type(); + const bool is_position_deletes_sys_table = is_iceberg_position_deletes_sys_table(range); // for compatibility, this logic is deprecated in 3.1 if (format_type == TFileFormatType::FORMAT_JNI && range.__isset.table_format_params) { if (range.table_format_params.table_format_type == "paimon" && @@ -1149,9 +1165,18 @@ Status FileScanner::_get_next_reader() { auto file_meta_cache_ptr = _should_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache() : nullptr; + if (is_position_deletes_sys_table) { + auto reader = IcebergPositionDeleteSysTableReader::create_unique( + _file_slot_descs, _state, _profile, range, _params, _io_ctx, + file_meta_cache_ptr); + init_status = reader->init_reader(); + _cur_reader = std::move(reader); + need_to_get_parsed_schema = false; + break; + } std::unique_ptr parquet_reader = ParquetReader::create_unique( _profile, *_params, range, _state->query_options().batch_size, - const_cast(&_state->timezone_obj()), _io_ctx.get(), _state, + const_cast(&_state->timezone_obj()), _io_ctx, _state, file_meta_cache_ptr, _state->query_options().enable_parquet_lazy_mat); if (_row_id_column_iterator_pair.second != -1) { @@ -1174,9 +1199,18 @@ Status FileScanner::_get_next_reader() { auto file_meta_cache_ptr = _should_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache() : nullptr; + if (is_position_deletes_sys_table) { + auto reader = IcebergPositionDeleteSysTableReader::create_unique( + _file_slot_descs, _state, _profile, range, _params, _io_ctx, + file_meta_cache_ptr); + init_status = reader->init_reader(); + _cur_reader = std::move(reader); + need_to_get_parsed_schema = false; + break; + } std::unique_ptr orc_reader = OrcReader::create_unique( _profile, _state, *_params, range, _state->query_options().batch_size, - _state->timezone(), _io_ctx.get(), file_meta_cache_ptr, + _state->timezone(), _io_ctx, file_meta_cache_ptr, _state->query_options().enable_orc_lazy_mat); if (_row_id_column_iterator_pair.second != -1) { RETURN_IF_ERROR(_create_row_id_column_iterator()); @@ -1201,17 +1235,17 @@ Status FileScanner::_get_next_reader() { case TFileFormatType::FORMAT_CSV_DEFLATE: case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK: case TFileFormatType::FORMAT_PROTO: { - auto reader = - CsvReader::create_unique(_state, _profile, &_counter, *_params, range, - _file_slot_descs, _state->batch_size(), _io_ctx.get()); + auto reader = CsvReader::create_unique(_state, _profile, &_counter, *_params, range, + _file_slot_descs, _state->batch_size(), nullptr, + _io_ctx); init_status = reader->init_reader(_is_load); _cur_reader = std::move(reader); break; } case TFileFormatType::FORMAT_TEXT: { auto reader = TextReader::create_unique(_state, _profile, &_counter, *_params, range, - _file_slot_descs, _state->batch_size(), - _io_ctx.get()); + _file_slot_descs, _state->batch_size(), nullptr, + _io_ctx); init_status = reader->init_reader(_is_load); _cur_reader = std::move(reader); break; @@ -1219,7 +1253,7 @@ Status FileScanner::_get_next_reader() { case TFileFormatType::FORMAT_JSON: { _cur_reader = NewJsonReader::create_unique(_state, _profile, &_counter, *_params, range, _file_slot_descs, &_scanner_eof, - _state->batch_size(), _io_ctx.get()); + _state->batch_size(), nullptr, _io_ctx); init_status = ((NewJsonReader*)(_cur_reader.get())) ->init_reader(_col_default_value_ctx, _is_load); break; @@ -1241,8 +1275,7 @@ Status FileScanner::_get_next_reader() { break; } case TFileFormatType::FORMAT_NATIVE: { - auto reader = - NativeReader::create_unique(_profile, *_params, range, _io_ctx.get(), _state); + auto reader = NativeReader::create_unique(_profile, *_params, range, _io_ctx, _state); init_status = reader->init_reader(); _cur_reader = std::move(reader); need_to_get_parsed_schema = false; @@ -1681,8 +1714,8 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, case TFileFormatType::FORMAT_PARQUET: { std::unique_ptr parquet_reader = ParquetReader::create_unique( _profile, *_params, range, 1, - const_cast(&_state->timezone_obj()), _io_ctx.get(), - _state, file_meta_cache_ptr, false); + const_cast(&_state->timezone_obj()), _io_ctx, _state, + file_meta_cache_ptr, false); RETURN_IF_ERROR(parquet_reader->read_by_rows(row_ids)); RETURN_IF_ERROR( @@ -1691,7 +1724,7 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, } case TFileFormatType::FORMAT_ORC: { std::unique_ptr orc_reader = OrcReader::create_unique( - _profile, _state, *_params, range, 1, _state->timezone(), _io_ctx.get(), + _profile, _state, *_params, range, 1, _state->timezone(), _io_ctx, file_meta_cache_ptr, false); RETURN_IF_ERROR(orc_reader->read_by_rows(row_ids)); diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 88bf97df142403..d8fc1643d3756f 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -77,6 +78,10 @@ class FileScanner : public Scanner { const VExprContextSPtrs& TEST_runtime_filter_partition_prune_ctxs() const { return _runtime_filter_partition_prune_ctxs; } + static TPushAggOp::type TEST_effective_push_down_agg_type( + TPushAggOp::type agg_type, const std::optional>& count_slot_ids) { + return _effective_push_down_agg_type(agg_type, count_slot_ids); + } #endif FileScanner(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -200,7 +205,8 @@ class FileScanner : public Scanner { std::unique_ptr _file_cache_statistics; std::unique_ptr _file_reader_stats; - std::unique_ptr _io_ctx; + // Reader stacks retain this context so delegate readers never outlive their scan state. + std::shared_ptr _io_ctx; // Whether to fill partition columns from path, default is true. bool _fill_partition_from_path = true; @@ -294,7 +300,7 @@ class FileScanner : public Scanner { }; Status _init_io_ctx() { - _io_ctx.reset(new io::IOContext()); + _io_ctx = std::make_shared(); _io_ctx->query_id = &_state->query_id(); return Status::OK(); }; @@ -304,11 +310,6 @@ class FileScanner : public Scanner { _counter.num_rows_filtered = 0; } - TPushAggOp::type _get_push_down_agg_type() const { - return _local_state == nullptr ? TPushAggOp::type::NONE - : _local_state->get_push_down_agg_type(); - } - void _reset_adaptive_batch_size_state(); void _init_adaptive_batch_size_state(TFileFormatType::type format_type); bool _should_enable_adaptive_batch_size(TFileFormatType::type format_type) const; @@ -317,6 +318,30 @@ class FileScanner : public Scanner { void _update_adaptive_batch_size_before_truncate(const Block& block); void _update_adaptive_batch_size_after_truncate(const Block& block); + static TPushAggOp::type _effective_push_down_agg_type( + TPushAggOp::type agg_type, const std::optional>& count_slot_ids) { + if (agg_type != TPushAggOp::type::COUNT) { + return agg_type; + } + // V1's CountReader receives only the file's total row count and emits that many synthetic + // rows. This is exact for COUNT(*)/COUNT(1), but it has no column metadata for NULL or CAST + // semantics. For example, a 10,000-row file with 9,015 non-null values must return 10,000 + // for COUNT(*) and 9,015 for COUNT(nullable_col); CountReader can produce only the former. + // Therefore a non-empty argument list must use the normal reader. nullopt is an old FE plan + // that predates the argument field; treating it as empty would silently reinterpret unknown + // semantics as COUNT(*). + return count_slot_ids.has_value() && count_slot_ids->empty() ? TPushAggOp::type::COUNT + : TPushAggOp::type::NONE; + } + + TPushAggOp::type _get_push_down_agg_type() const { + if (_local_state == nullptr) { + return TPushAggOp::type::NONE; + } + return _effective_push_down_agg_type(_local_state->get_push_down_agg_type(), + _local_state->get_push_down_count_slot_ids()); + } + // enable the file meta cache only when // 1. max_external_file_meta_cache_num is > 0 // 2. the file number is less than 1/3 of cache's capacibility diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 91320e080439b9..a058aac64cef8f 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,7 @@ #include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/format_common.h" +#include "format/table/iceberg_scan_semantics.h" #include "format_v2/column_mapper.h" #include "format_v2/jni/iceberg_sys_table_reader.h" #include "format_v2/jni/jdbc_reader.h" @@ -54,6 +56,7 @@ #include "format_v2/jni/trino_connector_jni_reader.h" #include "format_v2/table/hive_reader.h" #include "format_v2/table/hudi_reader.h" +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "format_v2/table/iceberg_reader.h" #include "format_v2/table/paimon_reader.h" #include "format_v2/table/remote_doris_reader.h" @@ -63,6 +66,7 @@ #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "service/backend_options.h" #include "storage/id_manager.h" @@ -70,6 +74,9 @@ namespace doris { namespace { +constexpr int kIcebergPositionDeleteContent = 1; +constexpr int kIcebergDeletionVectorContent = 3; + std::string table_format_name(const TFileRangeDesc& range) { return range.__isset.table_format_params ? range.table_format_params.table_format_type : "NotSet"; @@ -110,6 +117,15 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { table_format == "max_compute" || table_format == "trino_connector"; } +bool is_iceberg_position_deletes_sys_table(const TFileRangeDesc& range) { + return range.__isset.table_format_params && + range.table_format_params.table_format_type == "iceberg" && + range.table_format_params.__isset.iceberg_params && + range.table_format_params.iceberg_params.__isset.content && + (range.table_format_params.iceberg_params.content == kIcebergPositionDeleteContent || + range.table_format_params.iceberg_params.content == kIcebergDeletionVectorContent); +} + bool is_csv_format(TFileFormatType::type format_type) { switch (format_type) { case TFileFormatType::FORMAT_CSV_PLAIN: @@ -200,12 +216,26 @@ Status rewrite_slot_refs_to_global_index( return Status::OK(); } +Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { + DORIS_CHECK(expr != nullptr); + if (*expr == nullptr) { + return Status::OK(); + } + if (auto* branch_wrapper = dynamic_cast(expr->get()); + branch_wrapper != nullptr) { + *expr = branch_wrapper->to_runtime_filter_expr(); + } + return Status::OK(); +} + } // namespace #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) - : Scanner(state, profile), _table_reader(std::move(table_reader)) {} + : Scanner(state, profile), + _table_reader(std::move(table_reader)), + _scanner_profile(profile) {} Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -233,6 +263,10 @@ Status FileScannerV2::TEST_rewrite_slot_refs_to_global_index( return rewrite_slot_refs_to_global_index(expr, slot_id_to_global_index); } +Status FileScannerV2::TEST_adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { + return adapt_runtime_filter_for_table_reader(expr); +} + FileScannerV2::RealtimeCounterDeltas FileScannerV2::TEST_collect_realtime_counter_deltas( const io::FileReaderStats& file_reader_stats, const io::FileCacheStatistics& file_cache_statistics, @@ -253,6 +287,10 @@ void FileScannerV2::TEST_report_file_cache_profile( bool FileScannerV2::TEST_should_skip_not_found(const Status& status, bool ignore_not_found) { return _should_skip_not_found(status, ignore_not_found); } + +bool FileScannerV2::TEST_should_skip_empty(const Status& status, bool stopped) { + return _should_skip_empty(status, stopped); +} #endif bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -301,24 +339,49 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); - _get_block_timer = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); - _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "NotFoundFileNum", TUnit::UNIT, 1); - _file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1); - _file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadBytes", TUnit::BYTES, 1); - _file_read_calls_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadCalls", TUnit::UNIT, 1); + _initialize_scanner_residual_conjuncts(); + auto* profile = _local_state->scanner_profile(); + _scanner_profile = profile; + const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); + _scanner_total_timer = hierarchy.scanner; + _io_timer = hierarchy.io; + _init_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2InitTime", + file_scan_profile::SCANNER, 1); + _open_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2OpenTime", + file_scan_profile::SCANNER, 1); + _get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetBlockTime", + file_scan_profile::SCANNER, 1); + _prepare_split_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2PrepareSplitTime", + file_scan_profile::SCANNER, 1); + _get_next_range_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetNextRangeTime", + file_scan_profile::SCANNER, 1); + _close_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2CloseTime", + file_scan_profile::SCANNER, 1); + _empty_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptyFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _not_found_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NotFoundFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNumber", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_read_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadBytes", TUnit::BYTES, + file_scan_profile::IO, 1); + _file_read_calls_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadCalls", TUnit::UNIT, + file_scan_profile::IO, 1); _file_read_time_counter = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileReadTime", 1); - _adaptive_batch_predicted_rows_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchPredictedRows", TUnit::UNIT, 1); - _adaptive_batch_actual_bytes_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchActualBytes", TUnit::BYTES, 1); - _adaptive_batch_probe_count_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchProbeCount", TUnit::UNIT, 1); + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReadTime", file_scan_profile::IO, 1); + _adaptive_batch_predicted_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchPredictedRows", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _adaptive_batch_actual_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); + _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "ScannerResidualFilterTime", file_scan_profile::SCANNER, 1); + _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ScannerResidualRowsFiltered", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _refresh_scanner_residual_profile(); + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); _file_reader_stats = std::make_unique(); RETURN_IF_ERROR(_init_io_ctx()); @@ -329,6 +392,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc } Status FileScannerV2::_open_impl(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_open_timer); RETURN_IF_CANCELLED(state); RETURN_IF_ERROR(Scanner::_open_impl(state)); RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range)); @@ -342,6 +407,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { } Status FileScannerV2::_get_next_scan_range(bool* has_next) { + SCOPED_TIMER(_get_next_range_timer); DORIS_CHECK(has_next != nullptr); RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); if (*has_next) { @@ -351,8 +417,11 @@ Status FileScannerV2::_get_next_scan_range(bool* has_next) { } Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* eof) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); + RETURN_IF_ERROR(_sync_table_reader_conjuncts()); if (!_has_prepared_split) { RETURN_IF_ERROR(_prepare_next_split(eof)); if (*eof) { @@ -361,7 +430,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { - SCOPED_TIMER(_get_block_timer); if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -375,6 +443,20 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e *eof = false; continue; } + if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { + // END_OF_FILE here means the reader discovered a valid split with no data while + // opening or probing it, not that the Scanner has exhausted all splits. Examples + // are a zero-byte CSV with an explicit schema and a Doris Native file containing + // only its 12-byte header. Treat it like V1's empty-file path: finish this range, + // discard partial reader state, and let the loop fetch the next split. + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + _state->update_num_finished_scan_range(1); + _has_prepared_split = false; + block->clear_column_data(cast_set(_projected_columns.size())); + *eof = false; + continue; + } RETURN_IF_ERROR(status); } if (*eof) { @@ -388,7 +470,38 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } } +Status FileScannerV2::_filter_output_block(Block* block) { + if (_scanner_residual_conjuncts.empty() || block->rows() == 0) { + return Status::OK(); + } + SCOPED_TIMER(_scanner_residual_filter_timer); + const size_t rows_before_filter = block->rows(); + auto status = VExprContext::filter_block(_scanner_residual_conjuncts, block, block->columns()); + if (!status.ok() && _params != nullptr && + _get_current_format_type() == TFileFormatType::FORMAT_ORC) { + status.prepend("Orc row reader nextBatch failed. reason = "); + } + RETURN_IF_ERROR(status); + const int64_t filtered_rows = cast_set(rows_before_filter - block->rows()); + _counter.num_rows_unselected += filtered_rows; + if (_scanner_residual_rows_filtered_counter != nullptr) { + COUNTER_UPDATE(_scanner_residual_rows_filtered_counter, filtered_rows); + } + return Status::OK(); +} + +size_t FileScannerV2::_last_block_rows_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.rows : block.rows(); +} + +size_t FileScannerV2::_last_block_bytes_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes(); +} + Status FileScannerV2::_prepare_next_split(bool* eos) { + SCOPED_TIMER(_prepare_split_timer); while (true) { bool has_next = _first_scan_range; if (!_first_scan_range) { @@ -404,9 +517,12 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { const auto format_type = get_range_format_type(*_params, _current_range); _init_adaptive_batch_size_state(format_type); - if (_should_run_adaptive_batch_size()) { - // JNI readers open eagerly in prepare_split(). Seed the probe size first so readers - // such as Paimon also use it for their first physical read batch. + if (_block_size_predictor != nullptr) { + // JNI readers open eagerly in prepare_split(). Always seed the probe before preparing + // the next split: its metadata-COUNT decision is not available yet, and the state + // exposed by TableReader can still describe the preceding split. Metadata shortcuts + // ignore this batch size, while row-scan fallbacks need it for their first physical + // read batch. _table_reader->set_batch_size(_predict_reader_batch_rows()); } std::map partition_values; @@ -419,6 +535,16 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { _state->update_num_finished_scan_range(1); continue; } + if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { + // Schema discovery can reach EOF before a split becomes prepared. A header-only Native + // file follows this path, while a reader that discovers emptiness on its first + // get_block() follows the symmetric branch in _get_block_impl(). Both paths must + // advance exactly one scan range and preserve later files in the same scan. + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + _state->update_num_finished_scan_range(1); + continue; + } RETURN_IF_ERROR(status); if (_table_reader->current_split_pruned()) { _state->update_num_finished_scan_range(1); @@ -439,9 +565,25 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + std::optional> push_down_count_columns; + const auto& push_down_count_slot_ids = _local_state->get_push_down_count_slot_ids(); + if (push_down_count_slot_ids.has_value()) { + push_down_count_columns.emplace(); + push_down_count_columns->reserve(push_down_count_slot_ids->size()); + for (const auto slot_id : *push_down_count_slot_ids) { + const auto global_index_it = _slot_id_to_global_index.find(slot_id); + if (global_index_it == _slot_id_to_global_index.end()) { + return Status::InternalError( + "Pushed-down COUNT argument is not a projected file scan slot, slot_id={}", + slot_id); + } + push_down_count_columns->push_back(global_index_it->second); + } + } RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = const_cast(_params), .io_ctx = _io_ctx, @@ -449,8 +591,12 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .scanner_profile = _local_state->scanner_profile(), .file_slot_descs = &_file_slot_descs, .push_down_agg_type = _local_state->get_push_down_agg_type(), + .push_down_count_columns = std::move(push_down_count_columns), .condition_cache_digest = _local_state->get_condition_cache_digest(), })); + _table_reader_applied_rf_num = _applied_rf_num; + // RFs collected before TableReader initialization are already present in the full snapshot. + _late_arrival_rf_conjuncts.clear(); return Status::OK(); } @@ -463,7 +609,9 @@ Status FileScannerV2::_create_table_reader_for_format( } else if (table_format == "hive") { *reader = format::hive::HiveReader::create_unique(); } else if (table_format == "iceberg") { - if (get_range_format_type(*_params, range) == TFileFormatType::FORMAT_JNI) { + if (is_iceberg_position_deletes_sys_table(range)) { + *reader = std::make_unique(); + } else if (get_range_format_type(*_params, range) == TFileFormatType::FORMAT_JNI) { *reader = std::make_unique(); } else { *reader = std::make_unique(); @@ -493,15 +641,12 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values) { format::FileFormat current_split_format; RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), ¤t_split_format)); - VExprContextSPtrs conjuncts; - RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts)); VExprContextSPtrs partition_prune_conjuncts; if (_state->query_options().enable_runtime_filter_partition_prune) { RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts)); } RETURN_IF_ERROR(_table_reader->prepare_split({ .partition_values = std::move(partition_values), - .conjuncts = std::move(conjuncts), .partition_prune_conjuncts = std::move(partition_prune_conjuncts), // A metadata COUNT split may span scheduler turns. Do not enter that irreversible // synthetic-row path while a runtime filter can still arrive between batches. @@ -519,6 +664,14 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not return ignore_not_found && status.is(); } +bool FileScannerV2::_should_skip_empty(const Status& status, bool stopped) { + // Several readers use END_OF_FILE both for a valid zero-row split and for an interrupted IO. + // For example, DeletionVectorReader returns END_OF_FILE("stop read.") after try_stop() marks + // the shared IOContext. That status must unwind the stopped scanner; counting it as an empty + // file would incorrectly finish the scan range and increment EmptyFileNum. + return !stopped && status.is(); +} + bool FileScannerV2::_should_enable_file_meta_cache() const { return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; @@ -610,6 +763,9 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ .range = &_current_range, .runtime_state = _state, }; + // Field 34 is the rollout boundary for root and nested exact-name precedence. + const bool prefer_exact_name_match = + !_params->__isset.history_schema_info || supports_iceberg_scan_semantics_v1(_params); for (size_t slot_idx = 0; slot_idx < _params->required_slots.size(); ++slot_idx) { const auto& slot_info = _params->required_slots[slot_idx]; @@ -630,7 +786,8 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ // column's nested children. RETURN_IF_ERROR(AccessPathParser::build_nested_children( &column, it->second, - build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr)); + build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr, + prefer_exact_name_match)); if (is_partition_slot(slot_info, column.name)) { column.is_partition_key = true; _partition_slot_descs.emplace( @@ -675,18 +832,86 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor } Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const { + return _build_table_conjuncts(_conjuncts, conjuncts); +} + +Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const { DORIS_CHECK(conjuncts != nullptr); conjuncts->clear(); - conjuncts->reserve(_conjuncts.size()); - for (const auto& conjunct : _conjuncts) { + conjuncts->reserve(source.size()); + for (const auto& conjunct : source) { VExprSPtr root; RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index)); + RETURN_IF_ERROR(adapt_runtime_filter_for_table_reader(&root)); conjuncts->push_back(VExprContext::create_shared(std::move(root))); } return Status::OK(); } +size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + if (!format::TableReader::is_safe_to_pre_execute(conjuncts[conjunct_index])) { + return conjunct_index; + } + } + return conjuncts.size(); +} + +void FileScannerV2::_initialize_scanner_residual_conjuncts() { + _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts); + // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe + // predicate could run below Scanner before a stateful/error-preserving ordering barrier. + _scanner_residual_conjuncts.assign( + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count), + _conjuncts.end()); + _refresh_scanner_residual_profile(); +} + +void FileScannerV2::_refresh_scanner_residual_profile() { + if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty()) { + return; + } + std::ostringstream predicates; + predicates << "["; + for (size_t conjunct_index = 0; conjunct_index < _scanner_residual_conjuncts.size(); + ++conjunct_index) { + if (conjunct_index > 0) { + predicates << ", "; + } + predicates << _scanner_residual_conjuncts[conjunct_index]->root()->debug_string(); + } + predicates << "]"; + _scanner_profile->add_info_string("ScannerResidualPredicates", predicates.str()); +} + +Status FileScannerV2::_sync_table_reader_conjuncts() { + if (_table_reader == nullptr) { + return Status::OK(); + } + if (_table_reader_applied_rf_num == _applied_rf_num) { + return Status::OK(); + } + VExprContextSPtrs appended; + RETURN_IF_ERROR(_build_table_conjuncts(_late_arrival_rf_conjuncts, &appended)); + const size_t owned_count = _scanner_residual_conjuncts.empty() + ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) + : 0; + // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting + // may move a late RF ahead of an old stateful predicate in the full scanner snapshot. + RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count)); + _table_reader_owned_conjunct_count += owned_count; + _scanner_residual_conjuncts.insert( + _scanner_residual_conjuncts.end(), + _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), + _late_arrival_rf_conjuncts.end()); + _refresh_scanner_residual_profile(); + _late_arrival_rf_conjuncts.clear(); + _table_reader_applied_rf_num = _applied_rf_num; + return Status::OK(); +} + TFileFormatType::type FileScannerV2::_get_current_format_type() const { return get_range_format_type(*_params, _current_range); } @@ -785,10 +1010,17 @@ bool FileScannerV2::_should_enable_adaptive_batch_size(TFileFormatType::type for } bool FileScannerV2::_should_run_adaptive_batch_size() const { - // COUNT pushdown emits synthetic rows from file metadata and does not materialize file columns, - // so there is no useful row-width sample to learn from. - return _block_size_predictor != nullptr && - _local_state->get_push_down_agg_type() != TPushAggOp::type::COUNT; + DORIS_CHECK(_table_reader != nullptr); + return _should_run_adaptive_batch_size(_block_size_predictor != nullptr, + _table_reader->current_split_uses_metadata_count()); +} + +bool FileScannerV2::_should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count) { + // Metadata COUNT emits synthetic rows and has no physical row width to learn from. A raw COUNT + // opcode is not sufficient here: unsupported argument counts, mappings, filters, or deletes + // make TableReader fall back to materializing normal rows, which still need adaptive batching. + return predictor_initialized && !current_split_uses_metadata_count; } size_t FileScannerV2::_predict_reader_batch_rows() { @@ -804,20 +1036,24 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { if (!_should_run_adaptive_batch_size()) { return; } - COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(block.bytes())); - if (block.rows() == 0) { + const auto& stats = _table_reader->last_materialized_block_stats(); + const size_t rows = stats.has_materialized_input ? stats.rows : block.rows(); + const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes(); + COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(bytes)); + if (rows == 0) { return; } - // The sample is taken after TableReader has finalized file-local columns to table columns. - // This matches the memory shape seen by upstream operators and catches very wide nested - // columns, such as map/string payloads, after the first probe batch. + // Residual predicates run after wide table columns are materialized. Learn from that pre-filter + // shape so selective predicates cannot make the next reader batch dangerously large. if (!_block_size_predictor->has_history()) { COUNTER_UPDATE(_adaptive_batch_probe_count_counter, 1); } - _block_size_predictor->update(block); + _block_size_predictor->update(rows, bytes); } Status FileScannerV2::close(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_close_timer); if (_is_closed) { return Status::OK(); } @@ -954,14 +1190,25 @@ void FileScannerV2::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); if (config::enable_file_cache && _state->query_options().enable_file_cache && _profile != nullptr) { - _report_file_cache_profile(_profile, *_file_cache_statistics); + auto file_cache_delta = io::diff_file_cache_statistics(*_file_cache_statistics, + _reported_file_cache_statistics); + // Profile collection can run more than once, so publish only the new additive delta. + _report_file_cache_profile(_profile, file_cache_delta); _state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache( - _file_cache_statistics->bytes_write_into_cache); + file_cache_delta.bytes_write_into_cache); + _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + const auto read_time = cast_set(_file_reader_stats->read_time_ns); + DORIS_CHECK(read_time >= _reported_io_read_time); + // Some transports (for example Arrow Flight) record directly into IO, while filesystem + // reads arrive through FileReaderStats. Add only the new traced delta so both paths remain + // visible without double counting repeated profile publication. + COUNTER_UPDATE(_io_timer, read_time - _reported_io_read_time); + _reported_io_read_time = read_time; } // Query profiles can be collected before Scanner::close() runs. Publish condition-cache // counters here as well, using deltas so this method and close() cannot double count. @@ -970,7 +1217,8 @@ void FileScannerV2::_collect_profile_before_close() { void FileScannerV2::_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { - io::FileCacheProfileReporter cache_profile(profile); + file_scan_profile::ensure_hierarchy(profile); + io::FileCacheProfileReporter cache_profile(profile, file_scan_profile::IO); cache_profile.update(&file_cache_statistics); } @@ -992,9 +1240,8 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() { const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0; const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows; if (filtered_delta > 0) { - // File readers can evaluate localized conjuncts before a block reaches Scanner. Count - // those rows as scanner-level unselected rows so load statistics stay identical no matter - // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block(). + // FileReader and TableReader both report their owned predicate rows through the shared IO + // context. Preserve scanner-level load statistics without re-evaluating either predicate. _counter.num_rows_unselected += filtered_delta; _reported_predicate_filtered_rows = filtered_rows; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 2bddc5d5e69e6c..cabbc96baa8400 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -78,6 +78,7 @@ class FileScannerV2 final : public Scanner { static Status TEST_rewrite_slot_refs_to_global_index( VExprSPtr* expr, const std::unordered_map& slot_id_to_global_index); + static Status TEST_adapt_runtime_filter_for_table_reader(VExprSPtr* expr); static RealtimeCounterDeltas TEST_collect_realtime_counter_deltas( const io::FileReaderStats& file_reader_stats, const io::FileCacheStatistics& file_cache_statistics, @@ -87,6 +88,23 @@ class FileScannerV2 final : public Scanner { static void TEST_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); + static bool TEST_should_skip_empty(const Status& status, bool stopped); + static bool TEST_should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count) { + return _should_run_adaptive_batch_size(predictor_initialized, + current_split_uses_metadata_count); + } + void TEST_set_scanner_conjuncts(VExprContextSPtrs conjuncts) { + _conjuncts = std::move(conjuncts); + _initialize_scanner_residual_conjuncts(); + } + Status TEST_filter_output_block(Block* block) { return _filter_output_block(block); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } + size_t TEST_scanner_residual_conjunct_count() const { + return _scanner_residual_conjuncts.size(); + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -104,6 +122,9 @@ class FileScannerV2 final : public Scanner { protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; + Status _filter_output_block(Block* block) override; + size_t _last_block_rows_read(const Block& block) const override; + size_t _last_block_bytes_read(const Block& block) const override; void _collect_profile_before_close() override; bool _should_update_load_counters() const override; @@ -121,6 +142,7 @@ class FileScannerV2 final : public Scanner { Status _prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); + static bool _should_skip_empty(const Status& status, bool stopped); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -132,12 +154,20 @@ class FileScannerV2 final : public Scanner { Status _build_default_expr(const TFileScanSlotInfo& slot_info, VExprContextSPtr* ctx) const; static format::ColumnDefinition _build_table_column(const SlotDescriptor* slot_desc); Status _build_table_conjuncts(VExprContextSPtrs* conjuncts) const; + Status _build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const; + Status _sync_table_reader_conjuncts(); + static size_t _safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts); + void _initialize_scanner_residual_conjuncts(); + void _refresh_scanner_residual_profile(); static Status _to_file_format(TFileFormatType::type format_type, format::FileFormat* file_format); void _reset_adaptive_batch_size_state(); void _init_adaptive_batch_size_state(TFileFormatType::type format_type); bool _should_enable_adaptive_batch_size(TFileFormatType::type format_type) const; bool _should_run_adaptive_batch_size() const; + static bool _should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count); size_t _predict_reader_batch_rows(); void _update_adaptive_batch_size(const Block& block); static RealtimeCounterDeltas _collect_realtime_counter_deltas( @@ -165,6 +195,10 @@ class FileScannerV2 final : public Scanner { std::string _current_range_path; std::unique_ptr _table_reader; + size_t _table_reader_owned_conjunct_count = 0; + // Scanner owns one persistent context vector for the first unsafe conjunct and every later + // conjunct. Hybrid child readers may be recreated or switched, but this state must not be. + VExprContextSPtrs _scanner_residual_conjuncts; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to @@ -176,11 +210,20 @@ class FileScannerV2 final : public Scanner { std::unordered_map _partition_slot_descs; std::unique_ptr _file_cache_statistics; + io::FileCacheStatistics _reported_file_cache_statistics; std::unique_ptr _file_reader_stats; std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; + RuntimeProfile::Counter* _scanner_total_timer = nullptr; + RuntimeProfile::Counter* _init_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; RuntimeProfile::Counter* _get_block_timer = nullptr; + RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _prepare_split_timer = nullptr; + RuntimeProfile::Counter* _get_next_range_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _io_timer = nullptr; RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; @@ -189,6 +232,9 @@ class FileScannerV2 final : public Scanner { RuntimeProfile::Counter* _adaptive_batch_predicted_rows_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_actual_bytes_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr; + RuntimeProfile::Counter* _scanner_residual_filter_timer = nullptr; + RuntimeProfile::Counter* _scanner_residual_rows_filtered_counter = nullptr; + RuntimeProfile* _scanner_profile = nullptr; std::unique_ptr _block_size_predictor; int64_t _reported_predicate_filtered_rows = 0; int64_t _reported_condition_cache_hit_count = 0; @@ -197,6 +243,8 @@ class FileScannerV2 final : public Scanner { int64_t _last_read_rows = 0; int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; + int64_t _reported_io_read_time = 0; + int _table_reader_applied_rf_num = 0; }; } // namespace doris diff --git a/be/src/exec/scan/scanner.cpp b/be/src/exec/scan/scanner.cpp index 3069438c764b90..c5a74b358e72da 100644 --- a/be/src/exec/scan/scanner.cpp +++ b/be/src/exec/scan/scanner.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/config.h" #include "common/status.h" #include "core/block/column_with_type_and_name.h" @@ -145,8 +147,11 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) { DCHECK(block->rows() == 0); break; } - _num_rows_read += block->rows(); - _num_byte_read += block->allocated_bytes(); + // Some scanners apply owned predicates before returning the block. Account the + // materialized input, not only survivors, so the per-turn progress bound remains + // effective for highly selective predicates. + _num_rows_read += _last_block_rows_read(*block); + _num_byte_read += _last_block_bytes_read(*block); } // 2. Filter the output block finally. @@ -228,7 +233,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { } DCHECK(_applied_rf_num < _total_rf_num); int arrived_rf_num = 0; - RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter(_state, arrived_rf_num)); + VExprContextSPtrs arrived_conjuncts; + RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter( + _state, _applied_rf_num, arrived_rf_num, arrived_conjuncts)); if (arrived_rf_num == _applied_rf_num) { // No newly arrived runtime filters, just return; @@ -238,6 +245,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { // avoid conjunct destroy in used by storage layer _conjuncts.clear(); RETURN_IF_ERROR(_local_state->clone_conjunct_ctxs(_conjuncts)); + _late_arrival_rf_conjuncts.insert(_late_arrival_rf_conjuncts.end(), + std::make_move_iterator(arrived_conjuncts.begin()), + std::make_move_iterator(arrived_conjuncts.end())); _applied_rf_num = arrived_rf_num; return Status::OK(); } diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index 92119f7d3fff3a..f12b6b2849f3ae 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -137,7 +137,7 @@ class Scanner { virtual bool _should_update_load_counters() const { return _is_load; } // Filter the output block finally. - Status _filter_output_block(Block* block); + virtual Status _filter_output_block(Block* block); Status _do_projections(Block* origin_block, Block* output_block); @@ -204,6 +204,11 @@ class Scanner { void update_block_avg_bytes(size_t block_avg_bytes) { _block_avg_bytes = block_avg_bytes; } protected: + virtual size_t _last_block_rows_read(const Block& block) const { return block.rows(); } + virtual size_t _last_block_bytes_read(const Block& block) const { + return block.allocated_bytes(); + } + RuntimeState* _state = nullptr; ScanLocalStateBase* _local_state = nullptr; @@ -231,6 +236,9 @@ class Scanner { // Cloned from _conjuncts of scan node. // It includes predicate in SQL and runtime filters. VExprContextSPtrs _conjuncts; + // Exact append-only RF delta for readers that preserve state across multiple splits. It must + // not be reconstructed by position from the cost-sorted full conjunct snapshot. + VExprContextSPtrs _late_arrival_rf_conjuncts; VExprContextSPtrs _projections; // Used in common subexpression elimination to compute intermediate results. std::vector _intermediate_projections; diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index e2b142db8d9f86..ef167ecdbcaea3 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -35,6 +35,7 @@ #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" #include "exprs/vexpr.h" +#include "format/table/deletion_vector.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/transformer/vfile_format_transformer.h" #include "io/file_factory.h" @@ -106,6 +107,20 @@ Status load_rewritable_delete_rows(RuntimeState* state, RuntimeProfile* profile, } // namespace +Status calculate_iceberg_deletion_vector_content_size(size_t bitmap_size, int64_t* content_size) { + DORIS_CHECK(content_size != nullptr); + constexpr size_t max_bitmap_size = static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES) - + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES; + if (bitmap_size > max_bitmap_size) { + return Status::NotSupported( + "Iceberg deletion vector bitmap size exceeds Doris supported limit: {}, " + "maximum bitmap size: {}, content size limit: {}", + bitmap_size, max_bitmap_size, MAX_ICEBERG_DELETION_VECTOR_BYTES); + } + *content_size = static_cast(bitmap_size + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES); + return Status::OK(); +} + VIcebergDeleteSink::VIcebergDeleteSink(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs, std::shared_ptr dep, @@ -592,7 +607,8 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( blob.partition_spec_id = deletion.partition_spec_id; blob.partition_data_json = deletion.partition_data_json; blob.merged_count = static_cast(merged_rows.cardinality()); - blob.content_size_in_bytes = static_cast(4 + 4 + bitmap_size + 4); + RETURN_IF_ERROR(calculate_iceberg_deletion_vector_content_size( + bitmap_size, &blob.content_size_in_bytes)); blob.blob_data.resize(static_cast(blob.content_size_in_bytes)); merged_rows.write(blob.blob_data.data() + 8); @@ -604,7 +620,7 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( uint32_t crc = static_cast( ::crc32(0, reinterpret_cast(blob.blob_data.data() + 4), - 4 + (uInt)bitmap_size)); + static_cast(4 + bitmap_size))); BigEndian::Store32(blob.blob_data.data() + 8 + bitmap_size, crc); blobs.emplace_back(std::move(blob)); } diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 22ae98cc288100..647ee92f525733 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include #include @@ -41,6 +43,8 @@ class Dependency; class VIcebergDeleteFileWriter; +Status calculate_iceberg_deletion_vector_content_size(size_t bitmap_size, int64_t* content_size); + struct IcebergFileDeletion { IcebergFileDeletion() = default; IcebergFileDeletion(int32_t spec_id, std::string data_json) diff --git a/be/src/exec/sink/viceberg_merge_sink.cpp b/be/src/exec/sink/viceberg_merge_sink.cpp index 7d6c84cf068dbc..e9f4d6bf5c655a 100644 --- a/be/src/exec/sink/viceberg_merge_sink.cpp +++ b/be/src/exec/sink/viceberg_merge_sink.cpp @@ -254,6 +254,9 @@ Status VIcebergMergeSink::_build_inner_sinks() { if (merge_sink.__isset.broker_addresses) { table_sink.__set_broker_addresses(merge_sink.broker_addresses); } + if (merge_sink.__isset.collect_column_stats) { + table_sink.__set_collect_column_stats(merge_sink.collect_column_stats); + } _table_sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); _table_sink.__set_iceberg_table_sink(table_sink); diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.cpp b/be/src/exec/sink/writer/iceberg/partition_transformers.cpp index 8b49d68573c1e4..e38e0e4c5eb8a7 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.cpp +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.cpp @@ -18,12 +18,28 @@ #include "exec/sink/writer/iceberg/partition_transformers.h" #include +#include #include "core/types.h" #include "format/table/iceberg/partition_spec.h" namespace doris { +namespace { + +template +std::string floating_point_partition_value_to_string(T value) { + if (std::isnan(value)) { + return "NaN"; + } + if (std::isinf(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + return std::to_string(value); +} + +} // namespace + const std::chrono::sys_days PartitionColumnTransformUtils::EPOCH = std::chrono::sys_days( std::chrono::year {1970} / std::chrono::January / std::chrono::day {1}); @@ -225,10 +241,10 @@ std::string PartitionColumnTransform::get_partition_value(const DataTypePtr type return std::to_string(std::any_cast(value)); } case TYPE_FLOAT: { - return std::to_string(std::any_cast(value)); + return floating_point_partition_value_to_string(std::any_cast(value)); } case TYPE_DOUBLE: { - return std::to_string(std::any_cast(value)); + return floating_point_partition_value_to_string(std::any_cast(value)); } case TYPE_VARCHAR: case TYPE_CHAR: diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index 434488266bb3ac..25da8724b2899b 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -47,7 +47,11 @@ VIcebergPartitionWriter::VIcebergPartitionWriter( _file_name_index(file_name_index), _file_format_type(file_format_type), _compress_type(compress_type), - _hadoop_conf(hadoop_conf) {} + _hadoop_conf(hadoop_conf) { + if (t_sink.iceberg_table_sink.__isset.collect_column_stats) { + _collect_column_stats = t_sink.iceberg_table_sink.collect_column_stats; + } +} Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profile, const RowDescriptor* row_desc) { @@ -155,6 +159,10 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c commit_data->__set_file_size(_file_format_transformer->written_len()); commit_data->__set_file_content(TFileContent::DATA); commit_data->__set_partition_values(_partition_values); + // ORC collection reopens the file, so honor the FE policy before any footer work. + if (!_collect_column_stats) { + return Status::OK(); + } if (_file_format_type == TFileFormatType::FORMAT_PARQUET) { TIcebergColumnStats column_stats; RETURN_IF_ERROR(static_cast(_file_format_transformer.get()) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h index 97b4dd3efdfac0..b0839a82ed3796 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h @@ -67,6 +67,8 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { inline size_t written_len() const override { return _file_format_transformer->written_len(); } private: + friend class VIcebergPartitionWriterTest; + std::string _get_target_file_name(); Status _build_iceberg_commit_data(TIcebergCommitData* commit_data); @@ -91,6 +93,7 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { TFileFormatType::type _file_format_type; TFileCompressType::type _compress_type; const std::map& _hadoop_conf; + bool _collect_column_stats = true; std::shared_ptr _fs = nullptr; diff --git a/be/src/exprs/runtime_filter_expr.cpp b/be/src/exprs/runtime_filter_expr.cpp new file mode 100644 index 00000000000000..a0d4403086af53 --- /dev/null +++ b/be/src/exprs/runtime_filter_expr.cpp @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/runtime_filter_expr.h" + +#include + +#include + +#include "common/logging.h" +#include "core/block/block.h" +#include "core/block/column_numbers.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column.h" +#include "core/column/column_const.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "exec/common/util.hpp" +#include "exprs/vslot_ref.h" +#include "runtime/runtime_profile.h" +#include "storage/index/zone_map/zonemap_eval_context.h" + +namespace doris { + +class RowDescriptor; +class RuntimeState; +class TExprNode; +// branch-4.1 still builds VRuntimeFilterWrapper, which owns these shared threshold helpers. +} // namespace doris + +namespace doris { + +class VExprContext; + +RuntimeFilterExpr::RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, + bool null_aware, int filter_id, int sampling_frequency) + : VExpr(node), + _impl(std::move(impl)), + _ignore_thredhold(ignore_thredhold), + _null_aware(null_aware), + _filter_id(filter_id), + _sampling_frequency(sampling_frequency) { + DORIS_CHECK(_impl != nullptr); +} + +Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr) const { + DORIS_CHECK(cloned_expr != nullptr); + DORIS_CHECK(_impl != nullptr); + VExprSPtr cloned_impl; + RETURN_IF_ERROR(_impl->deep_clone(&cloned_impl)); + auto cloned_runtime_filter = RuntimeFilterExpr::create_shared( + clone_texpr_node(), std::move(cloned_impl), _ignore_thredhold, _null_aware, _filter_id, + _sampling_frequency); + cloned_runtime_filter->attach_profile_counter(_rf_input_rows, _rf_filter_rows, + _always_true_filter_rows); + *cloned_expr = std::move(cloned_runtime_filter); + return Status::OK(); +} + +Status RuntimeFilterExpr::prepare(RuntimeState* state, const RowDescriptor& desc, + VExprContext* context) { + RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); + _expr_name = fmt::format("RuntimeFilterExpr({})", _impl->expr_name()); + _prepare_finished = true; + return Status::OK(); +} + +Status RuntimeFilterExpr::open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) { + DCHECK(_prepare_finished); + RETURN_IF_ERROR(_impl->open(state, context, scope)); + context->get_runtime_filter_selectivity().set_sampling_frequency(_sampling_frequency); + _open_finished = true; + return Status::OK(); +} + +void RuntimeFilterExpr::close(VExprContext* context, FunctionContext::FunctionStateScope scope) { + _impl->close(context, scope); +} + +Status RuntimeFilterExpr::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { + // branch-4.1's VExpr API predates const selectors; RuntimeFilterExpr only forwards the + // selector and does not mutate it. + return _impl->execute_column(context, block, const_cast(selector), count, + result_column); +} + +const std::string& RuntimeFilterExpr::expr_name() const { + return _expr_name; +} + +Status RuntimeFilterExpr::execute_filter(VExprContext* context, const Block* block, + uint8_t* __restrict result_filter_data, size_t rows, + bool accept_null, bool* can_filter_all) const { + DCHECK(_open_finished); + if (accept_null) { + return Status::InternalError( + "Runtime filter does not support accept_null in execute_filter"); + } + + auto& rf_selectivity = context->get_runtime_filter_selectivity(); + Defer auto_update_judge_counter = [&]() { rf_selectivity.update_judge_counter(); }; + + // if always true, skip evaluate runtime filter + if (rf_selectivity.maybe_always_true_can_ignore()) { + COUNTER_UPDATE(_always_true_filter_rows, rows); + return Status::OK(); + } + + ColumnPtr filter_column; + ColumnPtr arg_column = nullptr; + + RETURN_IF_ERROR(_impl->execute_runtime_filter(context, block, result_filter_data, rows, + filter_column, &arg_column)); + + // bloom filter will handle null aware inside itself + if (_null_aware && TExprNodeType::BLOOM_PRED != node_type()) { + DCHECK(arg_column); + change_null_to_true(filter_column->assert_mutable(), arg_column); + } + + if (const auto* const_column = check_and_get_column(*filter_column)) { + // const(nullable) or const(bool) + if (!const_column->get_bool(0)) { + // filter all + COUNTER_UPDATE(_rf_filter_rows, rows); + COUNTER_UPDATE(_rf_input_rows, rows); + rf_selectivity.update_judge_selectivity(_filter_id, rows, rows, _ignore_thredhold); + *can_filter_all = true; + memset(result_filter_data, 0, rows); + return Status::OK(); + } else { + // filter none + COUNTER_UPDATE(_rf_input_rows, rows); + rf_selectivity.update_judge_selectivity(_filter_id, 0, rows, _ignore_thredhold); + return Status::OK(); + } + } else if (const auto* nullable_column = check_and_get_column(*filter_column)) { + // nullable(bool) + const ColumnPtr& nested_column = nullable_column->get_nested_column_ptr(); + const IColumn::Filter& filter = assert_cast(*nested_column).get_data(); + const auto* __restrict filter_data = filter.data(); + const auto* __restrict null_map_data = nullable_column->get_null_map_data().data(); + + const size_t input_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + for (size_t i = 0; i < rows; ++i) { + result_filter_data[i] &= (!null_map_data[i]) & filter_data[i]; + } + + const size_t output_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + COUNTER_UPDATE(_rf_filter_rows, input_rows - output_rows); + COUNTER_UPDATE(_rf_input_rows, input_rows); + rf_selectivity.update_judge_selectivity(_filter_id, input_rows - output_rows, input_rows, + _ignore_thredhold); + + if (output_rows == 0) { + *can_filter_all = true; + return Status::OK(); + } + } else { + // bool + const IColumn::Filter& filter = assert_cast(*filter_column).get_data(); + const auto* __restrict filter_data = filter.data(); + + const size_t input_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + for (size_t i = 0; i < rows; ++i) { + result_filter_data[i] &= filter_data[i]; + } + + const size_t output_rows = rows - simd::count_zero_num((int8_t*)result_filter_data, rows); + + COUNTER_UPDATE(_rf_filter_rows, input_rows - output_rows); + COUNTER_UPDATE(_rf_input_rows, input_rows); + rf_selectivity.update_judge_selectivity(_filter_id, input_rows - output_rows, input_rows, + _ignore_thredhold); + + if (output_rows == 0) { + *can_filter_all = true; + return Status::OK(); + } + } + return Status::OK(); +} + +ZoneMapFilterResult RuntimeFilterExpr::evaluate_zonemap_filter( + const ZoneMapEvalContext& ctx) const { + if (_null_aware) { + const auto child = _impl->get_child(0); + if (auto slot_ref = std::dynamic_pointer_cast(child); slot_ref != nullptr) { + auto zone_map = ctx.zone_map(slot_ref->column_id()); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + + if (zone_map->has_null) { + return ZoneMapFilterResult::kMayMatch; + } + } else { + // RuntimeFilterExpr implementations generated by RuntimeFilterConsumer put the probe + // slot as child 0. If a custom RF expression violates this shape, keep the null-aware + // evaluation conservative because we cannot decide whether the target zone has NULLs. + return unsupported_zonemap_filter(ctx); + } + } + return _impl->evaluate_zonemap_filter(ctx); +} + +bool RuntimeFilterExpr::can_evaluate_zonemap_filter() const { + return _impl->can_evaluate_zonemap_filter(); +} + +void RuntimeFilterExpr::collect_slot_column_ids(std::set& column_ids) const { + _impl->collect_slot_column_ids(column_ids); +} + +} // namespace doris diff --git a/be/src/exprs/runtime_filter_expr.h b/be/src/exprs/runtime_filter_expr.h new file mode 100644 index 00000000000000..7994d2a71ae14f --- /dev/null +++ b/be/src/exprs/runtime_filter_expr.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "exec/runtime_filter/runtime_filter_selectivity.h" +#include "exprs/function_context.h" +#include "exprs/vexpr.h" +#include "runtime/runtime_profile.h" + +namespace doris { +class RowDescriptor; +class RuntimeState; +class TExprNode; + +double get_in_list_ignore_thredhold(size_t list_size); +double get_comparison_ignore_thredhold(); +double get_bloom_filter_ignore_thredhold(); +} // namespace doris + +namespace doris { + +class Block; +class VExprContext; + +class RuntimeFilterExpr final : public VExpr { + ENABLE_FACTORY_CREATOR(RuntimeFilterExpr); + +public: + RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, + bool null_aware, int filter_id, + int sampling_frequency = RuntimeFilterSelectivity::DISABLE_SAMPLING); + ~RuntimeFilterExpr() override = default; + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override; + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override; + std::string debug_string() const override { return _impl->debug_string(); } + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; + const std::string& expr_name() const override; + const VExprSPtrs& children() const override { return _impl->children(); } + TExprNodeType::type node_type() const override { return _impl->node_type(); } + + double execute_cost() const override { return _impl->execute_cost(); } + + Status execute_filter(VExprContext* context, const Block* block, + uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, + bool* can_filter_all) const override; + + uint64_t get_digest(uint64_t seed) const override { + seed = _impl->get_digest(seed); + if (seed) { + return HashUtil::crc_hash64(&_null_aware, sizeof(_null_aware), seed); + } + return seed; + } + + VExprSPtr get_impl() const override { return _impl; } + void set_impl(VExprSPtr impl) { _impl = std::move(impl); } + Status clone_node(VExprSPtr* cloned_expr) const override; + + void attach_profile_counter(std::shared_ptr rf_input_rows, + std::shared_ptr rf_filter_rows, + std::shared_ptr always_true_filter_rows) { + DCHECK(rf_input_rows != nullptr); + DCHECK(rf_filter_rows != nullptr); + DCHECK(always_true_filter_rows != nullptr); + + if (rf_input_rows != nullptr) { + _rf_input_rows = rf_input_rows; + } + if (rf_filter_rows != nullptr) { + _rf_filter_rows = rf_filter_rows; + } + if (always_true_filter_rows != nullptr) { + _always_true_filter_rows = always_true_filter_rows; + } + } + + bool is_rf_wrapper() const override { return true; } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; + bool can_evaluate_zonemap_filter() const override; + void collect_slot_column_ids(std::set& column_ids) const override; + + int filter_id() const { return _filter_id; } + + std::shared_ptr predicate_filtered_rows_counter() const { + return _rf_filter_rows; + } + std::shared_ptr predicate_input_rows_counter() const { + return _rf_input_rows; + } + std::shared_ptr predicate_always_true_rows_counter() const { + return _always_true_filter_rows; + } + bool is_slot_ref() const override { return false; } + bool is_virtual_slot_ref() const override { return false; } + bool is_column_ref() const override { return false; } + +private: + VExprSPtr _impl; + + std::shared_ptr _rf_input_rows = + std::make_shared(TUnit::UNIT, 0); + std::shared_ptr _rf_filter_rows = + std::make_shared(TUnit::UNIT, 0); + std::shared_ptr _always_true_filter_rows = + std::make_shared(TUnit::UNIT, 0); + + std::string _expr_name; + double _ignore_thredhold; + bool _null_aware; + int _filter_id; + int _sampling_frequency; +}; + +using RuntimeFilterExprPtr = std::shared_ptr; + +} // namespace doris diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h index 8bf0839f274d1a..770d7a8d5f59a0 100644 --- a/be/src/exprs/vcompound_pred.h +++ b/be/src/exprs/vcompound_pred.h @@ -128,6 +128,14 @@ class VCompoundPred : public VectorizedFnCall { } } + bool is_safe_to_execute_on_selected_rows() const override { + // Boolean composition introduces no data-dependent failure of its own. Reuse the generic + // child walk so AND/OR remain eligible only when every nested expression is independently + // safe; applying VectorizedFnCall's scalar-function allowlist to this structural node would + // incorrectly disable selected-row execution for otherwise safe predicates. + return VExpr::is_safe_to_execute_on_selected_rows(); + } + ZoneMapFilterResult evaluate_dictionary_filter( const DictionaryEvalContext& ctx) const override { switch (_op) { diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 621d183fc84a05..58e3af4ad4c87b 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -24,9 +24,11 @@ #include #include +#include #include #include +#include "common/compare.h" #include "common/config.h" #include "common/exception.h" #include "common/logging.h" @@ -83,6 +85,82 @@ const static std::set DISTANCE_FUNCS = {L2DistanceApproximate::name const static std::set OPS_FOR_ANN_RANGE_SEARCH = { TExprOpcode::GE, TExprOpcode::LE, TExprOpcode::LE, TExprOpcode::GT, TExprOpcode::LT}; +namespace { + +enum class RawComparisonOp : uint8_t { EQ, NE, LT, LE, GT, GE }; + +std::optional raw_comparison_op(std::string_view function_name, bool reverse) { + RawComparisonOp op; + if (function_name == "eq") { + op = RawComparisonOp::EQ; + } else if (function_name == "ne") { + op = RawComparisonOp::NE; + } else if (function_name == "lt") { + op = RawComparisonOp::LT; + } else if (function_name == "le") { + op = RawComparisonOp::LE; + } else if (function_name == "gt") { + op = RawComparisonOp::GT; + } else if (function_name == "ge") { + op = RawComparisonOp::GE; + } else { + return std::nullopt; + } + if (!reverse || op == RawComparisonOp::EQ || op == RawComparisonOp::NE) { + return op; + } + switch (op) { + case RawComparisonOp::LT: + return RawComparisonOp::GT; + case RawComparisonOp::LE: + return RawComparisonOp::GE; + case RawComparisonOp::GT: + return RawComparisonOp::LT; + case RawComparisonOp::GE: + return RawComparisonOp::LE; + case RawComparisonOp::EQ: + case RawComparisonOp::NE: + break; + } + __builtin_unreachable(); +} + +template +void execute_raw_comparison(const uint8_t* values, size_t num_values, const Field& literal, + RawComparisonOp op, uint8_t* matches) { + const T rhs = literal.get(); + for (size_t row = 0; row < num_values; ++row) { + if (matches[row] == 0) { + continue; + } + const T lhs = unaligned_load(values + row * sizeof(T)); + bool keep = false; + switch (op) { + case RawComparisonOp::EQ: + keep = Compare::equal(lhs, rhs); + break; + case RawComparisonOp::NE: + keep = Compare::not_equal(lhs, rhs); + break; + case RawComparisonOp::LT: + keep = Compare::less(lhs, rhs); + break; + case RawComparisonOp::LE: + keep = Compare::less_equal(lhs, rhs); + break; + case RawComparisonOp::GT: + keep = Compare::greater(lhs, rhs); + break; + case RawComparisonOp::GE: + keep = Compare::greater_equal(lhs, rhs); + break; + } + matches[row] = static_cast(keep); + } +} + +} // namespace + VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) { _function_name = _fn.name.function_name; } @@ -339,6 +417,71 @@ Status VectorizedFnCall::execute_column(VExprContext* context, const Block* bloc return _do_execute(context, block, selector, count, result_column, nullptr); } +bool VectorizedFnCall::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + if (data_type == nullptr || !raw_comparison_op(_function_name, false).has_value()) { + return false; + } + auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + if (!slot_literal.has_value() || slot_literal->slot_index != column_id || + slot_literal->literal.is_null()) { + return false; + } + const auto raw_type = remove_nullable(data_type); + if (!remove_nullable(slot_literal->slot_type)->equals(*raw_type) || + !remove_nullable(slot_literal->literal_type)->equals(*raw_type)) { + return false; + } + const auto primitive_type = raw_type->get_primitive_type(); + return primitive_type == TYPE_INT || primitive_type == TYPE_BIGINT || + primitive_type == TYPE_FLOAT || primitive_type == TYPE_DOUBLE; +} + +Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const { + if (!can_execute_on_raw_fixed_values(data_type, column_id)) { + return Status::NotSupported("Expression {} cannot evaluate raw fixed-width values", + expr_name()); + } + DORIS_CHECK(values != nullptr || num_values == 0); + DORIS_CHECK(matches != nullptr || num_values == 0); + const auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + DORIS_CHECK(slot_literal.has_value()); + const auto op = raw_comparison_op(_function_name, slot_literal->literal_on_left); + DORIS_CHECK(op.has_value()); + const auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + const size_t expected_width = primitive_type == TYPE_INT || primitive_type == TYPE_FLOAT + ? sizeof(uint32_t) + : sizeof(uint64_t); + if (value_width != expected_width) { + return Status::Corruption("Raw expression width {} does not match expected {}", value_width, + expected_width); + } + switch (primitive_type) { + case TYPE_INT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_BIGINT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_FLOAT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_DOUBLE: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + default: + __builtin_unreachable(); + } + return Status::OK(); +} + const std::string& VectorizedFnCall::expr_name() const { return _expr_name; } @@ -386,8 +529,12 @@ bool VectorizedFnCall::is_deterministic() const { } bool VectorizedFnCall::is_safe_to_execute_on_selected_rows() const { - static const std::set ERROR_PRESERVING_FUNCTIONS = {"assert_true"}; - return !ERROR_PRESERVING_FUNCTIONS.contains(_function_name) && + static const std::set TOTAL_PREDICATE_FUNCTIONS = { + "eq", "ne", "lt", "le", "gt", "ge", "in", "not_in", "is_null_pred", "is_not_null_pred"}; + // Selected-row execution may hide data-dependent errors in rows rejected by an earlier + // predicate. Keep function calls unsafe by default and opt in only operations that are total + // for their input domain; child checks then reject expressions such as gt(mod(x, -1), 0). + return TOTAL_PREDICATE_FUNCTIONS.contains(_function_name) && VExpr::is_safe_to_execute_on_selected_rows(); } diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index 765113f0b1cafc..749133a098c625 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -58,6 +58,11 @@ class VectorizedFnCall : public VExpr { Status execute_runtime_filter(VExprContext* context, const Block* block, const uint8_t* __restrict filter, size_t count, ColumnPtr& result_column, ColumnPtr* arg_column) const override; + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override; + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const override; Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index e7f9851b61a44e..dd61d299a7fe0e 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -164,6 +165,15 @@ class VExpr { return execute_column_impl(context, block, selector, count, result_column); } + template , int> = 0> + Status execute_column(VExprContext* context, const Block* block, SelectorType* selector, + size_t count, ColumnPtr& result_column) const { + // format_v2 treats selectors as read-only; the branch-4.1 expression API still exposes + // a mutable pointer even though implementations must not modify it. + return execute_column(context, block, const_cast(selector), count, + result_column); + } + virtual Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const { @@ -179,6 +189,19 @@ class VExpr { uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, bool* can_filter_all) const; + // Raw fixed-width evaluation is an optional expression capability used before a storage reader + // materializes a column. `matches` is ANDed in place; callers handle NULL rows separately + // because raw value streams contain only non-NULL payloads. + virtual bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + return false; + } + virtual Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, const DataTypePtr& data_type, + int column_id, uint8_t* matches) const { + return Status::NotSupported("{} cannot evaluate raw fixed-width values", expr_name()); + } + // `is_blockable` means this expr will be blocked in `execute` (e.g. AI Function, Remote Function) [[nodiscard]] virtual bool is_blockable() const { return std::any_of(_children.begin(), _children.end(), diff --git a/be/src/exprs/vruntimefilter_wrapper.cpp b/be/src/exprs/vruntimefilter_wrapper.cpp index 370575ba8661a1..94a2b2221def2e 100644 --- a/be/src/exprs/vruntimefilter_wrapper.cpp +++ b/be/src/exprs/vruntimefilter_wrapper.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type.h" #include "core/types.h" #include "exec/common/util.hpp" +#include "exprs/runtime_filter_expr.h" #include "exprs/vslot_ref.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -84,6 +85,17 @@ Status VRuntimeFilterWrapper::clone_node(VExprSPtr* cloned_expr) const { return Status::OK(); } +VExprSPtr VRuntimeFilterWrapper::to_runtime_filter_expr() const { + // FileScannerV2 backports master TableReader code, whose localization boundary recognizes the + // renamed wrapper type; preserve the same implementation and counters across that boundary. + auto runtime_filter = + RuntimeFilterExpr::create_shared(clone_texpr_node(), _impl, _ignore_thredhold, + _null_aware, _filter_id, _sampling_frequency); + runtime_filter->attach_profile_counter(_rf_input_rows, _rf_filter_rows, + _always_true_filter_rows); + return runtime_filter; +} + Status VRuntimeFilterWrapper::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); diff --git a/be/src/exprs/vruntimefilter_wrapper.h b/be/src/exprs/vruntimefilter_wrapper.h index 1d903832de353b..d6b736eeebee49 100644 --- a/be/src/exprs/vruntimefilter_wrapper.h +++ b/be/src/exprs/vruntimefilter_wrapper.h @@ -83,6 +83,7 @@ class VRuntimeFilterWrapper final : public VExpr { VExprSPtr get_impl() const override { return _impl; } void set_impl(VExprSPtr impl) { _impl = std::move(impl); } + VExprSPtr to_runtime_filter_expr() const; Status clone_node(VExprSPtr* cloned_expr) const override; void attach_profile_counter(std::shared_ptr rf_input_rows, diff --git a/be/src/format/arrow/arrow_stream_reader.cpp b/be/src/format/arrow/arrow_stream_reader.cpp index 5fa9143dab2ac8..26d8aacd350fa8 100644 --- a/be/src/format/arrow/arrow_stream_reader.cpp +++ b/be/src/format/arrow/arrow_stream_reader.cpp @@ -58,7 +58,8 @@ ArrowStreamReader::~ArrowStreamReader() = default; Status ArrowStreamReader::init_reader() { io::FileReaderSPtr file_reader; RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &file_reader, _state, false)); - _file_reader = _io_ctx ? std::make_shared(std::move(file_reader), + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), _io_ctx->file_reader_stats) : file_reader; _pip_stream = ArrowPipInputStream::create_unique(_file_reader); diff --git a/be/src/format/csv/csv_reader.cpp b/be/src/format/csv/csv_reader.cpp index 7bc340e7f21774..d6bee41acdcf91 100644 --- a/be/src/format/csv/csv_reader.cpp +++ b/be/src/format/csv/csv_reader.cpp @@ -606,7 +606,8 @@ Status CsvReader::_create_file_reader(bool need_schema) { io::DelegateReader::AccessMode::SEQUENTIAL, _io_ctx, io::PrefetchRange(_range.start_offset, _range.start_offset + _range.size))); } - _file_reader = _io_ctx ? std::make_shared(std::move(file_reader), + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), _io_ctx->file_reader_stats) : file_reader; } diff --git a/be/src/format/file_reader/new_plain_text_line_reader.cpp b/be/src/format/file_reader/new_plain_text_line_reader.cpp index a4df306bf49db5..25a7a0a7ac2476 100644 --- a/be/src/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/format/file_reader/new_plain_text_line_reader.cpp @@ -45,7 +45,6 @@ // leave these 2 size small for debugging namespace doris { -#include "common/compile_check_begin.h" const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, const size_t length) { if (_skip_utf8_bom && !_first_record_prefix_checked && _idx == 0) { constexpr uint8_t UTF8_BOM[] = {0xEF, 0xBB, 0xBF}; @@ -54,8 +53,12 @@ const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, con if (std::memcmp(start, UTF8_BOM, prefix_size) != 0) { _first_record_prefix_checked = true; } else if (length < UTF8_BOM_SIZE) { + // The input buffer can end inside the BOM. Wait for the remaining prefix bytes instead + // of feeding a partial BOM into START, which would permanently select NORMAL state. return nullptr; } else { + // Keep offsets relative to the original buffer. CsvReader removes these three bytes + // from the returned line and shifts separator positions by the same amount. _idx = UTF8_BOM_SIZE; _first_record_prefix_checked = true; } @@ -64,7 +67,10 @@ const uint8_t* EncloseCsvLineReaderCtx::read_line_impl(const uint8_t* start, con // causing parse column separator error. if (_state.curr_state == ReaderState::NORMAL || _state.curr_state == ReaderState::MATCH_ENCLOSE) { - _idx -= std::min(_column_sep_len - 1, _idx); + const size_t last_column_sep_end = + _column_sep_positions.empty() ? 0 : _column_sep_positions.back() + _column_sep_len; + DORIS_CHECK_LE(last_column_sep_end, _idx); + _idx -= std::min(_column_sep_len - 1, _idx - last_column_sep_end); } _total_len = length; size_t bound = update_reading_bound(start); @@ -148,7 +154,8 @@ void EncloseCsvLineReaderCtx::_on_start(const uint8_t* start, size_t& len) { void EncloseCsvLineReaderCtx::_on_normal(const uint8_t* start, size_t& len) { const uint8_t* curr_start = start + _idx; - size_t curr_len = len - _idx; + const size_t field_search_bound = _result == nullptr ? len : _result - start; + const size_t curr_len = field_search_bound - _idx; const uint8_t* col_sep_pos = find_col_sep_func(curr_start, curr_len, _column_sep.c_str(), _column_sep_len); @@ -204,7 +211,8 @@ void EncloseCsvLineReaderCtx::_on_pre_match_enclose(const uint8_t* start, size_t void EncloseCsvLineReaderCtx::_on_match_enclose(const uint8_t* start, size_t& len) { const uint8_t* curr_start = start + _idx; - size_t curr_len = len - _idx; + const size_t field_search_bound = _result == nullptr ? len : _result - start; + const size_t curr_len = field_search_bound - _idx; const uint8_t* delim_pos = find_col_sep_func(curr_start, curr_len, _column_sep.c_str(), _column_sep_len); @@ -541,5 +549,4 @@ void NewPlainTextLineReader::_collect_profile_before_close() { } } -#include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/format/file_reader/new_plain_text_line_reader.h b/be/src/format/file_reader/new_plain_text_line_reader.h index 3f18f28c673ad1..ed7f80493b02d4 100644 --- a/be/src/format/file_reader/new_plain_text_line_reader.h +++ b/be/src/format/file_reader/new_plain_text_line_reader.h @@ -29,7 +29,6 @@ #include "runtime/runtime_profile.h" namespace doris { -#include "common/compile_check_begin.h" namespace io { struct IOContext; } @@ -213,9 +212,11 @@ class EncloseCsvLineReaderCtx final : public BaseTextLineReaderContext(std::move(file_reader), + _file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(std::move(file_reader), _io_ctx->file_reader_stats) : file_reader; } diff --git a/be/src/format/native/native_reader.cpp b/be/src/format/native/native_reader.cpp index 0cb0c1ca72a6c0..408e71213e089c 100644 --- a/be/src/format/native/native_reader.cpp +++ b/be/src/format/native/native_reader.cpp @@ -19,6 +19,8 @@ #include +#include + #include "core/block/block.h" #include "format/native/native_format.h" #include "io/file_factory.h" @@ -40,6 +42,16 @@ NativeReader::NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& _io_ctx(io_ctx), _state(state) {} +NativeReader::NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, + const TFileRangeDesc& range, + std::shared_ptr io_ctx_holder, RuntimeState* state) + : _profile(profile), + _scan_params(params), + _scan_range(range), + _io_ctx(io_ctx_holder ? io_ctx_holder.get() : nullptr), + _io_ctx_holder(std::move(io_ctx_holder)), + _state(state) {} + NativeReader::~NativeReader() { (void)close(); } @@ -129,15 +141,20 @@ Status NativeReader::init_reader() { io::FileReaderOptions reader_options = FileFactory::get_reader_options(_state, file_description); - auto reader_res = io::DelegateReader::create_file_reader( - _profile, system_properties, file_description, reader_options, - io::DelegateReader::AccessMode::RANDOM, _io_ctx); + auto reader_res = + _io_ctx_holder ? io::DelegateReader::create_file_reader( + _profile, system_properties, file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, + std::static_pointer_cast(_io_ctx_holder)) + : io::DelegateReader::create_file_reader( + _profile, system_properties, file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, _io_ctx); if (!reader_res.has_value()) { return reader_res.error(); } _file_reader = reader_res.value(); - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { _file_reader = std::make_shared(_file_reader, _io_ctx->file_reader_stats); } diff --git a/be/src/format/native/native_reader.h b/be/src/format/native/native_reader.h index 65d70816628eea..d72544ea472c08 100644 --- a/be/src/format/native/native_reader.h +++ b/be/src/format/native/native_reader.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -53,6 +54,10 @@ class NativeReader : public GenericReader { NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, io::IOContext* io_ctx, RuntimeState* state); + NativeReader(RuntimeProfile* profile, const TFileScanRangeParams& params, + const TFileRangeDesc& range, std::shared_ptr io_ctx_holder, + RuntimeState* state); + ~NativeReader() override; // Initialize underlying file reader and any format specific state. @@ -82,6 +87,7 @@ class NativeReader : public GenericReader { io::FileReaderSPtr _file_reader; io::IOContext* _io_ctx = nullptr; + std::shared_ptr _io_ctx_holder; RuntimeState* _state = nullptr; bool _eof = false; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 8b42d2cceb3057..d2224ff2ccec5d 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -2481,7 +2481,7 @@ Status OrcReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { _reader_metrics.SelectedRowGroupCount); COUNTER_UPDATE(_orc_profile.evaluated_row_group_count, _reader_metrics.EvaluatedRowGroupCount); - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { _io_ctx->file_reader_stats->read_rows += _reader_metrics.ReadRowCount; } } @@ -3492,7 +3492,7 @@ void ORCFileInputStream::_build_small_ranges_input_stripe_streams( std::make_shared(_profile, _file_reader, merged_range); std::shared_ptr tracing_file_reader; - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { tracing_file_reader = std::make_shared( std::move(merge_range_file_reader), _io_ctx->file_reader_stats); } else { @@ -3525,7 +3525,8 @@ void ORCFileInputStream::_build_large_ranges_input_stripe_streams( for (const auto& range : ranges) { auto stripe_stream_input_stream = std::make_shared( getName(), - _io_ctx ? std::make_shared(_file_reader, + _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(_file_reader, _io_ctx->file_reader_stats) : _file_reader, _io_ctx, _profile); diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 7305e5637ff8dd..3f77d41f63990f 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -836,9 +836,10 @@ class ORCFileInputStream : public orc::InputStream, public ProfileCollector { : _file_name(file_name), _inner_reader(inner_reader), _file_reader(inner_reader), - _tracing_file_reader(io_ctx ? std::make_shared( - _file_reader, io_ctx->file_reader_stats) - : _file_reader), + _tracing_file_reader(io_ctx && io_ctx->file_reader_stats + ? std::make_shared( + _file_reader, io_ctx->file_reader_stats) + : _file_reader), _orc_once_max_read_bytes(orc_once_max_read_bytes), _orc_max_merge_distance_bytes(orc_max_merge_distance_bytes), _io_ctx(io_ctx), diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index a1e376ce55ebfe..5fc1c92d06d0e7 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -324,10 +324,18 @@ Status ParquetReader::_open_file() { _scan_range.__isset.modification_time ? _scan_range.modification_time : 0; io::FileReaderOptions reader_options = FileFactory::get_reader_options(_state, _file_description); - _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( - _profile, _system_properties, _file_description, reader_options, - io::DelegateReader::AccessMode::RANDOM, _io_ctx)); - _tracing_file_reader = _io_ctx ? std::make_shared( + if (_io_ctx_holder) { + _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( + _profile, _system_properties, _file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, + std::static_pointer_cast(_io_ctx_holder))); + } else { + _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( + _profile, _system_properties, _file_description, reader_options, + io::DelegateReader::AccessMode::RANDOM, _io_ctx)); + } + _tracing_file_reader = _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared( _file_reader, _io_ctx->file_reader_stats) : _file_reader; } @@ -833,7 +841,7 @@ Status ParquetReader::_next_row_group_reader() { } _reader_statistics.read_rows += candidate_row_ranges.count(); - if (_io_ctx) { + if (_io_ctx && _io_ctx->file_reader_stats) { _io_ctx->file_reader_stats->read_rows += candidate_row_ranges.count(); } @@ -883,7 +891,8 @@ Status ParquetReader::_next_row_group_reader() { : _file_reader; } _current_group_reader.reset(new RowGroupReader( - _io_ctx ? std::make_shared(group_file_reader, + _io_ctx && _io_ctx->file_reader_stats + ? std::make_shared(group_file_reader, _io_ctx->file_reader_stats) : group_file_reader, _read_table_columns, _current_row_group_index.row_group_id, row_group, _ctz, _io_ctx, diff --git a/be/src/format_v2/deletion_vector.h b/be/src/format/table/deletion_vector.h similarity index 68% rename from be/src/format_v2/deletion_vector.h rename to be/src/format/table/deletion_vector.h index 2c89771b2e6b0e..12ec7dbc6ebc80 100644 --- a/be/src/format_v2/deletion_vector.h +++ b/be/src/format/table/deletion_vector.h @@ -17,10 +17,22 @@ #pragma once +#include +#include + #include "roaring/roaring64map.hh" namespace doris { +// Doris materializes an Iceberg deletion vector as one buffer. This is an implementation limit, +// not an Iceberg format limit. +inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30; +inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12; + +// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum serialized size is below this +// limit. Keep its guard independent from the Iceberg capability limit. +inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30; + // A deletion vector is already a bitmap on the wire. Keep decoded DVs compressed in the // query-local cache instead of expanding every set bit into an int64_t. Position delete files use // a different representation because their input is a stream of (file_path, row_position) rows. diff --git a/be/src/format/table/deletion_vector_reader.cpp b/be/src/format/table/deletion_vector_reader.cpp index bfe34a5f555f94..8e8404839be9a8 100644 --- a/be/src/format/table/deletion_vector_reader.cpp +++ b/be/src/format/table/deletion_vector_reader.cpp @@ -17,29 +17,145 @@ #include "format/table/deletion_vector_reader.h" +#include + #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "util/block_compression.h" namespace doris { +namespace { + +constexpr int64_t ICEBERG_DELETION_VECTOR_MIN_BYTES = + static_cast(ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES); +constexpr int64_t PAIMON_DELETION_VECTOR_MIN_BYTES = 8; +constexpr int64_t PAIMON_LENGTH_PREFIX_BYTES = 4; + +enum class DeletionVectorSizeLimitStatus { + DATA_QUALITY, + NOT_SUPPORTED, +}; + +Status validate_deletion_vector_read_range(const char* description, int64_t offset, int64_t size, + int64_t min_size, int64_t max_size, + DeletionVectorSizeLimitStatus size_limit_status, + size_t& bytes_read) { + if (offset < 0) { + return Status::DataQualityError("{} offset must be non-negative: {}", description, offset); + } + if (size < min_size) { + return Status::DataQualityError("{} size too small: {}, minimum: {}", description, size, + min_size); + } + if (size > max_size) { + if (size_limit_status == DeletionVectorSizeLimitStatus::NOT_SUPPORTED) { + return Status::NotSupported("{} size exceeds Doris supported limit: {}, limit: {}", + description, size, max_size); + } + return Status::DataQualityError("{} size exceeds limit: {}, limit: {}", description, size, + max_size); + } + if (offset > std::numeric_limits::max() - size) { + return Status::DataQualityError("{} offset plus size overflows: offset {}, size {}", + description, offset, size); + } + bytes_read = static_cast(size); + return Status::OK(); +} + +} // namespace + +Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, + size_t& bytes_read) { + return validate_deletion_vector_read_range( + "Iceberg deletion vector", offset, size, ICEBERG_DELETION_VECTOR_MIN_BYTES, + MAX_ICEBERG_DELETION_VECTOR_BYTES, DeletionVectorSizeLimitStatus::NOT_SUPPORTED, + bytes_read); +} + +Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, + size_t& bytes_read) { + if (length < 0) { + return Status::DataQualityError("Paimon deletion vector length must be non-negative: {}", + length); + } + if (length > std::numeric_limits::max() - PAIMON_LENGTH_PREFIX_BYTES) { + return Status::DataQualityError("Paimon deletion vector length overflows: {}", length); + } + return validate_deletion_vector_read_range( + "Paimon deletion vector", offset, length + PAIMON_LENGTH_PREFIX_BYTES, + PAIMON_DELETION_VECTOR_MIN_BYTES, MAX_PAIMON_DELETION_VECTOR_BYTES, + DeletionVectorSizeLimitStatus::DATA_QUALITY, bytes_read); +} + +DeletionVectorReader::~DeletionVectorReader() { + // The file reader may retain the child IOContext. Destroy it before merging and before the + // child statistics storage goes away. + _file_reader.reset(); + _merge_io_statistics(); +} + +void DeletionVectorReader::_init_io_context() { + if (_parent_io_ctx == nullptr) { + return; + } + _reader_io_ctx = *_parent_io_ctx; + _reader_io_ctx.file_cache_stats = &_file_cache_stats; + _reader_io_ctx.file_reader_stats = &_file_reader_stats; + _io_ctx = &_reader_io_ctx; +} + +void DeletionVectorReader::_merge_io_statistics() { + if (_statistics_merged || _parent_io_ctx == nullptr) { + return; + } + if (_parent_io_ctx->file_cache_stats != nullptr) { + _parent_io_ctx->file_cache_stats->merge_from(_file_cache_stats); + } + if (_parent_io_ctx->file_reader_stats != nullptr) { + _parent_io_ctx->file_reader_stats->read_calls += _file_reader_stats.read_calls; + _parent_io_ctx->file_reader_stats->read_bytes += _file_reader_stats.read_bytes; + _parent_io_ctx->file_reader_stats->read_time_ns += _file_reader_stats.read_time_ns; + _parent_io_ctx->file_reader_stats->read_rows += _file_reader_stats.read_rows; + } + _statistics_merged = true; +} + Status DeletionVectorReader::open() { if (_is_opened) [[unlikely]] { return Status::OK(); } + if (_desc.start_offset < 0 || _desc.size < 0) { + return Status::DataQualityError( + "Deletion vector range must be non-negative: path {}, offset {}, size {}", + _desc.path, _desc.start_offset, _desc.size); + } + _init_system_properties(); _init_file_description(); RETURN_IF_ERROR(_create_file_reader()); - _file_size = _file_reader->size(); + const size_t file_size = _file_reader->size(); + const size_t start_offset = static_cast(_desc.start_offset); + const size_t range_size = static_cast(_desc.size); + if (start_offset > file_size || range_size > file_size - start_offset) { + return Status::DataQualityError( + "Deletion vector range exceeds file size: path {}, offset {}, size {}, file size " + "{}", + _desc.path, _desc.start_offset, _desc.size, file_size); + } _is_opened = true; return Status::OK(); } Status DeletionVectorReader::read_at(size_t offset, Slice result) { - if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + if (UNLIKELY(_parent_io_ctx && _parent_io_ctx->should_stop)) { return Status::EndOfFile("stop read."); } + if (_io_ctx != nullptr) { + _io_ctx->should_stop = _parent_io_ctx->should_stop; + } size_t bytes_read = 0; RETURN_IF_ERROR(_file_reader->read_at(offset, result, &bytes_read, _io_ctx)); if (bytes_read != result.size) [[unlikely]] { @@ -50,13 +166,16 @@ Status DeletionVectorReader::read_at(size_t offset, Slice result) { } Status DeletionVectorReader::_create_file_reader() { - if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + if (UNLIKELY(_parent_io_ctx && _parent_io_ctx->should_stop)) { return Status::EndOfFile("stop read."); } - _file_description.mtime = _range.__isset.modification_time ? _range.modification_time : 0; + _file_description.mtime = _desc.modification_time; + // Keep DV blobs on the normal remote-file path. When file cache is enabled this creates a + // CachedRemoteFileReader, so a Puffin/delete-file range is persisted in the disk File Cache; + // the query-local decoded cache above it only owns the Roaring bitmap. io::FileReaderOptions reader_options = - FileFactory::get_reader_options(_state, _file_description); + FileFactory::get_reader_options(_state->query_options(), _file_description); _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( _profile, _system_properties, _file_description, reader_options, io::DelegateReader::AccessMode::RANDOM, _io_ctx)); @@ -64,20 +183,13 @@ Status DeletionVectorReader::_create_file_reader() { } void DeletionVectorReader::_init_file_description() { - _file_description.path = _range.path; - _file_description.file_size = _range.__isset.file_size ? _range.file_size : -1; - if (_range.__isset.fs_name) { - _file_description.fs_name = _range.fs_name; - } + _file_description.path = _desc.path; + _file_description.file_size = _desc.file_size; + _file_description.fs_name = _desc.fs_name; } void DeletionVectorReader::_init_system_properties() { - if (_range.__isset.file_type) { - // for compatibility - _system_properties.system_type = _range.file_type; - } else { - _system_properties.system_type = _params.file_type; - } + _system_properties.system_type = _params.file_type; _system_properties.properties = _params.properties; _system_properties.hdfs_params = _params.hdfs_params; if (_params.__isset.broker_addresses) { @@ -86,4 +198,4 @@ void DeletionVectorReader::_init_system_properties() { } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/format/table/deletion_vector_reader.h b/be/src/format/table/deletion_vector_reader.h index 0663f3b28490ef..c187e2871687fe 100644 --- a/be/src/format/table/deletion_vector_reader.h +++ b/be/src/format/table/deletion_vector_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -24,10 +25,11 @@ #include "common/status.h" #include "format/generic_reader.h" +#include "format/table/deletion_vector.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" -#include "roaring/roaring64map.hh" +#include "io/io_common.h" #include "util/profile_collector.h" #include "util/slice.h" @@ -36,6 +38,28 @@ struct IOContext; } // namespace io namespace doris { +Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, + size_t& bytes_read); + +Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, + size_t& bytes_read); + +struct DeleteFileDesc { + enum class Format { + PAIMON, + ICEBERG, + }; + + std::string key = ""; + std::string path = ""; + std::string fs_name = ""; + int64_t start_offset = 0; + int64_t size = 0; + int64_t file_size = -1; + int64_t modification_time = 0; + Format format = Format::PAIMON; +}; + class DeletionVectorReader { ENABLE_FACTORY_CREATOR(DeletionVectorReader); @@ -43,27 +67,57 @@ class DeletionVectorReader { DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, io::IOContext* io_ctx) - : _state(state), _profile(profile), _range(range), _params(params), _io_ctx(io_ctx) {} - ~DeletionVectorReader() = default; + : _state(state), _profile(profile), _params(params), _parent_io_ctx(io_ctx) { + _desc = DeleteFileDesc { + .key = "", + .path = range.path, + .fs_name = range.__isset.fs_name ? range.fs_name : "", + .start_offset = range.start_offset, + .size = range.size, + .file_size = range.__isset.file_size ? range.file_size : -1, + .modification_time = range.__isset.modification_time ? range.modification_time : 0}; + _init_io_context(); + } + DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, + const TFileScanRangeParams& params, const DeleteFileDesc& desc, + io::IOContext* io_ctx) + : _state(state), _profile(profile), _params(params), _parent_io_ctx(io_ctx) { + _desc = desc; + _init_io_context(); + } + ~DeletionVectorReader(); + DeletionVectorReader(const DeletionVectorReader&) = delete; + DeletionVectorReader& operator=(const DeletionVectorReader&) = delete; Status open(); Status read_at(size_t offset, Slice result); + // These are deliberately exposed separately from the decoded-DV cache counters. Local I/O is + // a hit in the disk-backed File Cache; remote I/O is a File Cache miss. A decoded-DV cache hit + // does not create a DeletionVectorReader and therefore changes neither value. + const io::FileCacheStatistics& file_cache_statistics() const { return _file_cache_stats; } + private: void _init_system_properties(); void _init_file_description(); + void _init_io_context(); + void _merge_io_statistics(); Status _create_file_reader(); private: RuntimeState* _state = nullptr; RuntimeProfile* _profile = nullptr; - const TFileRangeDesc& _range; + DeleteFileDesc _desc; const TFileScanRangeParams& _params; + io::IOContext* _parent_io_ctx = nullptr; + io::IOContext _reader_io_ctx; io::IOContext* _io_ctx = nullptr; + io::FileCacheStatistics _file_cache_stats; + io::FileReaderStats _file_reader_stats; + bool _statistics_merged = false; io::FileSystemProperties _system_properties; io::FileDescription _file_description; io::FileReaderSPtr _file_reader; - int64_t _file_size = 0; bool _is_opened = false; }; } // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index 521c2c07400c44..7ee77e94eff255 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -17,7 +17,7 @@ #include "format/table/iceberg_delete_file_reader_helper.h" -#include +#include #include #include @@ -29,12 +29,13 @@ #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_dictionary.h" +#include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "exec/common/endian.h" -#include "exprs/vexpr_context.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_column_chunk_reader.h" #include "format/parquet/vparquet_reader.h" @@ -42,9 +43,10 @@ #include "format/table/iceberg_reader.h" #include "format/table/table_format_reader.h" #include "io/hdfs_builder.h" -#include "runtime/descriptors.h" #include "runtime/runtime_state.h" #include "storage/predicate/column_predicate.h" +#include "util/debug_points.h" +#include "util/hash_util.hpp" namespace doris { @@ -52,6 +54,7 @@ namespace { constexpr const char* ICEBERG_FILE_PATH = "file_path"; constexpr const char* ICEBERG_ROW_POS = "pos"; +constexpr size_t ICEBERG_DELETION_VECTOR_MIN_BYTES = 12; const std::vector DELETE_COL_NAMES {ICEBERG_FILE_PATH, ICEBERG_ROW_POS}; std::unordered_map DELETE_COL_NAME_TO_BLOCK_IDX = {{ICEBERG_FILE_PATH, 0}, @@ -90,9 +93,26 @@ Status visit_position_delete_block(const Block& block, size_t read_rows, return Status::InternalError("Position delete block is missing required columns"); } - const auto* pos_column = - assert_cast(block.get_by_position(pos_it->second).column.get()); - const auto* path_column = block.get_by_position(path_it->second).column.get(); + ColumnPtr path_column_ptr = block.get_by_position(path_it->second).column; + ColumnPtr pos_column_ptr = block.get_by_position(pos_it->second).column; + if (const auto* nullable_col = check_and_get_column(*path_column_ptr); + nullable_col != nullptr) { + if (nullable_col->has_null(0, read_rows)) { + return Status::Corruption( + "Iceberg position delete column file_path contains null values"); + } + path_column_ptr = remove_nullable(path_column_ptr); + } + if (const auto* nullable_col = check_and_get_column(*pos_column_ptr); + nullable_col != nullptr) { + if (nullable_col->has_null(0, read_rows)) { + return Status::Corruption("Iceberg position delete column pos contains null values"); + } + pos_column_ptr = remove_nullable(pos_column_ptr); + } + + const auto* pos_column = assert_cast(pos_column_ptr.get()); + const auto* path_column = path_column_ptr.get(); if (const auto* string_column = check_and_get_column(path_column); string_column != nullptr) { @@ -121,16 +141,15 @@ Status init_parquet_delete_reader(ParquetReader* reader, bool* dictionary_coded) return Status::InvalidArgument("invalid parquet delete reader arguments"); } + VExprContextSPtrs conjuncts; phmap::flat_hash_map>> slot_id_to_predicates; - RETURN_IF_ERROR(reader->init_reader(DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, {}, + RETURN_IF_ERROR(reader->init_reader(DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, conjuncts, slot_id_to_predicates, nullptr, nullptr, nullptr, nullptr, nullptr, TableSchemaChangeHelper::ConstNode::get_instance(), false)); - - std::unordered_map> - partition_columns; - std::unordered_map missing_columns; - RETURN_IF_ERROR(reader->set_fill_columns(partition_columns, missing_columns)); + // branch-4.1 initializes its projected-column state separately from init_reader(). Skipping + // this step makes every row group look like a path-only scan and yields empty batches forever. + RETURN_IF_ERROR(reader->set_fill_columns({}, {})); const tparquet::FileMetaData* meta_data = reader->get_meta_data(); *dictionary_coded = true; @@ -142,6 +161,7 @@ Status init_parquet_delete_reader(ParquetReader* reader, bool* dictionary_coded) break; } } + return Status::OK(); } @@ -150,14 +170,10 @@ Status init_orc_delete_reader(OrcReader* reader) { return Status::InvalidArgument("orc delete reader is null"); } - RETURN_IF_ERROR(reader->init_reader(&DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, {}, false, - nullptr, nullptr, nullptr, nullptr, + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(reader->init_reader(&DELETE_COL_NAMES, &DELETE_COL_NAME_TO_BLOCK_IDX, conjuncts, + false, nullptr, nullptr, nullptr, nullptr, TableSchemaChangeHelper::ConstNode::get_instance())); - - std::unordered_map> - partition_columns; - std::unordered_map missing_columns; - RETURN_IF_ERROR(reader->set_fill_columns(partition_columns, missing_columns)); return Status::OK(); } @@ -166,14 +182,14 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, if (buf == nullptr || rows_to_delete == nullptr) { return Status::InvalidArgument("invalid deletion vector decode arguments"); } - if (buffer_size < 12) { + if (buffer_size < ICEBERG_DELETION_VECTOR_MIN_BYTES) { return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); } - auto total_length = BigEndian::Load32(buf); - if (total_length + 8 != buffer_size) { + const uint32_t total_length = BigEndian::Load32(buf); + if (static_cast(total_length) + 8 != buffer_size) { return Status::DataQualityError("Deletion vector length mismatch, expected: {}, actual: {}", - total_length + 8, buffer_size); + static_cast(total_length) + 8, buffer_size); } constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; @@ -181,8 +197,17 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, return Status::DataQualityError("Deletion vector magic number mismatch"); } + const uint32_t expected_crc = BigEndian::Load32(buf + sizeof(total_length) + total_length); + const uint32_t actual_crc = + HashUtil::zlib_crc_hash(buf + sizeof(total_length), total_length, 0); + if (actual_crc != expected_crc) { + return Status::DataQualityError("Deletion vector CRC32 mismatch, expected: {}, actual: {}", + expected_crc, actual_crc); + } + try { - *rows_to_delete |= roaring::Roaring64Map::readSafe(buf + 8, buffer_size - 12); + *rows_to_delete |= roaring::Roaring64Map::readSafe( + buf + 8, buffer_size - ICEBERG_DELETION_VECTOR_MIN_BYTES); } catch (const std::runtime_error& e) { return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); } @@ -195,7 +220,12 @@ IcebergDeleteFileIOContext::IcebergDeleteFileIOContext(RuntimeState* state) { io_ctx.file_cache_stats = &file_cache_stats; io_ctx.file_reader_stats = &file_reader_stats; if (state != nullptr) { + // branch-4.1 has no shared FileScanIOContext helper; keep delete-file reads attributed to + // the parent query and classified identically to ordinary external scans. io_ctx.query_id = &state->query_id(); + if (state->query_options().query_type == TQueryType::SELECT) { + io_ctx.reader_type = ReaderType::READER_QUERY; + } } } @@ -227,6 +257,25 @@ bool is_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file) { return delete_file.__isset.content && delete_file.content == 3; } +std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, + const TIcebergDeleteFileDesc& delete_file) { + return fmt::format("delete_dv_{}:{}{}:{}#{}#{}", data_file_path.size(), data_file_path, + delete_file.path.size(), delete_file.path, delete_file.content_offset, + delete_file.content_size_in_bytes); +} + +Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& delete_file, + size_t& bytes_read) { + if (!delete_file.__isset.path || !delete_file.__isset.content_offset || + !delete_file.__isset.content_size_in_bytes) { + return Status::DataQualityError( + "Iceberg deletion vector descriptor misses " + "path/content_offset/content_size_in_bytes"); + } + return validate_iceberg_deletion_vector_read_range( + delete_file.content_offset, delete_file.content_size_in_bytes, bytes_read); +} + Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, IcebergPositionDeleteVisitor* visitor) { @@ -255,18 +304,23 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi while (!eof) { Block block; if (dictionary_coded) { - block.insert(ColumnWithTypeAndName(ColumnDictI32::create(), - std::make_shared(), - ICEBERG_FILE_PATH)); + block.insert(ColumnWithTypeAndName( + ColumnNullable::create(ColumnDictI32::create(), ColumnUInt8::create()), + make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); } else { - block.insert(ColumnWithTypeAndName(ColumnString::create(), - std::make_shared(), - ICEBERG_FILE_PATH)); + block.insert(ColumnWithTypeAndName( + make_nullable(std::make_shared()), ICEBERG_FILE_PATH)); } - block.insert(ColumnWithTypeAndName(ColumnInt64::create(), - std::make_shared(), ICEBERG_ROW_POS)); + block.insert(ColumnWithTypeAndName(make_nullable(std::make_shared()), + ICEBERG_ROW_POS)); size_t read_rows = 0; RETURN_IF_ERROR(reader.get_next_block(&block, &read_rows, &eof)); + if (read_rows == 0 && !eof) { + // Position-delete scans have no predicates, so an empty non-EOF batch cannot + // make progress and must not spin forever on a malformed reader state. + return Status::InternalError( + "Parquet position delete reader returned an empty non-EOF batch"); + } RETURN_IF_ERROR(visit_position_delete_block(block, read_rows, visitor)); } return Status::OK(); @@ -297,14 +351,17 @@ Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_fi Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, - roaring::Roaring64Map* rows_to_delete) { + DeletionVector* rows_to_delete) { if (options.state == nullptr || options.profile == nullptr || options.scan_params == nullptr || options.io_ctx == nullptr || rows_to_delete == nullptr) { return Status::InvalidArgument("invalid deletion vector reader options"); } - if (!delete_file.__isset.content_offset || !delete_file.__isset.content_size_in_bytes) { - return Status::InternalError("Deletion vector is missing content offset or length"); - } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(delete_file, bytes_read)); + DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.io_error", + { return Status::IOError("injected Iceberg deletion vector read failure"); }); + DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.should_stop", + { return Status::EndOfFile("stop read."); }); TFileRangeDesc delete_range = build_iceberg_delete_file_range(delete_file.path); if (options.fs_name != nullptr && !options.fs_name->empty()) { @@ -313,14 +370,24 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, delete_range.start_offset = delete_file.content_offset; delete_range.size = delete_file.content_size_in_bytes; + // Iceberg v3 deletion-vector-v1 blobs are uncompressed and metadata provides the exact range. + // Parse the Puffin footer first if future blob types or compression codecs are supported. DeletionVectorReader dv_reader(options.state, options.profile, *options.scan_params, delete_range, options.io_ctx); RETURN_IF_ERROR(dv_reader.open()); - std::vector buf(delete_range.size); - RETURN_IF_ERROR(dv_reader.read_at(delete_range.start_offset, - {buf.data(), cast_set(delete_range.size)})); - return decode_deletion_vector_buffer(buf.data(), delete_range.size, rows_to_delete); + std::vector buf(bytes_read); + const auto read_status = dv_reader.read_at(delete_range.start_offset, {buf.data(), bytes_read}); + if (options.deletion_vector_file_cache_stats != nullptr) { + options.deletion_vector_file_cache_stats->merge_from(dv_reader.file_cache_statistics()); + } + RETURN_IF_ERROR(read_status); + return decode_deletion_vector_buffer(buf.data(), bytes_read, rows_to_delete); +} + +Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete) { + return decode_deletion_vector_buffer(buf, buffer_size, rows_to_delete); } } // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.h b/be/src/format/table/iceberg_delete_file_reader_helper.h index a6771212851ae8..adc0ef196f4b75 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.h +++ b/be/src/format/table/iceberg_delete_file_reader_helper.h @@ -26,6 +26,7 @@ #include #include "common/status.h" +#include "format/table/deletion_vector.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" @@ -50,6 +51,9 @@ struct IcebergDeleteFileReaderOptions { io::IOContext* io_ctx = nullptr; FileMetaCache* meta_cache = nullptr; const std::string* fs_name = nullptr; + // Optional per-DV attribution. The reader still merges these values into io_ctx so the + // scanner-wide File Cache profile remains complete. + io::FileCacheStatistics* deletion_vector_file_cache_stats = nullptr; size_t batch_size = 102400; }; @@ -67,12 +71,21 @@ TFileRangeDesc build_iceberg_delete_file_range(const std::string& path); bool is_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file); +std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, + const TIcebergDeleteFileDesc& delete_file); + +Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& delete_file, + size_t& bytes_read); + +Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete); + Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, IcebergPositionDeleteVisitor* visitor); Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, - roaring::Roaring64Map* rows_to_delete); + DeletionVector* rows_to_delete); } // namespace doris diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp new file mode 100644 index 00000000000000..0ef726318d9d9a --- /dev/null +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp @@ -0,0 +1,628 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/table/iceberg_position_delete_sys_table_reader.h" + +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/types.h" +#include "format/orc/vorc_reader.h" +#include "format/parquet/schema_desc.h" +#include "format/parquet/vparquet_reader.h" +#include "format/table/parquet_utils.h" +#include "format/table/table_format_reader.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { + +constexpr const char* kFilePathColumn = "file_path"; +constexpr const char* kPosColumn = "pos"; +constexpr const char* kRowColumn = "row"; +constexpr const char* kPartitionColumn = "partition"; +constexpr const char* kSpecIdColumn = "spec_id"; +constexpr const char* kDeleteFilePathColumn = "delete_file_path"; +constexpr const char* kContentOffsetColumn = "content_offset"; +constexpr const char* kContentSizeInBytesColumn = "content_size_in_bytes"; +constexpr const char* kIcebergOrcAttribute = "iceberg.id"; +constexpr int kPositionDeleteContent = 1; + +bool block_has_row(const Block& block, size_t row) { + return block.columns() > 0 && row < block.rows(); +} + +void insert_int64_nullable(MutableColumnPtr& column, const int64_t* value) { + if (value == nullptr) { + parquet_utils::insert_null(column); + } else { + parquet_utils::insert_int64(column, *value); + } +} + +// Fail loudly if the filled output block is malformed. Each output column must be produced by a +// file slot; a projected system-table column that is not backed by a file slot would be skipped +// during fill and left shorter than the rest, producing a block with inconsistent column lengths. +void check_output_columns_aligned(const MutableColumns& columns) { + if (columns.empty()) { + return; + } + const size_t expected_rows = columns.front()->size(); + for (const auto& column : columns) { + DORIS_CHECK(column->size() == expected_rows) + << "Iceberg position delete system table output block has inconsistent column " + "sizes; a projected column is not backed by a file slot"; + } +} + +const ColumnString* get_string_column(const Block& block, const std::string& name) { + auto pos = block.get_position_by_name(name); + if (pos < 0) { + return nullptr; + } + return check_and_get_column(block.get_by_position(pos).column.get()); +} + +const ColumnInt64* get_int64_column(const Block& block, const std::string& name) { + auto pos = block.get_position_by_name(name); + if (pos < 0) { + return nullptr; + } + return check_and_get_column(block.get_by_position(pos).column.get()); +} + +const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { + if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { + return nullptr; + } + return field_ptr.field_ptr.get(); +} + +const schema::external::TField* find_current_schema_field(const TFileScanRangeParams* params, + const std::string& name) { + if (params == nullptr || !params->__isset.history_schema_info || + params->history_schema_info.empty()) { + return nullptr; + } + const schema::external::TSchema* schema = ¶ms->history_schema_info.front(); + if (params->__isset.current_schema_id) { + for (const auto& candidate : params->history_schema_info) { + if (candidate.__isset.schema_id && candidate.schema_id == params->current_schema_id) { + schema = &candidate; + break; + } + } + } + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return nullptr; + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.name && field->name == name) { + return field; + } + } + return nullptr; +} + +template +std::shared_ptr create_position_delete_root_node( + const ReadColumns& read_columns) { + auto root_node = std::make_shared(); + for (const auto& column : read_columns) { + if (column.name == kRowColumn) { + continue; + } + root_node->add_children(column.name, column.name, + TableSchemaChangeHelper::ConstNode::get_instance()); + } + return root_node; +} + +} // namespace + +IcebergPositionDeleteSysTableReader::IcebergPositionDeleteSysTableReader( + const std::vector& file_slot_descs, RuntimeState* state, + RuntimeProfile* profile, const TFileRangeDesc& range, + const TFileScanRangeParams* range_params, std::shared_ptr io_ctx, + FileMetaCache* meta_cache) + : _file_slot_descs(file_slot_descs), + _state(state), + _profile(profile), + _range(range), + _range_params(range_params), + _io_ctx(std::move(io_ctx)) { + _meta_cache = meta_cache; +} + +IcebergPositionDeleteSysTableReader::~IcebergPositionDeleteSysTableReader() = default; + +Status IcebergPositionDeleteSysTableReader::init_reader() { + if (_state == nullptr || _profile == nullptr || _range_params == nullptr || + _io_ctx == nullptr) { + return Status::InvalidArgument( + "invalid Iceberg position delete system table reader context"); + } + if (!_range.__isset.table_format_params || !_range.table_format_params.__isset.iceberg_params) { + return Status::InternalError("Iceberg position delete system table range misses params"); + } + + _iceberg_file_desc = &_range.table_format_params.iceberg_params; + if (!_iceberg_file_desc->__isset.delete_files || _iceberg_file_desc->delete_files.size() != 1) { + return Status::InternalError( + "Iceberg position delete system table range should contain exactly one delete " + "file"); + } + _delete_file_desc = _iceberg_file_desc->delete_files.data(); + if (is_iceberg_deletion_vector(*_delete_file_desc)) { + _delete_file_kind = DeleteFileKind::DELETION_VECTOR; + } else if (_delete_file_desc->__isset.content && + _delete_file_desc->content == kPositionDeleteContent) { + _delete_file_kind = DeleteFileKind::POSITION_DELETE; + } else if (!_delete_file_desc->__isset.content) { + return Status::InternalError( + "Iceberg position delete system table delete file misses content"); + } else { + return Status::InternalError( + "Iceberg position delete system table does not support delete file content {}", + _delete_file_desc->content); + } + _batch_size = _state->batch_size(); + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _init_deletion_vector_reader(); + } + return _init_position_delete_reader(); +} + +Status IcebergPositionDeleteSysTableReader::get_columns( + std::unordered_map* name_to_type, + std::unordered_set* missing_cols) { + missing_cols->clear(); + for (const auto* slot : _file_slot_descs) { + name_to_type->emplace(slot->col_name(), slot->get_data_type_ptr()); + } + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::set_fill_columns( + const std::unordered_map>& + partition_columns, + const std::unordered_map& missing_columns, + const std::unordered_map& partition_value_is_null) { + if (_position_reader == nullptr) { + return Status::OK(); + } + // Native readers finalize their physical read-column plan in set_fill_columns even when all + // maps are empty. Skipping this leaves the nested reader yielding empty non-EOF batches. + return _position_reader->set_fill_columns(partition_columns, missing_columns, + partition_value_is_null); +} + +bool IcebergPositionDeleteSysTableReader::count_read_rows() { + return _delete_file_kind == DeleteFileKind::POSITION_DELETE; +} + +void IcebergPositionDeleteSysTableReader::_collect_profile_before_close() { + if (_position_reader != nullptr) { + _position_reader->collect_profile_before_close(); + } +} + +Status IcebergPositionDeleteSysTableReader::close() { + if (_position_reader != nullptr) { + RETURN_IF_ERROR(_position_reader->close()); + } + _partition_value.reset(); + _next_dv_position.reset(); + _dv_positions = roaring::Roaring64Map(); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { + if (!_delete_file_desc->__isset.file_format) { + return Status::InternalError("Iceberg position delete file misses file format"); + } + + // `row` is optional in position delete files and expensive to read, so only read it when the + // query actually projects it. Whether the delete file physically stores `row` is decided from + // the reader's own footer/type below, reusing the same reader instance that is initialized + // afterwards so the delete file is opened and its footer parsed only once. + const bool row_requested = _output_column_requested(kRowColumn); + + if (_delete_file_desc->file_format == TFileFormatType::FORMAT_PARQUET) { + auto parquet_reader = + ParquetReader::create_unique(_profile, *_range_params, _range, _batch_size, + const_cast(&_state->timezone_obj()), + _io_ctx, _state, _meta_cache); + + const FieldDescriptor* schema = nullptr; + int row_index = -1; + if (row_requested) { + RETURN_IF_ERROR(parquet_reader->get_file_metadata_schema(&schema)); + DORIS_CHECK(schema != nullptr); + row_index = schema->get_column_index(kRowColumn); + } + const bool read_row = row_requested && row_index >= 0; + _init_read_columns(read_row); + std::vector read_column_names; + read_column_names.reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + read_column_names.push_back(column.name); + } + + std::shared_ptr table_info_node = + TableSchemaChangeHelper::ConstNode::get_instance(); + if (read_row) { + const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); + if (table_row_field == nullptr) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + const auto* file_row_field = schema->get_column(static_cast(row_index)); + std::shared_ptr row_node; + // The branch-4.1 helper selects ID or name-mapping mode up front instead of doing so + // inside each nested node, so seed that mode from the projected row field. + const bool exist_field_id = file_row_field->field_id != -1; + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( + *table_row_field, *file_row_field, exist_field_id, row_node)); + auto root_node = create_position_delete_root_node(_read_columns); + root_node->add_children(kRowColumn, file_row_field->name, row_node); + table_info_node = std::move(root_node); + } + // branch-4.1's legacy reader API predates ReaderInitContext. Position-delete scans have no + // predicates, so pass empty filter state while preserving the schema mapping built above. + VExprContextSPtrs conjuncts; + phmap::flat_hash_map>> + slot_id_to_predicates; + RETURN_IF_ERROR(parquet_reader->init_reader( + read_column_names, &_read_col_name_to_block_idx, conjuncts, slot_id_to_predicates, + nullptr, nullptr, nullptr, nullptr, nullptr, std::move(table_info_node), false)); + _position_reader = std::move(parquet_reader); + return Status::OK(); + } + + if (_delete_file_desc->file_format == TFileFormatType::FORMAT_ORC) { + auto orc_reader = + OrcReader::create_unique(_profile, _state, *_range_params, _range, _batch_size, + _state->timezone(), _io_ctx, _meta_cache); + + const orc::Type* row_type = nullptr; + if (row_requested) { + const orc::Type* root_type = nullptr; + RETURN_IF_ERROR(orc_reader->get_file_type(&root_type)); + DORIS_CHECK(root_type != nullptr); + for (uint64_t i = 0; i < root_type->getSubtypeCount(); ++i) { + if (root_type->getFieldName(i) == kRowColumn) { + row_type = root_type->getSubtype(i); + break; + } + } + } + const bool read_row = row_requested && row_type != nullptr; + _init_read_columns(read_row); + std::vector read_column_names; + read_column_names.reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + read_column_names.push_back(column.name); + } + + std::shared_ptr table_info_node = + TableSchemaChangeHelper::ConstNode::get_instance(); + if (read_row) { + const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); + if (table_row_field == nullptr) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + std::shared_ptr row_node; + const bool exist_field_id = row_type->hasAttributeKey(kIcebergOrcAttribute); + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id( + *table_row_field, row_type, kIcebergOrcAttribute, exist_field_id, row_node)); + auto root_node = create_position_delete_root_node(_read_columns); + root_node->add_children(kRowColumn, kRowColumn, row_node); + table_info_node = std::move(root_node); + } + VExprContextSPtrs conjuncts; + RETURN_IF_ERROR(orc_reader->init_reader(&read_column_names, &_read_col_name_to_block_idx, + conjuncts, false, nullptr, nullptr, nullptr, + nullptr, std::move(table_info_node))); + _position_reader = std::move(orc_reader); + return Status::OK(); + } + + return Status::NotSupported("Unsupported Iceberg position delete file format {}", + _delete_file_desc->file_format); +} + +Status IcebergPositionDeleteSysTableReader::_init_deletion_vector_reader() { + if (!_delete_file_desc->__isset.referenced_data_file_path || + _delete_file_desc->referenced_data_file_path.empty()) { + return Status::InternalError("Iceberg deletion vector misses referenced data file path"); + } + + IcebergDeleteFileReaderOptions options; + options.state = _state; + options.profile = _profile; + options.scan_params = _range_params; + options.io_ctx = _io_ctx.get(); + options.meta_cache = _meta_cache; + if (_range.__isset.fs_name) { + options.fs_name = &_range.fs_name; + } + + _dv_positions = roaring::Roaring64Map(); + RETURN_IF_ERROR(read_iceberg_deletion_vector(*_delete_file_desc, options, &_dv_positions)); + _next_dv_position.emplace(_dv_positions.begin()); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::get_next_block(Block* block, size_t* read_rows, + bool* eof) { + DORIS_CHECK(_io_ctx != nullptr); + if (_io_ctx->should_stop) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _append_deletion_vector_block(block, read_rows, eof); + } + + if (_position_reader == nullptr) { + return Status::InternalError("Iceberg position delete reader is not initialized"); + } + + while (true) { + Block delete_block = _create_delete_block(); + size_t delete_rows = 0; + bool position_reader_eof = false; + RETURN_IF_ERROR(_position_reader->get_next_block(&delete_block, &delete_rows, + &position_reader_eof)); + if (delete_rows > 0) { + RETURN_IF_ERROR( + _append_position_delete_block(block, delete_block, delete_rows, read_rows)); + *eof = position_reader_eof; + return Status::OK(); + } + if (position_reader_eof) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + } +} + +Status IcebergPositionDeleteSysTableReader::_append_position_delete_block(Block* output_block, + const Block& delete_block, + size_t delete_rows, + size_t* appended_rows) { + auto columns_guard = output_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = output_block->get_name_to_pos_map(); + + for (size_t row = 0; row < delete_rows; ++row) { + for (const auto* slot : _file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, &delete_block, row, 0)); + } + } + // Every output column must be produced by a file slot above; a projected system-table column + // that is not backed by a file slot would be left short and yield a malformed block with + // inconsistent column lengths. + check_output_columns_aligned(columns); + *appended_rows = delete_rows; + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::_append_deletion_vector_block(Block* block, + size_t* read_rows, + bool* eof) { + if (!_next_dv_position.has_value() || *_next_dv_position == _dv_positions.end()) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + const size_t rows_limit = std::max(_batch_size, 1); + + auto columns_guard = block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = block->get_name_to_pos_map(); + + size_t rows = 0; + while (rows < rows_limit && *_next_dv_position != _dv_positions.end()) { + const uint64_t dv_pos = **_next_dv_position; + for (const auto* slot : _file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, nullptr, 0, dv_pos)); + } + ++(*_next_dv_position); + ++rows; + } + check_output_columns_aligned(columns); + *read_rows = rows; + *eof = *_next_dv_position == _dv_positions.end(); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableReader::_append_sys_column(MutableColumnPtr& column, + const SlotDescriptor& slot, + const Block* delete_block, + size_t source_row, uint64_t dv_pos) { + const std::string& name = slot.col_name(); + if (name == kFilePathColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_string(column, _delete_file_desc->referenced_data_file_path); + return Status::OK(); + } + const auto* path_column = get_string_column(*delete_block, kFilePathColumn); + if (path_column == nullptr || !block_has_row(*delete_block, source_row)) { + return Status::InternalError("Iceberg position delete file_path column is missing"); + } + parquet_utils::insert_string(column, path_column->get_data_at(source_row).to_string()); + return Status::OK(); + } + + if (name == kPosColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_int64(column, cast_set(dv_pos)); + return Status::OK(); + } + const auto* pos_column = get_int64_column(*delete_block, kPosColumn); + if (pos_column == nullptr || !block_has_row(*delete_block, source_row)) { + return Status::InternalError("Iceberg position delete pos column is missing"); + } + parquet_utils::insert_int64(column, pos_column->get_element(source_row)); + return Status::OK(); + } + + if (name == kRowColumn) { + if (delete_block != nullptr) { + auto row_pos = delete_block->get_position_by_name(kRowColumn); + if (row_pos >= 0) { + const auto& row_column = *delete_block->get_by_position(row_pos).column; + if (source_row < row_column.size()) { + column->insert_from(row_column, source_row); + return Status::OK(); + } + } + } + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (name == kPartitionColumn) { + return _append_partition_column(column, slot); + } + + if (name == kSpecIdColumn) { + if (_iceberg_file_desc->__isset.partition_spec_id) { + parquet_utils::insert_int32(column, + cast_set(_iceberg_file_desc->partition_spec_id)); + } else { + parquet_utils::insert_null(column); + } + return Status::OK(); + } + + if (name == kDeleteFilePathColumn) { + parquet_utils::insert_string(column, _delete_file_output_path()); + return Status::OK(); + } + + if (name == kContentOffsetColumn) { + const int64_t* value = _delete_file_desc->__isset.content_offset + ? &_delete_file_desc->content_offset + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + if (name == kContentSizeInBytesColumn) { + const int64_t* value = _delete_file_desc->__isset.content_size_in_bytes + ? &_delete_file_desc->content_size_in_bytes + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + return Status::InternalError("Unknown Iceberg position delete system table column: {}", name); +} + +Status IcebergPositionDeleteSysTableReader::_append_partition_column(MutableColumnPtr& column, + const SlotDescriptor& slot) { + if (!_iceberg_file_desc->__isset.partition_data_json || + _iceberg_file_desc->partition_data_json.empty()) { + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (!_partition_value) { + auto partition_value = slot.get_data_type_ptr()->create_column(); + auto serde = slot.get_data_type_ptr()->get_serde(); + StringRef partition_data(_iceberg_file_desc->partition_data_json.data(), + _iceberg_file_desc->partition_data_json.size()); + DataTypeSerDe::FormatOptions options = DataTypeSerDe::get_default_format_options(); + RETURN_IF_ERROR(serde->from_string(partition_data, *partition_value, options)); + DORIS_CHECK(partition_value->size() == 1); + _partition_value = std::move(partition_value); + } + column->insert_from(*_partition_value, 0); + return Status::OK(); +} + +Block IcebergPositionDeleteSysTableReader::_create_delete_block() const { + Block block; + for (const auto& column : _read_columns) { + block.insert(ColumnWithTypeAndName(column.type->create_column(), column.type, column.name)); + } + return block; +} + +bool IcebergPositionDeleteSysTableReader::_output_column_requested(const std::string& name) const { + return std::any_of(_file_slot_descs.begin(), _file_slot_descs.end(), + [&name](const SlotDescriptor* slot) { return slot->col_name() == name; }); +} + +void IcebergPositionDeleteSysTableReader::_init_read_columns(bool read_row) { + _read_columns.clear(); + _read_columns.push_back({kFilePathColumn, std::make_shared()}); + _read_columns.push_back({kPosColumn, std::make_shared()}); + if (read_row) { + for (const auto* slot : _file_slot_descs) { + if (slot->col_name() == kRowColumn) { + _read_columns.push_back({kRowColumn, slot->get_data_type_ptr()}); + break; + } + } + } + + _read_col_name_to_block_idx.clear(); + for (size_t i = 0; i < _read_columns.size(); ++i) { + _read_col_name_to_block_idx.emplace(_read_columns[i].name, cast_set(i)); + } +} + +const std::string& IcebergPositionDeleteSysTableReader::_delete_file_output_path() const { + if (_delete_file_desc->__isset.original_path && !_delete_file_desc->original_path.empty()) { + return _delete_file_desc->original_path; + } + return _delete_file_desc->path; +} + +} // namespace doris diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.h b/be/src/format/table/iceberg_position_delete_sys_table_reader.h new file mode 100644 index 00000000000000..c09a879dd0f463 --- /dev/null +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/factory_creator.h" +#include "common/status.h" +#include "core/block/column_with_type_and_name.h" +#include "format/generic_reader.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "runtime/descriptors.h" + +namespace doris { + +class FileMetaCache; +class RuntimeProfile; +class RuntimeState; + +class IcebergPositionDeleteSysTableReader : public GenericReader { + ENABLE_FACTORY_CREATOR(IcebergPositionDeleteSysTableReader); + +public: + IcebergPositionDeleteSysTableReader(const std::vector& file_slot_descs, + RuntimeState* state, RuntimeProfile* profile, + const TFileRangeDesc& range, + const TFileScanRangeParams* range_params, + std::shared_ptr io_ctx, + FileMetaCache* meta_cache); + ~IcebergPositionDeleteSysTableReader() override; + + Status init_reader(); + Status get_next_block(Block* block, size_t* read_rows, bool* eof) override; + Status get_columns(std::unordered_map* name_to_type, + std::unordered_set* missing_cols) override; + Status set_fill_columns( + const std::unordered_map>& + partition_columns, + const std::unordered_map& missing_columns, + const std::unordered_map& partition_value_is_null = {}) override; + void set_batch_size(size_t batch_size) override { _batch_size = batch_size; } + size_t get_batch_size() const override { return _batch_size; } + bool count_read_rows() override; + Status close() override; + +protected: + void _collect_profile_before_close() override; + +private: + enum class DeleteFileKind { + POSITION_DELETE, + DELETION_VECTOR, + }; + + struct ReadColumn { + std::string name; + DataTypePtr type; + }; + + Status _init_position_delete_reader(); + Status _init_deletion_vector_reader(); + Status _append_position_delete_block(Block* output_block, const Block& delete_block, + size_t delete_rows, size_t* appended_rows); + Status _append_deletion_vector_block(Block* block, size_t* read_rows, bool* eof); + Status _append_sys_column(MutableColumnPtr& column, const SlotDescriptor& slot, + const Block* delete_block, size_t source_row, uint64_t dv_pos); + Status _append_partition_column(MutableColumnPtr& column, const SlotDescriptor& slot); + Block _create_delete_block() const; + bool _output_column_requested(const std::string& name) const; + void _init_read_columns(bool read_row); + const std::string& _delete_file_output_path() const; + + const std::vector& _file_slot_descs; + RuntimeState* _state = nullptr; + RuntimeProfile* _profile = nullptr; + const TFileRangeDesc& _range; + const TFileScanRangeParams* _range_params = nullptr; + + std::shared_ptr _io_ctx; + const TIcebergFileDesc* _iceberg_file_desc = nullptr; + const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; + DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; + + size_t _batch_size = 102400; + std::unique_ptr _position_reader; + std::vector _read_columns; + std::unordered_map _read_col_name_to_block_idx; + ColumnPtr _partition_value; + roaring::Roaring64Map _dv_positions; + std::optional _next_dv_position; +}; + +} // namespace doris diff --git a/be/src/format/table/iceberg_scan_semantics.h b/be/src/format/table/iceberg_scan_semantics.h new file mode 100644 index 00000000000000..f579f063b76327 --- /dev/null +++ b/be/src/format/table/iceberg_scan_semantics.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +inline constexpr int32_t ICEBERG_SCAN_SEMANTICS_VERSION_1 = 1; + +inline bool supports_iceberg_scan_semantics_v1(const TFileScanRangeParams* params) { + // Old FE plans can carry IDs and encoded defaults too, so only this explicit version marker + // may opt a new BE into result-changing semantics during a rolling upgrade. + return params != nullptr && params->__isset.iceberg_scan_semantics_version && + params->iceberg_scan_semantics_version >= ICEBERG_SCAN_SEMANTICS_VERSION_1; +} + +} // namespace doris diff --git a/be/src/format/table/paimon_jni_reader.cpp b/be/src/format/table/paimon_jni_reader.cpp index c5de5b35eac7e3..772c97e2aa6067 100644 --- a/be/src/format/table/paimon_jni_reader.cpp +++ b/be/src/format/table/paimon_jni_reader.cpp @@ -18,10 +18,14 @@ #include "format/table/paimon_jni_reader.h" #include +#include +#include #include "core/types.h" #include "runtime/descriptors.h" +#include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "util/string_util.h" namespace doris { class RuntimeProfile; class RuntimeState; @@ -31,8 +35,14 @@ class Block; namespace doris { #include "common/compile_check_begin.h" +namespace { +constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp"; +} // namespace + const std::string PaimonJniReader::PAIMON_OPTION_PREFIX = "paimon."; const std::string PaimonJniReader::HADOOP_OPTION_PREFIX = "hadoop."; +const std::string PaimonJniReader::DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; +const std::string PaimonJniReader::DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; PaimonJniReader::PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, @@ -75,6 +85,19 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d params[PAIMON_OPTION_PREFIX + kv.first] = kv.second; } } + const std::string enable_io_manager_key = PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; + const std::string io_manager_tmp_dir_key = PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; + auto enable_io_manager_it = params.find(enable_io_manager_key); + if (enable_io_manager_it != params.end() && iequal(enable_io_manager_it->second, "true") && + params.find(io_manager_tmp_dir_key) == params.end()) { + std::vector tmp_dirs; + for (const auto& store_path : state->exec_env()->store_paths()) { + tmp_dirs.push_back(store_path.path + "/" + std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); + } + DORIS_CHECK(!tmp_dirs.empty()); + // Paimon owns its paimon-* children; Doris only supplies stable storage-root parents. + params[io_manager_tmp_dir_key] = join(tmp_dirs, ":"); + } // Prefer hadoop conf from scan node level (range_params->properties) over split level // to avoid redundant configuration in each split if (range_params->__isset.properties && !range_params->properties.empty()) { diff --git a/be/src/format/table/paimon_jni_reader.h b/be/src/format/table/paimon_jni_reader.h index 0026ab4118103d..feab10b2d39ca9 100644 --- a/be/src/format/table/paimon_jni_reader.h +++ b/be/src/format/table/paimon_jni_reader.h @@ -48,6 +48,8 @@ class PaimonJniReader : public JniReader { public: static const std::string PAIMON_OPTION_PREFIX; static const std::string HADOOP_OPTION_PREFIX; + static const std::string DORIS_ENABLE_JNI_IO_MANAGER; + static const std::string DORIS_JNI_IO_MANAGER_TMP_DIR; PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, const TFileRangeDesc& range, const TFileScanRangeParams* range_params); diff --git a/be/src/format/table/paimon_reader.cpp b/be/src/format/table/paimon_reader.cpp index 0667ad8efff7f4..9c292cdbde224b 100644 --- a/be/src/format/table/paimon_reader.cpp +++ b/be/src/format/table/paimon_reader.cpp @@ -17,14 +17,75 @@ #include "format/table/paimon_reader.h" +#include #include #include "common/status.h" +#include "exec/common/endian.h" #include "format/table/deletion_vector_reader.h" #include "runtime/runtime_state.h" namespace doris { #include "common/compile_check_begin.h" + +std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file) { + // A shared file can contain multiple vectors, so the cache identity must include its range. + return deletion_file.path + "#" + std::to_string(deletion_file.offset) + "#" + + std::to_string(deletion_file.length); +} + +Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, + size_t& bytes_read) { + if (!deletion_file.__isset.path || !deletion_file.__isset.offset || + !deletion_file.__isset.length) { + return Status::DataQualityError( + "Paimon deletion file descriptor misses path/offset/length"); + } + return validate_paimon_deletion_vector_read_range(deletion_file.offset, deletion_file.length, + bytes_read); +} + +Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* deletion_vector) { + if (deletion_vector == nullptr) { + return Status::InvalidArgument("deletion vector output must not be null"); + } + if (buf == nullptr) { + return Status::DataQualityError("Paimon deletion vector blob is null"); + } + if (buffer_size < 8) [[unlikely]] { + return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); + } + + const uint32_t actual_length = BigEndian::Load32(buf); + if (static_cast(actual_length) + 4 != buffer_size) [[unlikely]] { + return Status::DataQualityError( + "Paimon deletion vector length mismatch, expected: {}, actual: {}", + static_cast(actual_length) + 4, buffer_size); + } + + // Paimon deletion vectors always prefix the portable Roaring payload with this magic value. + constexpr char paimon_bitmap_magic[] = {'\x5E', '\x43', '\xF2', '\xD0'}; + if (memcmp(buf + sizeof(actual_length), paimon_bitmap_magic, 4) != 0) [[unlikely]] { + return Status::DataQualityError( + "Paimon deletion vector magic number mismatch, expected: {}, actual: {}", + BigEndian::Load32(paimon_bitmap_magic), + BigEndian::Load32(buf + sizeof(actual_length))); + } + + roaring::Roaring roaring_bitmap; + try { + roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8); + } catch (const std::runtime_error& e) { + return Status::RuntimeError( + "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", + e.what()); + } + + *deletion_vector |= DeletionVector(std::move(roaring_bitmap)); + return Status::OK(); +} + PaimonReader::PaimonReader(std::unique_ptr file_format_reader, RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, const TFileRangeDesc& range, @@ -55,14 +116,11 @@ Status PaimonReader::init_row_filters() { _file_format_reader->set_push_down_agg_type(TPushAggOp::NONE); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); Status create_status = Status::OK(); - - std::string key; - key.resize(deletion_file.path.size() + sizeof(deletion_file.offset)); - memcpy(key.data(), deletion_file.path.data(), deletion_file.path.size()); - memcpy(key.data() + deletion_file.path.size(), &deletion_file.offset, - sizeof(deletion_file.offset)); + const std::string key = build_paimon_deletion_vector_cache_key(deletion_file); SCOPED_TIMER(_paimon_profile.delete_files_read_time); using DeleteRows = std::vector; @@ -74,7 +132,7 @@ Status PaimonReader::init_row_filters() { delete_range.__set_fs_name(_range.fs_name); delete_range.path = deletion_file.path; delete_range.start_offset = deletion_file.offset; - delete_range.size = deletion_file.length + 4; + delete_range.size = static_cast(bytes_read); delete_range.file_size = -1; DeletionVectorReader dv_reader(_state, _profile, _params, delete_range, _io_ctx); @@ -83,47 +141,31 @@ Status PaimonReader::init_row_filters() { return nullptr; } - // the reason of adding 4: https://github.com/apache/paimon/issues/3313 - size_t bytes_read = deletion_file.length + 4; - // TODO: better way to alloc memeory std::vector buffer(bytes_read); create_status = dv_reader.read_at(deletion_file.offset, {buffer.data(), bytes_read}); if (!create_status.ok()) [[unlikely]] { return nullptr; } - // parse deletion vector - const char* buf = buffer.data(); - uint32_t actual_length; - std::memcpy(reinterpret_cast(&actual_length), buf, 4); - // change byte order to big endian - std::reverse(reinterpret_cast(&actual_length), - reinterpret_cast(&actual_length) + 4); - buf += 4; - if (actual_length != bytes_read - 4) [[unlikely]] { - create_status = Status::RuntimeError( - "DeletionVector deserialize error: length not match, " - "actual length: {}, expect length: {}", - actual_length, bytes_read - 4); + const uint32_t actual_length = BigEndian::Load32(buffer.data()); + if (static_cast(actual_length) + 4 != bytes_read) [[unlikely]] { + create_status = Status::DataQualityError( + "Paimon deletion vector length mismatch, expected: {}, actual: {}", + static_cast(actual_length) + 4, bytes_read); return nullptr; } - uint32_t magic_number; - std::memcpy(reinterpret_cast(&magic_number), buf, 4); - // change byte order to big endian - std::reverse(reinterpret_cast(&magic_number), - reinterpret_cast(&magic_number) + 4); - buf += 4; - const static uint32_t MAGIC_NUMBER = 1581511376; - if (magic_number != MAGIC_NUMBER) [[unlikely]] { - create_status = Status::RuntimeError( - "DeletionVector deserialize error: invalid magic number {}", magic_number); + constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; + if (memcmp(buffer.data() + 4, PAIMON_BITMAP_MAGIC, 4) != 0) [[unlikely]] { + create_status = Status::DataQualityError( + "Paimon deletion vector magic number mismatch, expected: {}, actual: {}", + BigEndian::Load32(PAIMON_BITMAP_MAGIC), BigEndian::Load32(buffer.data() + 4)); return nullptr; } roaring::Roaring roaring_bitmap; SCOPED_TIMER(_paimon_profile.parse_deletion_vector_time); try { - roaring_bitmap = roaring::Roaring::readSafe(buf, bytes_read - 4); + roaring_bitmap = roaring::Roaring::readSafe(buffer.data() + 8, bytes_read - 8); } catch (const std::runtime_error& e) { create_status = Status::RuntimeError( "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", diff --git a/be/src/format/table/paimon_reader.h b/be/src/format/table/paimon_reader.h index de16c63cdd9a75..e08d8bb59a8629 100644 --- a/be/src/format/table/paimon_reader.h +++ b/be/src/format/table/paimon_reader.h @@ -17,15 +17,26 @@ #pragma once +#include #include #include #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_reader.h" +#include "format/table/deletion_vector.h" #include "format/table/table_format_reader.h" namespace doris { #include "common/compile_check_begin.h" + +std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file); + +Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, + size_t& bytes_read); + +Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, + DeletionVector* deletion_vector); + class PaimonReader : public TableFormatReader, public TableSchemaChangeHelper { public: PaimonReader(std::unique_ptr file_format_reader, RuntimeProfile* profile, diff --git a/be/src/format/text/text_reader.cpp b/be/src/format/text/text_reader.cpp index 388cd56a0ddb90..bf06d915c8e9e1 100644 --- a/be/src/format/text/text_reader.cpp +++ b/be/src/format/text/text_reader.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/compiler_util.h" // IWYU pragma: keep @@ -114,8 +115,9 @@ void HiveTextFieldSplitter::_split_field_multi_char(const Slice& line, TextReader::TextReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, size_t batch_size, - io::IOContext* io_ctx) - : CsvReader(state, profile, counter, params, range, file_slot_descs, batch_size, io_ctx) {} + io::IOContext* io_ctx, std::shared_ptr io_ctx_holder) + : CsvReader(state, profile, counter, params, range, file_slot_descs, batch_size, io_ctx, + std::move(io_ctx_holder)) {} Status TextReader::_init_options() { // get column_separator and line_delimiter diff --git a/be/src/format/text/text_reader.h b/be/src/format/text/text_reader.h index 60b0fb2f8b544c..9667e05b19945d 100644 --- a/be/src/format/text/text_reader.h +++ b/be/src/format/text/text_reader.h @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -57,7 +58,7 @@ class TextReader : public CsvReader { TextReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, size_t batch_size, - io::IOContext* io_ctx); + io::IOContext* io_ctx, std::shared_ptr io_ctx_holder = nullptr); ~TextReader() override = default; diff --git a/be/src/format/transformer/vorc_transformer.cpp b/be/src/format/transformer/vorc_transformer.cpp index 3e3949cd924469..db2ca74c64acff 100644 --- a/be/src/format/transformer/vorc_transformer.cpp +++ b/be/src/format/transformer/vorc_transformer.cpp @@ -386,12 +386,19 @@ Status VOrcTransformer::collect_file_statistics_after_close(TIcebergColumnStats* const iceberg::StructType& root_struct = _iceberg_schema->root_struct(); const auto& nested_fields = root_struct.fields(); + const orc::Type& orc_root_type = reader->getType(); for (uint32_t i = 0; i < nested_fields.size(); i++) { - uint32_t orc_col_id = i + 1; // skip root struct - if (orc_col_id >= file_stats->getNumberOfColumns()) { + if (i >= orc_root_type.getSubtypeCount()) { + continue; + } + // ORC IDs are depth-first, so top-level fields after a complex field are not i + 1. + const uint64_t raw_orc_col_id = orc_root_type.getSubtype(i)->getColumnId(); + if (raw_orc_col_id >= file_stats->getNumberOfColumns()) { continue; } + // The uint32_t column-count check above makes narrowing to the ORC API width safe. + const uint32_t orc_col_id = static_cast(raw_orc_col_id); const orc::ColumnStatistics* col_stats = file_stats->getColumnStatistics(orc_col_id); if (col_stats == nullptr) { continue; diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 377a26ce530db8..e417617cf5a64f 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -108,6 +108,107 @@ instructions as well; this file adds format-v2-specific review expectations. - For JNI readers, review local/global reference lifetime, exception propagation, type conversion, thread attachment assumptions, and cleanup on partial initialization. +### Parquet Native Decode Kernel + +- Keep new production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and + performance baseline, but v2 owns an independent page/encoding reader and must not call the v1 + `ParquetColumnReader`. Do not modify `be/src/format/parquet/` for a v2 decoder change. Reimplement + the required behavior under the v2 tree and keep v1 unchanged so differential correctness and + performance results remain meaningful. +- Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema + objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, + and Dremel-level thresholds. Review constructor arguments and stored references for ownership and + lifetime; metadata owned by a temporary schema adapter must not escape into a persistent reader. +- Treat selection positions as logical Row Group rows, including null rows. Selection indices must + be sorted, unique, and bounded by the batch's logical row count. Dense identity, empty selection, + and fragmented selection must have explicit representations and tests; do not silently mix row + ordinals with non-null value ordinals or dictionary IDs. +- Verify the three decode counts separately: logical rows consumed, encoded non-null payload values + consumed, and output values materialized. Null and filtered-null runs consume no payload; + selected and filtered non-null runs both consume payload. Every page transition, skip, error, and + end-of-batch path must leave all three cursors aligned for the next call. +- A flat scalar fast path may run only when `max_repetition_level == 0`. Repeated leaves require a + definition/repetition-level plan that identifies parent-row boundaries, empty and null + collections, null ancestors, and rows spanning pages. Choose one physical leaf as the parent + shape owner; sibling leaf streams advance over the same parent-row range and validate their + payload counts instead of independently redefining the parent shape. +- The old Arrow value-reader hierarchy (`ParquetLeafBatch`, scalar/list/map/struct readers, and the + nested load/build/consume protocol) has been removed. Do not reintroduce an intermediate decoded + batch or a stateful load-before-build phase. Ordinary predicate/output scans construct + `NativeColumnReader`, consume compressed page data through the native decoder, and append directly + into the final Doris column. +- Keep logical schema changes distinct from physical decoding. The native reader materializes the + projected file type only; `ColumnMapper` and `TableReader` own every file-to-table cast. A type + mismatch at the native reader boundary is an invariant violation, not a reason to create a + reader-local conversion path or expose a decoder-owned value batch. +- `CountColumnReader` uses the v2 native `LevelReader`; it selects one representative leaf (the key + for MAP) and advances only definition/repetition levels. It exposes no value API and must not be + reused as a scan reader or expanded into a fallback path. +- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one traversal + of the owning leaf's levels. For example, `[[1, 2], NULL, []]` must yield entry counts + `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning that leaf. MAP key levels own entry + existence; value levels validate against that shape. STRUCT siblings validate parent-row + alignment. If every projected STRUCT child is missing, consume only a retained physical leaf's + levels and never materialize its payload into a temporary Doris column. +- Do not size level or selection scratch from a 16-bit batch-row assumption. A repeated parent row + can contain more level entries than the requested parent-row batch. Split large runs without + changing alternation or row-boundary semantics, and check overflow before narrowing counts. +- Decoder dispatch must reject an incompatible physical type, encoding, or type length explicitly. + Review PLAIN, dictionary/RLE, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, BOOLEAN RLE, and level RLE/bit-packed paths for identical + selection and malformed-input behavior. Page V1 and V2 must feed the same decoder contract after + their different level/decompression layouts are parsed. +- Keep every index coordinate domain explicit: table-local column ID, physical leaf-column ID, + Row Group ID, data-page ordinal, OffsetIndex row ordinal, logical batch-row ordinal, non-null + payload ordinal, and dictionary-entry ID are different types of identity. Data-page ordinals must + exclude dictionary pages consistently. Dictionary-entry bitmaps are local to one Column Chunk + dictionary and cannot be reused after a Row Group, dictionary, or encoding transition. +- Review index composition, not only each index in isolation. Row Group statistics, dictionary, + Bloom, ColumnIndex/OffsetIndex, page cache registration, page skip plans, SelectionVector, and + lazy column cursors must describe the same surviving logical rows. Missing or unusable optional + indexes retain candidates; structurally inconsistent indexes or out-of-range IDs return an + explicit corruption error. A mixed dictionary/plain Column Chunk must leave dictionary-ID + filtering before any cursor is consumed. +- Decoder owns encoded-stream parsing and cursor movement only. `DataTypeSerDe` owns Parquet + physical/logical interpretation and writes directly into Doris columns. Dictionary pages are + materialized once through the same SerDe and data pages expose only validated dictionary indices. + Decimal and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, + precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. +- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, + unsupported physical types, encodings, page layouts, or malformed inputs return an explicit + status. Arrow may be used by metadata planning and as a test oracle; no Arrow array, builder, + RecordReader, or metadata lifetime belongs in data-page value or level materialization. +- Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across + batches. String-like decoders should gather selected `StringRef` values and append once per batch, + rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded + high-water mark but must be reset logically between pages, Row Groups, files, and errors. +- Adaptive batch sizing must measure completed Doris output rows/bytes and must not recreate native + readers, builders, or scratch when only the row cap changes. Compare the v1 and v2 lifecycle: + probe batches must not turn persistent setup into a per-batch cost or amplify highly fragmented + selection work. +- Footer and metadata caching must key on stable file identity and cache the serialized footer plus + parsed native metadata at the same lifecycle as v1. Never reuse metadata when path, file size, + modification/version identity, encryption state, or schema-affecting options differ. Cache misses + and uncacheable identities must remain correct without a fallback to stale entries. +- Page-cache behavior must match v1 for cache key, stable file identity, registered byte range, + compressed/decompressed entry kind, checksum/decompression ownership, subrange coverage, + invalidation, admission, and fallback I/O. Any intentional difference needs benchmark and memory + evidence showing it is no worse for v1 workloads. Cache lookup must never alter page ordinal, + decoder, level, or dictionary cursor state. +- Apply v1's MergeRange decision to the native data-page reader, after metadata/dictionary probes + finish. Predicate and lazy readers for one Row Group must share one ordered-range wrapper, and + native per-column prefetch must be disabled while that wrapper is active. Never allocate one + MergeRange buffer per leaf; wide complex projections would multiply its bounded scratch memory. +- Preserve observability inside aggregate counters. `TotalBatches` must be decomposable into probe, + dense, selected, empty, page-crossing, and nested/fragmented work where relevant; decode, level, + selection, conversion, allocation, and materialization time must remain attributable without + adding per-row timer overhead. +- Profile index attempts, successes, conservative fallbacks, and corrupt rejections separately for + statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skipping. Footer/page/file/ + condition-cache counters must expose requests, hits, misses, writes/admissions, bytes, wait/I/O + time, and bypass reasons with semantics aligned to v1. A lower total timer without its internal + work counters is not sufficient observability. + ## Detailed FileReader Review Guides - Before reviewing any FileReader implementation, index, predicate path, cache, or virtual column, @@ -174,6 +275,15 @@ instructions as well; this file adds format-v2-specific review expectations. malformed input. - For bug fixes, require a test that fails for the original reachable path and validates the result, row count, or explicit error after the fix. +- For native Parquet work, require a matrix across physical/logical types, every supported encoding, + Page V1/V2, required/optional/repeated levels, dictionary fallback, page and batch boundaries, + dense/empty/fragmented selections, null placement, malformed/truncated input, and files from + representative external writers. Include focused cursor-invariant tests and end-to-end nested + reconstruction tests; a scalar happy-path benchmark is not sufficient coverage. +- Performance-sensitive decoder changes need a reproducible comparison against v1 and a relevant + external baseline such as DuckDB. Report data shape, encoding, selectivity, null/cardinality + distribution, compression, storage path, batch policy, warm/cold cache state, CPU time, rows/s, + bytes/s, allocation behavior, and Profile counter deltas. ## Review Output diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index ac510caaf07ac4..d504305fad4014 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -248,6 +248,9 @@ struct ColumnDefinition { // Historical or external names for the same logical field. Table formats such as Iceberg can // use this to resolve partition path keys after column rename. std::vector name_mapping {}; + // Distinguishes no table-level mapping from an explicit empty field mapping. The latter must + // not fall back to the current name when matching fields in legacy files. + bool has_name_mapping = false; DataTypePtr type; // Semantic nested children for this schema node. // @@ -255,6 +258,9 @@ struct ColumnDefinition { // FileReader::get_schema() also expose semantic children, not physical reader wrappers. For // example, MAP children are key/value and ARRAY children contain only the element field. std::vector children {}; + // Full table-schema identity subtree before access-path pruning. ID-less physical complex + // wrappers must be discovered from this view without adding unrequested children to output. + std::vector identity_children {}; // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 61f35023963eca..ed0dcda9739d6b 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -37,6 +37,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/short_circuit_evaluation_expr.h" #include "exprs/vcase_expr.h" #include "exprs/vcast_expr.h" @@ -45,18 +46,83 @@ #include "exprs/vexpr_context.h" #include "exprs/vin_predicate.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "format_v2/column_mapper_nested.h" #include "format_v2/expr/cast.h" #include "format_v2/file_reader.h" #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "util/url_coding.h" namespace doris::format { namespace { +Status build_initial_default_column(const ColumnDefinition& column, ColumnPtr* value) { + DORIS_CHECK(value != nullptr); + *value = nullptr; + if (!column.initial_default_value.has_value()) { + return Status::OK(); + } + const auto nested_type = remove_nullable(column.type); + Field parsed; + if (column.initial_default_value_is_base64 || + nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(*column.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + column.name); + } + parsed = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // Variable-width Fields borrow their input. Materialize while decoded is alive so the + // resulting column owns the payload before it crosses a mapping/literal boundary. + *value = column.type->create_column_const(1, parsed); + return Status::OK(); + } else { + RETURN_IF_ERROR( + nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); + } + *value = column.type->create_column_const(1, parsed); + return Status::OK(); +} + +Status build_initial_default_literal(const ColumnDefinition& column, VExprContextSPtr* literal) { + DORIS_CHECK(literal != nullptr); + ColumnPtr owned_value; + RETURN_IF_ERROR(build_initial_default_column(column, &owned_value)); + DORIS_CHECK(static_cast(owned_value)); + Field value; + owned_value->get(0, value); + // VLiteral copies the borrowed Field into its own column while owned_value is still alive. + *literal = VExprContext::create_shared(VLiteral::create_shared(column.type, value)); + return Status::OK(); +} + +bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { + const auto& table_children = + table.identity_children.empty() ? table.children : table.identity_children; + for (const auto& table_child : table_children) { + if (!table_child.has_identifier_field_id()) { + continue; + } + const auto file_child = + std::ranges::find_if(file.children, [&](const ColumnDefinition& candidate) { + return candidate.has_identifier_field_id() && + candidate.get_identifier_field_id() == + table_child.get_identifier_field_id(); + }); + if (file_child != file.children.end() || + std::ranges::any_of(file.children, [&](const ColumnDefinition& candidate) { + return has_shared_descendant_field_id(table_child, candidate); + })) { + return true; + } + } + return false; +} + std::string mapping_mode_to_string(TableColumnMappingMode mode) { switch (mode) { case TableColumnMappingMode::BY_FIELD_ID: @@ -82,12 +148,16 @@ bool column_has_name(const ColumnDefinition& column, const std::string& name) { } bool column_names_match(const ColumnDefinition& lhs, const ColumnDefinition& rhs) { - if (column_has_name(rhs, lhs.name)) { - return true; - } - if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { - return true; + if (!lhs.has_name_mapping) { + if (column_has_name(rhs, lhs.name)) { + return true; + } + if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { + return true; + } } + // Explicit Iceberg name mapping is authoritative: an empty alias list represents a field that + // did not exist in the imported file, so only transported aliases may match. return std::ranges::any_of(lhs.name_mapping, [&](const std::string& alias) { return column_has_name(rhs, alias); }); @@ -302,30 +372,72 @@ static bool is_cast_expr(const VExprSPtr& expr) { } static bool is_binary_comparison_predicate(const VExprSPtr& expr) { - // BINARY_PRED and NULL_AWARE_BINARY_PRED are comparison-only node kinds. Nereids does not - // always populate the legacy opcode after resolving the comparison through TFunction, so - // requiring expr->op() to be set leaves an otherwise ordinary slot-literal predicate with - // mismatched implicit-coercion types. Row evaluation still works through the resolved - // function, but Parquet zonemap evaluation conservatively rejects the mismatched types. - return expr != nullptr && expr->get_num_children() == 2 && - (expr->node_type() == TExprNodeType::BINARY_PRED || - expr->node_type() == TExprNodeType::NULL_AWARE_BINARY_PRED); + if (expr == nullptr || expr->get_num_children() != 2 || + (expr->node_type() != TExprNodeType::BINARY_PRED && + expr->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED)) { + return false; + } + switch (expr->op()) { + case TExprOpcode::EQ: + case TExprOpcode::EQ_FOR_NULL: + case TExprOpcode::NE: + case TExprOpcode::GE: + case TExprOpcode::GT: + case TExprOpcode::LE: + case TExprOpcode::LT: + return true; + default: + return false; + } } std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; - out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << "}"; + out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) + << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection + << ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns << "}"; return out.str(); } +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr) { + return false; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto primitive_type = table_type->get_primitive_type(); + if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { + return false; + } + const auto target_len = assert_cast(table_type.get())->len(); + if (target_len <= 0) { + return false; + } + if (mapping.file_type == nullptr) { + return true; + } + const auto file_type = remove_nullable(mapping.file_type); + DORIS_CHECK(file_type != nullptr); + int file_len = -1; + if (file_type->get_primitive_type() == TYPE_VARCHAR || + file_type->get_primitive_type() == TYPE_CHAR || + file_type->get_primitive_type() == TYPE_STRING) { + file_len = assert_cast(file_type.get())->len(); + } + return file_len < 0 || target_len < file_len; +} + std::string ColumnDefinition::debug_string() const { std::ostringstream out; out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) << ", name_mapping=" << join_debug_strings(name_mapping, [](const std::string& name) { return name; }) - << ", local_id=" << local_id << ", type=" << data_type_debug_string(type) << ", children=" + << ", has_name_mapping=" << has_name_mapping << ", local_id=" << local_id + << ", type=" << data_type_debug_string(type) << ", children=" << join_debug_strings(children, [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", identity_children=" + << join_debug_strings(identity_children, + [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) << ", is_partition_key=" << is_partition_key << "}"; return out.str(); @@ -438,8 +550,20 @@ static bool table_filter_has_only_local_entries( return true; } -static bool is_lossless_file_to_table_numeric_cast(const DataTypePtr& file_type, - const DataTypePtr& table_type); +static VExprSPtr unwrap_literal_for_file_cast(const VExprSPtr& expr, + const DataTypePtr& table_type) { + if (expr == nullptr) { + return nullptr; + } + if (expr->is_literal()) { + return expr; + } + if (is_cast_expr(expr) && expr->get_num_children() == 1 && expr->children()[0]->is_literal() && + expr->children()[0]->data_type()->equals(*table_type)) { + return expr->children()[0]; + } + return nullptr; +} static Field literal_field_from_expr(const VExpr& literal_expr) { DORIS_CHECK(literal_expr.is_literal()); @@ -450,48 +574,6 @@ static Field literal_field_from_expr(const VExpr& literal_expr) { return field; } -static VExprSPtr unwrap_literal_for_file_cast(const VExprSPtr& expr, const DataTypePtr& table_type, - RewriteContext* rewrite_context) { - if (expr == nullptr) { - return nullptr; - } - - VExprSPtr literal; - if (expr->is_literal()) { - literal = expr; - } else if (is_cast_expr(expr) && expr->get_num_children() == 1 && - expr->children()[0]->is_literal() && expr->data_type()->equals(*table_type)) { - literal = expr->children()[0]; - } else { - return nullptr; - } - - if (literal->data_type()->equals(*table_type)) { - return literal; - } - // Nereids may leave a narrow numeric literal directly under a comparison and rely on the - // comparison's implicit type coercion, for example `INT slot = TINYINT 1`. Materialize that - // coercion before localizing the predicate so metadata readers still see slot-literal form. - if (!is_lossless_file_to_table_numeric_cast(literal->data_type(), table_type)) { - return nullptr; - } - - Field table_field; - try { - convert_field_to_type(literal_field_from_expr(*literal), *table_type, &table_field, - literal->data_type().get()); - } catch (const Exception&) { - return nullptr; - } - if (table_field.is_null() || - table_field.get_type() != remove_nullable(table_type)->get_primitive_type()) { - return nullptr; - } - auto normalized_literal = VLiteral::create_shared(table_type, std::move(table_field)); - rewrite_context->add_created_expr(normalized_literal); - return normalized_literal; -} - // Table filter localization clones an already-prepared table expr and then rewrites it to file // slots. Only split-local literals and BE cast nodes need table-reader-specific clone behavior; // plain slot refs and literals use their own VExpr::clone_node(). @@ -695,8 +777,8 @@ static bool rewrite_binary_slot_literal_predicate( if (rewrite_info == nullptr || slot_ref == nullptr) { return false; } - auto literal_expr = unwrap_literal_for_file_cast(children[literal_child_idx], - rewrite_info->table_type, rewrite_context); + auto literal_expr = + unwrap_literal_for_file_cast(children[literal_child_idx], rewrite_info->table_type); if (literal_expr == nullptr) { return false; } @@ -733,8 +815,8 @@ static bool rewrite_in_slot_literal_predicate( VExprSPtrs rewritten_literals; rewritten_literals.reserve(children.size() - 1); for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { - auto literal_expr = unwrap_literal_for_file_cast(children[child_idx], - rewrite_info->table_type, rewrite_context); + auto literal_expr = + unwrap_literal_for_file_cast(children[child_idx], rewrite_info->table_type); if (literal_expr == nullptr) { return false; } @@ -742,8 +824,8 @@ static bool rewrite_in_slot_literal_predicate( rewrite_literal_to_file_type(literal_expr, *rewrite_info, rewrite_context); if (rewritten_literal == nullptr) { for (size_t restore_idx = 1; restore_idx < children.size(); ++restore_idx) { - auto restore_literal = unwrap_literal_for_file_cast( - children[restore_idx], rewrite_info->table_type, rewrite_context); + auto restore_literal = unwrap_literal_for_file_cast(children[restore_idx], + rewrite_info->table_type); if (restore_literal != nullptr) { children[restore_idx] = original_table_literal(restore_literal, rewrite_context); @@ -811,6 +893,19 @@ static bool collect_struct_element_chain(const VExprSPtr& expr, std::vector, table + // STRUCT, rows [NULL, 20], and `s.a > 10`, filtering in the file domain would drop + // NULL first and hide the table-contract violation. The reverse direction is safe: a required + // file value can always be wrapped as a nullable table value after filtering. + return !file_type->is_nullable() || table_type->is_nullable(); +} + static bool rewrite_struct_element_path_to_file_expr( const VExprSPtr& expr, const std::vector& mappings, const std::map& global_to_file_slot, @@ -837,6 +932,22 @@ static bool rewrite_struct_element_path_to_file_expr( return false; } + // Check every value-producing level, including the root struct. A nullable parent also makes + // a child access nullable even when the child type itself is required, so checking only the + // final leaf is insufficient. If any file level is more nullable than its table counterpart, + // keep the complete predicate above TableReader so schema validation observes all NULLs before + // row filtering. + if (!can_filter_before_table_nullability_alignment(rewrite_it->second.file_type, + rewrite_it->second.table_type)) { + return false; + } + for (size_t idx = 0; idx < struct_element_chain.size(); ++idx) { + if (!can_filter_before_table_nullability_alignment( + resolved.file_child_types[idx], struct_element_chain[idx]->data_type())) { + return false; + } + } + // File-local conjuncts are prepared against the file-reader Block, so both the root slot and // every struct selector must be expressed in file schema terms. For a renamed Iceberg field, // keeping the table selector would prepare `element_at(file_struct, 'renamed')` and @@ -865,6 +976,209 @@ static bool rewrite_struct_element_path_to_file_expr( return true; } +static VExprSPtr cast_file_expr_to_table_type(const VExprSPtr& file_expr, + const DataTypePtr& table_type, + RewriteContext* rewrite_context) { + DORIS_CHECK(file_expr != nullptr); + DORIS_CHECK(table_type != nullptr); + DORIS_CHECK(rewrite_context != nullptr); + auto cast_expr = Cast::create_shared(table_type); + cast_expr->add_child(file_expr); + rewrite_context->add_created_expr(cast_expr); + return cast_expr; +} + +// Prefer comparing in the physical file leaf type when a table predicate uses a promoted struct +// child. For example, with table STRUCT, old-file STRUCT, and `s.a = 10`, the +// localized predicate should be `file_s.a::INT = 10::INT`, not +// `CAST(file_s.a::INT AS BIGINT) = 10::BIGINT`. Converting one literal avoids a cast for every row. +// +// This rewrite is valid only when every possible file value survives file-to-table conversion and +// the particular literal survives a table-to-file-to-table round trip. A value such as BIGINT +// 2147483648 cannot be represented by an INT file leaf, so that case deliberately falls back to +// `CAST(file_s.a AS BIGINT) = 2147483648`, which preserves the original table-level semantics. +static bool rewrite_binary_struct_literal_predicate( + const VExprSPtr& expr, const std::vector& filter_mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context, bool* can_localize) { + DORIS_CHECK(can_localize != nullptr); + if (!is_binary_comparison_predicate(expr)) { + return false; + } + auto children = expr->children(); + int struct_child_idx = -1; + int literal_child_idx = -1; + if (is_struct_element_expr(children[0])) { + struct_child_idx = 0; + literal_child_idx = 1; + } else if (is_struct_element_expr(children[1])) { + struct_child_idx = 1; + literal_child_idx = 0; + } else { + return false; + } + + const auto table_leaf_type = children[struct_child_idx]->data_type(); + DORIS_CHECK(table_leaf_type != nullptr); + auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type); + if (table_literal == nullptr || + !rewrite_struct_element_path_to_file_expr(children[struct_child_idx], filter_mappings, + global_to_file_slot, rewrite_context)) { + return false; + } + + const auto file_leaf_type = children[struct_child_idx]->data_type(); + DORIS_CHECK(file_leaf_type != nullptr); + const FileSlotRewriteInfo leaf_rewrite_info { + .block_position = 0, + .file_type = file_leaf_type, + .table_type = table_leaf_type, + .file_column_name = {}, + }; + auto file_literal = + rewrite_literal_to_file_type(table_literal, leaf_rewrite_info, rewrite_context); + if (file_literal != nullptr) { + children[literal_child_idx] = std::move(file_literal); + } else { + if (!is_lossless_file_to_table_numeric_cast(file_leaf_type, table_leaf_type)) { + // A narrowing or otherwise lossy cast can fail or produce NULL while TableReader + // materializes the table schema. Evaluating it here could filter the offending row + // before that validation, so keep the complete predicate above TableReader. + *can_localize = false; + return true; + } + children[struct_child_idx] = cast_file_expr_to_table_type(children[struct_child_idx], + table_leaf_type, rewrite_context); + children[literal_child_idx] = original_table_literal(table_literal, rewrite_context); + } + expr->set_children(std::move(children)); + return true; +} + +// IN must use one comparison type for its probe and every candidate. Rewrite the complete literal +// set only when all values are exactly representable in the file leaf type; one unsafe value makes +// the whole predicate fall back to a table-type cast. For example, an INT file leaf can evaluate +// `BIGINT IN (10, 20)` as `INT IN (10, 20)`, but `BIGINT IN (10, 2147483648)` must stay BIGINT. +static bool rewrite_in_struct_literal_predicate( + const VExprSPtr& expr, const std::vector& filter_mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context, bool* can_localize) { + DORIS_CHECK(can_localize != nullptr); + if (expr->node_type() != TExprNodeType::IN_PRED || expr->get_num_children() < 2 || + !is_struct_element_expr(expr->children()[0])) { + return false; + } + auto children = expr->children(); + const auto table_leaf_type = children[0]->data_type(); + DORIS_CHECK(table_leaf_type != nullptr); + VExprSPtrs table_literals; + table_literals.reserve(children.size() - 1); + for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { + auto table_literal = unwrap_literal_for_file_cast(children[child_idx], table_leaf_type); + if (table_literal == nullptr) { + return false; + } + table_literals.push_back(std::move(table_literal)); + } + if (!rewrite_struct_element_path_to_file_expr(children[0], filter_mappings, global_to_file_slot, + rewrite_context)) { + return false; + } + + const auto file_leaf_type = children[0]->data_type(); + DORIS_CHECK(file_leaf_type != nullptr); + const FileSlotRewriteInfo leaf_rewrite_info { + .block_position = 0, + .file_type = file_leaf_type, + .table_type = table_leaf_type, + .file_column_name = {}, + }; + VExprSPtrs file_literals; + file_literals.reserve(table_literals.size()); + for (const auto& table_literal : table_literals) { + auto file_literal = + rewrite_literal_to_file_type(table_literal, leaf_rewrite_info, rewrite_context); + if (file_literal == nullptr) { + if (!is_lossless_file_to_table_numeric_cast(file_leaf_type, table_leaf_type)) { + *can_localize = false; + return true; + } + children[0] = + cast_file_expr_to_table_type(children[0], table_leaf_type, rewrite_context); + for (size_t literal_idx = 0; literal_idx < table_literals.size(); ++literal_idx) { + children[literal_idx + 1] = + original_table_literal(table_literals[literal_idx], rewrite_context); + } + expr->set_children(std::move(children)); + return true; + } + file_literals.push_back(std::move(file_literal)); + } + + for (size_t literal_idx = 0; literal_idx < file_literals.size(); ++literal_idx) { + children[literal_idx + 1] = std::move(file_literals[literal_idx]); + } + expr->set_children(std::move(children)); + return true; +} + +static VExprSPtr rewrite_struct_or_slot_expr_to_file_expr( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + const std::vector& filter_mappings, RewriteContext* rewrite_context, + bool* can_localize) { + if (is_struct_element_expr(expr)) { + const auto table_leaf_type = expr->data_type(); + if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, + rewrite_context)) { + // The scanner still evaluates the original table-level conjunct after TableReader + // finalizes the output block. Skipping an unlocalizable file conjunct is therefore + // safer than preparing a partially rewritten expression against the wrong struct + // layout. In particular, do not generate file-local conjuncts for computed complex + // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct + // slot-rooted struct chains are supported here. + *can_localize = false; + return expr; + } + DORIS_CHECK(table_leaf_type != nullptr); + DORIS_CHECK(expr->data_type() != nullptr); + if (!expr->data_type()->equals(*table_leaf_type)) { + if (!is_lossless_file_to_table_numeric_cast(expr->data_type(), table_leaf_type)) { + *can_localize = false; + return expr; + } + // Path localization changes the leaf to the physical file type. For example, after an + // Iceberg evolution from STRUCT to STRUCT, the localized old-file + // predicate is initially `element_at(file_col, 'a')::INT = 10::BIGINT`. Cast only the + // leaf back to BIGINT so the comparison has matching operands without forcing a cast + // of the entire evolved struct (whose children may also have been added or reordered). + return cast_file_expr_to_table_type(expr, table_leaf_type, rewrite_context); + } + return expr; + } + + DORIS_CHECK(expr->is_slot_ref()); + const auto* slot_ref = assert_cast(expr.get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); + if (rewrite_it == global_to_file_slot.end()) { + return expr; + } + const auto& rewrite_info = rewrite_it->second; + auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); + if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { + return file_slot; + } + if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { + // Generic file-local expressions cannot safely cast an evolved complex file slot back to + // the table type. For example, ARRAY_CONTAINS(MAP_KEYS(m), 'person5') only reads map keys, + // but CAST(file_m AS table_m) first forces an incompatible old value struct into the new + // layout. Keep such predicates at table level, after TableReader materializes evolution. + *can_localize = false; + return expr; + } + return cast_file_expr_to_table_type(file_slot, rewrite_info.table_type, rewrite_context); +} + static VExprSPtr rewrite_table_expr_to_file_expr( const VExprSPtr& expr, const std::map& global_to_file_slot, @@ -875,7 +1189,7 @@ static VExprSPtr rewrite_table_expr_to_file_expr( } DORIS_CHECK(rewrite_context != nullptr); DORIS_CHECK(can_localize != nullptr); - if (auto* runtime_filter = dynamic_cast(expr.get()); + if (auto* runtime_filter = dynamic_cast(expr.get()); runtime_filter != nullptr) { auto impl = runtime_filter->get_impl(); if (impl == nullptr) { @@ -896,52 +1210,18 @@ static VExprSPtr rewrite_table_expr_to_file_expr( if (rewrite_in_slot_literal_predicate(expr, global_to_file_slot, rewrite_context)) { return expr; } - if (is_struct_element_expr(expr)) { - if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, - rewrite_context)) { - // The scanner still evaluates the original table-level conjunct after TableReader - // finalizes the output block. Skipping an unlocalizable file conjunct is therefore - // safer than preparing a partially rewritten expression against the wrong struct - // layout. In particular, do not generate file-local conjuncts for computed complex - // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct - // slot-rooted struct chains are supported here. - *can_localize = false; - } + if (rewrite_binary_struct_literal_predicate(expr, filter_mappings, global_to_file_slot, + rewrite_context, can_localize)) { return expr; } - if (expr->is_slot_ref()) { - const auto* slot_ref = assert_cast(expr.get()); - const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); - if (rewrite_it != global_to_file_slot.end()) { - const auto& rewrite_info = rewrite_it->second; - auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); - if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { - return file_slot; - } - if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { - // Generic file-local expressions cannot safely cast an evolved complex file slot - // back to the table type. Example: - // - // table filter: ARRAY_CONTAINS(MAP_KEYS(m), 'person5') - // old file: m MAP> - // table: m MAP> - // - // Although MAP_KEYS only reads the key column, wrapping the file slot as - // `CAST(file_m AS table_m)` forces the value struct cast first and fails because - // the old and new value structs have different fields. Keep such filters at the - // table level, where TableReader materializes the evolved complex value before - // Scanner evaluates the original conjunct. Direct slot-rooted struct child paths - // are handled by rewrite_struct_element_path_to_file_expr() above. - *can_localize = false; - return expr; - } - auto cast_expr = Cast::create_shared(rewrite_info.table_type); - cast_expr->add_child(std::move(file_slot)); - rewrite_context->add_created_expr(cast_expr); - return cast_expr; - } + if (rewrite_in_struct_literal_predicate(expr, filter_mappings, global_to_file_slot, + rewrite_context, can_localize)) { return expr; } + if (is_struct_element_expr(expr) || expr->is_slot_ref()) { + return rewrite_struct_or_slot_expr_to_file_expr(expr, global_to_file_slot, filter_mappings, + rewrite_context, can_localize); + } // The input is a split-local cloned tree. A previous split-local clone may already have // inserted Cast(slot). Keep that rewrite idempotent: rewrite the cast child from table slot to // the current split's file slot, and drop the cast when the current split no longer needs it. @@ -1751,7 +2031,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab mapping->global_index = global_index; mapping->table_column_name = table_column.name; mapping->table_type = table_column.type; - const auto row_lineage_type = row_lineage_virtual_column_type(table_column, _options.mode); + // Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive, + // Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis. + const auto row_lineage_type = + _options.enable_row_lineage_virtual_columns + ? row_lineage_virtual_column_type(table_column, _options.mode) + : TableVirtualColumnType::INVALID; if (const auto* partition_value = find_partition_value(table_column, _partition_values); table_column.is_partition_key && partition_value != nullptr) { // Partition values are split constants and must take precedence over defaults. @@ -1783,6 +2068,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built // from file path, row position and partition metadata for delete/update/merge. mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; + } else if (table_column.initial_default_value.has_value()) { + VExprContextSPtr initial_default; + RETURN_IF_ERROR(build_initial_default_literal(table_column, &initial_default)); + // Iceberg metadata is the authoritative logical value for files written before the field + // existed; the generic FE expression may still contain its Base64 transport text. + _set_constant_mapping(mapping, std::move(initial_default)); } else if (table_column.default_expr != nullptr) { // Missing schema-evolution column with an explicit default expression. _set_constant_mapping(mapping, table_column.default_expr); @@ -1897,11 +2188,12 @@ Status TableColumnMapper::_build_filter_entries(const FileScanRequest& file_requ Status TableColumnMapper::create_scan_request( const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, FilterLocalizationResult* localization_result) { // FileReader evaluates expressions against a file-local block. This mapper owns the // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); file_request->non_predicate_columns.clear(); + file_request->predicate_only_columns.clear(); file_request->local_positions.clear(); file_request->conjuncts.clear(); file_request->delete_conjuncts.clear(); @@ -1932,7 +2224,32 @@ Status TableColumnMapper::create_scan_request( // 2. Build referenced predicate columns // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); - RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + RETURN_IF_ERROR( + localize_filters(table_filters, file_request, runtime_state, localization_result)); + for (const auto& mapping : _hidden_mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + const auto local_id = LocalColumnId(*mapping.file_local_id); + const bool is_visible_output = + std::ranges::any_of(_mappings, [local_id](const ColumnMapping& visible_mapping) { + return visible_mapping.file_local_id.has_value() && + LocalColumnId(*visible_mapping.file_local_id) == local_id; + }); + if (is_visible_output) { + continue; + } + // A localized predicate is enforced exactly before TableReader materializes output. Only + // truly hidden mappings are absent from the final table block and may discard their + // payload after that file-local evaluation. + if (std::ranges::any_of(file_request->predicate_columns, + [local_id](const LocalColumnIndex& projection) { + return projection.column_id() == local_id; + }) && + !file_request->is_predicate_only(local_id)) { + file_request->predicate_only_columns.push_back(local_id); + } + } // 3. Rebuild output projection expressions for projected columns. localize_filters() has // already applied the final scan projection to mapping.file_type/projected_file_children before // rewriting filter expressions. @@ -1973,7 +2290,11 @@ ColumnMapping* TableColumnMapper::_find_filter_mapping(GlobalIndex global_index) Status TableColumnMapper::localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, + FilterLocalizationResult* localization_result) { + if (localization_result != nullptr) { + localization_result->localized_filters.assign(table_filters.size(), false); + } std::set localized_predicate_columns; FilterProjectionMap filter_projections; auto filter_mappings = _filter_visible_mappings(); @@ -2011,9 +2332,28 @@ Status TableColumnMapper::localize_filters(const std::vector& table // This keeps expression localization independent from filter iteration order. filter_mappings = _filter_visible_mappings(); const auto global_to_file_slot = build_file_slot_rewrite_map(filter_mappings, _filter_entries); - for (const auto& table_filter : table_filters) { - if (table_filter.conjunct != nullptr && - table_filter_has_only_local_entries(table_filter, _filter_entries)) { + for (size_t filter_index = 0; filter_index < table_filters.size(); ++filter_index) { + const auto& table_filter = table_filters[filter_index]; + if (table_filter.conjunct != nullptr && table_filter.conjunct->root() != nullptr) { + const auto root = table_filter.conjunct->root(); + const auto impl = root->get_impl(); + const auto predicate = impl != nullptr ? impl : root; + if (!table_filter.can_localize || !predicate->is_deterministic() || + !table_filter_has_only_local_entries(table_filter, _filter_entries)) { + continue; + } + if (runtime_state != nullptr && + runtime_state->query_options().truncate_char_or_varchar_columns && + std::ranges::any_of(table_filter.global_indices, [&](GlobalIndex global_index) { + const auto* mapping = _find_filter_mapping(global_index); + return mapping != nullptr && requires_char_or_varchar_truncation(*mapping); + })) { + // The table predicate observes the bounded value after finalize; evaluating it on + // a wider file string would change equality and range semantics. + continue; + } + // FileReader becomes the exact owner only for a stable predicate whose complete + // expression can be rewritten against this split's physical schema. RewriteContext rewrite_context {.runtime_state = runtime_state}; VExprSPtr rewrite_root; Status clone_status; @@ -2024,8 +2364,7 @@ Status TableColumnMapper::localize_filters(const std::vector& table // `element_at(MAP_VALUES(m)[1], 'age') > 30`. The current file-local rewrite only // understands top-level slots and struct-element paths rooted at top-level slots; // cloning such expressions can hit the generic TExpr complex-type limitation. - // Leave them above TableReader, where Scanner evaluates the original table-level - // conjunct after final materialization. + // Leave them for TableReader after final table-schema materialization. #ifndef NDEBUG return Status::InternalError( "Failed to clone table filter for file-local rewrite: {}, expr={}", @@ -2061,6 +2400,9 @@ Status TableColumnMapper::localize_filters(const std::vector& table auto localized_conjunct = VExprContext::create_shared(std::move(localized_root)); RETURN_IF_ERROR(rewrite_context.prepare_created_exprs(localized_conjunct.get())); file_request->conjuncts.push_back(std::move(localized_conjunct)); + if (localization_result != nullptr) { + localization_result->localized_filters[filter_index] = true; + } for (const auto global_index : table_filter.global_indices) { const auto* mapping = _find_filter_mapping(global_index); if (mapping != nullptr && mapping->file_local_id.has_value() && @@ -2105,7 +2447,25 @@ const ColumnDefinition* TableColumnMapper::_find_file_field( }); return field_it == file_schema.end() ? nullptr : &*field_it; } - return matcher_for_mode(_options.mode).find(table_column, file_schema); + const auto* matched = matcher_for_mode(_options.mode).find(table_column, file_schema); + if (matched != nullptr || _options.mode != TableColumnMappingMode::BY_FIELD_ID || + !_options.allow_idless_complex_wrapper_projection || table_column.children.empty()) { + return matched; + } + const ColumnDefinition* wrapper = nullptr; + for (const auto& candidate : file_schema) { + if (candidate.has_identifier_field_id() || candidate.children.empty() || + !has_shared_descendant_field_id(table_column, candidate)) { + continue; + } + if (wrapper != nullptr) { + return nullptr; + } + wrapper = &candidate; + } + // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when a nested field ID is + // selected. Descendant IDs, not aliases, identify that wrapper; ambiguity remains unmapped. + return wrapper; } Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_column, @@ -2143,6 +2503,13 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c const auto* file_child = find_file_child_for_mapping(table_child, file_field, _options.mode, table_child_idx, synthesized_table_children); + if (file_child == nullptr && !synthesized_table_children && + _options.mode == TableColumnMappingMode::BY_FIELD_ID && + _options.allow_idless_complex_wrapper_projection) { + // Parquet can retain an ID-less wrapper at any depth when a selected descendant + // has an ID; apply the same opt-in fallback used for root lookup recursively. + file_child = _find_file_field(table_child, file_field.children); + } if (synthesized_table_children && file_child != nullptr) { const auto file_child_id = file_child->file_local_id(); if (std::ranges::find(synthesized_used_file_child_ids, file_child_id) != @@ -2168,6 +2535,10 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + // A missing nested field still has its Iceberg initial-default value in every row + // written before the field was added; carry it into recursive materialization. + RETURN_IF_ERROR(build_initial_default_column( + table_child, &child_mapping.initial_default_column)); mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 4f417a680aad60..0860b687e4abc6 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -40,6 +40,13 @@ namespace doris::format { struct ColumnDefinition; struct TableFilter; +// Reports which table filters were fully rewritten into exact file-local predicates for the +// current split. The result is aligned with the TableFilter input vector and must not be reused for +// another split because schema evolution can change localization independently for every file. +struct FilterLocalizationResult { + std::vector localized_filters; +}; + enum class TableColumnMappingMode { // Match by ColumnDefinition::identifier TYPE_INT as field id. BY_FIELD_ID, @@ -150,16 +157,23 @@ struct ColumnMapping { FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; + // One-row constant owns variable-width payloads; Field is only a borrowed + // StringView and cannot safely outlive the Base64 decode buffer used to construct it. + ColumnPtr initial_default_column; std::string debug_string() const; }; struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + bool allow_idless_complex_wrapper_projection = false; + bool enable_row_lineage_virtual_columns = false; std::string debug_string() const; }; +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping); + Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values); @@ -191,7 +205,8 @@ class TableColumnMapper { virtual Status create_scan_request(const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); // Localize table-level filters to the file schema. // Trivial mappings can copy structured predicates directly. Type changes may be localized with @@ -199,7 +214,8 @@ class TableColumnMapper { // table-level finalize/filter fallback. virtual Status localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); void clear() { _mappings.clear(); _hidden_mappings.clear(); diff --git a/be/src/format_v2/deletion_vector_reader.cpp b/be/src/format_v2/deletion_vector_reader.cpp deleted file mode 100644 index d70956de8b60fc..00000000000000 --- a/be/src/format_v2/deletion_vector_reader.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/deletion_vector_reader.h" - -#include - -#include -#include - -#include "exec/common/endian.h" - -namespace doris::format { - -std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, - const TIcebergDeleteFileDesc& delete_file) { - return fmt::format("delete_dv_{}:{}{}:{}#{}#{}", data_file_path.size(), data_file_path, - delete_file.path.size(), delete_file.path, delete_file.content_offset, - delete_file.content_size_in_bytes); -} - -Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* rows_to_delete) { - if (buf == nullptr || rows_to_delete == nullptr) { - return Status::InvalidArgument("invalid deletion vector decode arguments"); - } - if (buffer_size < 12) { - return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); - } - - const auto total_length = BigEndian::Load32(buf); - if (total_length + 8 != buffer_size) { - return Status::DataQualityError("Deletion vector length mismatch, expected: {}, actual: {}", - total_length + 8, buffer_size); - } - - constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; - if (std::memcmp(buf + sizeof(total_length), MAGIC_NUMBER, 4) != 0) { - return Status::DataQualityError("Deletion vector magic number mismatch"); - } - - try { - *rows_to_delete |= roaring::Roaring64Map::readSafe(buf + 8, buffer_size - 12); - } catch (const std::runtime_error& e) { - return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); - } - return Status::OK(); -} - -std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file) { - return fmt::format("paimon_dv_{}#{}#{}", deletion_file.path, deletion_file.offset, - deletion_file.length); -} - -Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* deletion_vector) { - if (deletion_vector == nullptr) { - return Status::InvalidArgument("deletion_vector must not be null"); - } - if (buffer_size < 8) [[unlikely]] { - return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); - } - const uint32_t actual_length = BigEndian::Load32(buf); - if (actual_length + 4 != buffer_size) [[unlikely]] { - return Status::RuntimeError( - "DeletionVector deserialize error: length not match, actual length: {}, expect " - "length: {}", - actual_length, buffer_size - 4); - } - constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; - if (std::memcmp(buf + sizeof(actual_length), PAIMON_BITMAP_MAGIC, 4) != 0) [[unlikely]] { - return Status::RuntimeError("DeletionVector deserialize error: invalid magic number {}", - BigEndian::Load32(buf + sizeof(actual_length))); - } - roaring::Roaring roaring_bitmap; - try { - roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8); - } catch (const std::runtime_error& e) { - return Status::RuntimeError( - "DeletionVector deserialize error: failed to deserialize roaring bitmap, {}", - e.what()); - } - *deletion_vector |= DeletionVector(std::move(roaring_bitmap)); - return Status::OK(); -} - -DeletionVectorReader::~DeletionVectorReader() { - _file_reader.reset(); - _merge_io_statistics(); -} - -void DeletionVectorReader::_init_io_context() { - if (_parent_io_ctx == nullptr) { - return; - } - _reader_io_ctx = *_parent_io_ctx; - _reader_io_ctx.file_cache_stats = &_file_cache_stats; - _reader_io_ctx.file_reader_stats = &_file_reader_stats; - _io_ctx = &_reader_io_ctx; -} - -void DeletionVectorReader::_merge_io_statistics() { - if (_statistics_merged || _parent_io_ctx == nullptr) { - return; - } - if (_parent_io_ctx->file_cache_stats != nullptr) { - _parent_io_ctx->file_cache_stats->merge_from(_file_cache_stats); - } - if (_parent_io_ctx->file_reader_stats != nullptr) { - _parent_io_ctx->file_reader_stats->read_calls += _file_reader_stats.read_calls; - _parent_io_ctx->file_reader_stats->read_bytes += _file_reader_stats.read_bytes; - _parent_io_ctx->file_reader_stats->read_time_ns += _file_reader_stats.read_time_ns; - _parent_io_ctx->file_reader_stats->read_rows += _file_reader_stats.read_rows; - } - _statistics_merged = true; -} - -Status DeletionVectorReader::open() { - if (_is_opened) [[unlikely]] { - return Status::OK(); - } - - _init_system_properties(); - _init_file_description(); - RETURN_IF_ERROR(_create_file_reader()); - - _file_size = _file_reader->size(); - _is_opened = true; - return Status::OK(); -} - -Status DeletionVectorReader::read_at(size_t offset, Slice result) { - if (UNLIKELY(_parent_io_ctx != nullptr && _parent_io_ctx->should_stop)) { - return Status::EndOfFile("stop read."); - } - if (_io_ctx != nullptr) { - _io_ctx->should_stop = _parent_io_ctx->should_stop; - } - size_t bytes_read = 0; - RETURN_IF_ERROR(_file_reader->read_at(offset, result, &bytes_read, _io_ctx)); - if (bytes_read != result.size) [[unlikely]] { - return Status::IOError("Failed to read fully at offset {}, expected {}, got {}", offset, - result.size, bytes_read); - } - return Status::OK(); -} - -Status DeletionVectorReader::_create_file_reader() { - if (UNLIKELY(_parent_io_ctx != nullptr && _parent_io_ctx->should_stop)) { - return Status::EndOfFile("stop read."); - } - - _file_description.mtime = _desc.modification_time; - io::FileReaderOptions reader_options = - FileFactory::get_reader_options(_state->query_options(), _file_description); - _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( - _profile, _system_properties, _file_description, reader_options, - io::DelegateReader::AccessMode::RANDOM, _io_ctx)); - return Status::OK(); -} - -void DeletionVectorReader::_init_file_description() { - _file_description.path = _desc.path; - _file_description.file_size = _desc.file_size; - _file_description.fs_name = _desc.fs_name; -} - -void DeletionVectorReader::_init_system_properties() { - _system_properties.system_type = _params.file_type; - _system_properties.properties = _params.properties; - _system_properties.hdfs_params = _params.hdfs_params; - if (_params.__isset.broker_addresses) { - _system_properties.broker_addresses.assign(_params.broker_addresses.begin(), - _params.broker_addresses.end()); - } -} - -} // namespace doris::format diff --git a/be/src/format_v2/deletion_vector_reader.h b/be/src/format_v2/deletion_vector_reader.h deleted file mode 100644 index 7ecdb6f1ab219f..00000000000000 --- a/be/src/format_v2/deletion_vector_reader.h +++ /dev/null @@ -1,111 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "common/status.h" -#include "format/generic_reader.h" -#include "format_v2/deletion_vector.h" -#include "gen_cpp/PlanNodes_types.h" -#include "io/file_factory.h" -#include "io/fs/buffered_reader.h" -#include "io/fs/file_reader.h" -#include "io/io_common.h" -#include "util/profile_collector.h" -#include "util/slice.h" - -namespace doris::format { - -struct DeleteFileDesc { - enum class Format { - PAIMON, - ICEBERG, - }; - - std::string key = ""; - std::string path = ""; - std::string fs_name = ""; - int64_t start_offset = 0; - int64_t size = 0; - int64_t file_size = -1; - int64_t modification_time = 0; - Format format = Format::PAIMON; -}; - -std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, - const TIcebergDeleteFileDesc& delete_file); - -Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* rows_to_delete); - -std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file); - -Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, - DeletionVector* deletion_vector); - -class DeletionVectorReader { -public: - DeletionVectorReader(RuntimeState* state, RuntimeProfile* profile, - const TFileScanRangeParams& params, const DeleteFileDesc& desc, - io::IOContext* io_ctx) - : _state(state), - _profile(profile), - _params(params), - _desc(desc), - _parent_io_ctx(io_ctx) { - _init_io_context(); - } - ~DeletionVectorReader(); - - DeletionVectorReader(const DeletionVectorReader&) = delete; - DeletionVectorReader& operator=(const DeletionVectorReader&) = delete; - - Status open(); - Status read_at(size_t offset, Slice result); - - const io::FileCacheStatistics& file_cache_statistics() const { return _file_cache_stats; } - -private: - void _init_system_properties(); - void _init_file_description(); - void _init_io_context(); - void _merge_io_statistics(); - Status _create_file_reader(); - - RuntimeState* _state = nullptr; - RuntimeProfile* _profile = nullptr; - const TFileScanRangeParams& _params; - DeleteFileDesc _desc; - io::IOContext* _parent_io_ctx = nullptr; - io::IOContext _reader_io_ctx; - io::IOContext* _io_ctx = nullptr; - io::FileCacheStatistics _file_cache_stats; - io::FileReaderStats _file_reader_stats; - bool _statistics_merged = false; - - io::FileSystemProperties _system_properties; - io::FileDescription _file_description; - io::FileReaderSPtr _file_reader; - int64_t _file_size = 0; - bool _is_opened = false; -}; - -} // namespace doris::format diff --git a/be/src/format_v2/delimited_text/csv_reader.cpp b/be/src/format_v2/delimited_text/csv_reader.cpp index 5935e9323f71b6..bb6c55d30847b2 100644 --- a/be/src/format_v2/delimited_text/csv_reader.cpp +++ b/be/src/format_v2/delimited_text/csv_reader.cpp @@ -17,7 +17,6 @@ #include "format_v2/delimited_text/csv_reader.h" -#include #include #include @@ -34,226 +33,6 @@ #include "util/utf8_check.h" namespace doris::format::csv { - -enum class CsvReaderState { START, NORMAL, PRE_MATCH_ENCLOSE, MATCH_ENCLOSE }; - -struct CsvReaderStateWrapper { - void forward_to(CsvReaderState state) { - prev_state = curr_state; - curr_state = state; - } - - void reset() { - curr_state = CsvReaderState::START; - prev_state = CsvReaderState::START; - } - - CsvReaderState curr_state = CsvReaderState::START; - CsvReaderState prev_state = CsvReaderState::START; -}; - -// The v1 CSV context cannot distinguish a column separator that overlaps the line delimiter and -// may rewind into a separator that was already emitted when the output buffer is refilled. Keep -// the stricter behavior local to format v2 so the v1 reader remains unchanged. -class EncloseCsvLineReaderV2Ctx final - : public BaseTextLineReaderContext { - using FindDelimiterFunc = const uint8_t* (*)(const uint8_t*, size_t, const char*, size_t); - -public: - EncloseCsvLineReaderV2Ctx(const std::string& line_delimiter_, size_t line_delimiter_len_, - std::string column_separator, size_t column_separator_len, - size_t column_separator_count, char enclose, char escape, - bool keep_cr_, bool skip_utf8_bom) - : BaseTextLineReaderContext(line_delimiter_, line_delimiter_len_, keep_cr_), - _enclose(enclose), - _escape(escape), - _skip_utf8_bom(skip_utf8_bom), - _column_separator_len(column_separator_len), - _column_separator(std::move(column_separator)) { - if (_column_separator_len == 1) { - _find_column_separator = &look_for_column_separator; - } else { - _find_column_separator = &look_for_column_separator; - } - _column_separator_positions.reserve(column_separator_count); - } - - void refresh_impl() { - _idx = 0; - _should_escape = false; - _quote_escape = false; - _result = nullptr; - _column_separator_positions.clear(); - _state.reset(); - } - - [[nodiscard]] const std::vector& column_sep_positions() const { - return _column_separator_positions; - } - - void adjust_column_sep_positions(size_t offset) { - for (auto& position : _column_separator_positions) { - position -= offset; - } - } - - const uint8_t* read_line_impl(const uint8_t* start, size_t length) { - if (_skip_utf8_bom && !_first_record_prefix_checked && _idx == 0) { - constexpr uint8_t UTF8_BOM[] = {0xEF, 0xBB, 0xBF}; - constexpr size_t UTF8_BOM_SIZE = sizeof(UTF8_BOM); - const size_t prefix_size = std::min(length, UTF8_BOM_SIZE); - if (std::memcmp(start, UTF8_BOM, prefix_size) != 0) { - _first_record_prefix_checked = true; - } else if (length < UTF8_BOM_SIZE) { - return nullptr; - } else { - _idx = UTF8_BOM_SIZE; - _first_record_prefix_checked = true; - } - } - - if (_state.curr_state == CsvReaderState::NORMAL || - _state.curr_state == CsvReaderState::MATCH_ENCLOSE) { - const size_t last_separator_end = - _column_separator_positions.empty() - ? 0 - : _column_separator_positions.back() + _column_separator_len; - DORIS_CHECK_LE(last_separator_end, _idx); - _idx -= std::min(_column_separator_len - 1, _idx - last_separator_end); - } - - _total_len = length; - size_t bound = update_reading_bound(start); - while (_idx != bound) { - switch (_state.curr_state) { - case CsvReaderState::START: - on_start(start); - break; - case CsvReaderState::NORMAL: - on_normal(start, bound); - break; - case CsvReaderState::PRE_MATCH_ENCLOSE: - on_pre_match_enclose(start, bound); - break; - case CsvReaderState::MATCH_ENCLOSE: - on_match_enclose(start, bound); - break; - } - } - return _result; - } - -private: - template - static const uint8_t* look_for_column_separator(const uint8_t* start, size_t length, - const char* separator, size_t separator_len) { - if constexpr (SingleChar) { - for (size_t i = 0; i < length; ++i) { - if (start[i] == separator[0]) { - return start + i; - } - } - return nullptr; - } - return static_cast(memmem(start, length, separator, separator_len)); - } - - size_t update_reading_bound(const uint8_t* start) { - _result = call_find_line_sep(start + _idx, _total_len - _idx); - return _result == nullptr ? _total_len : _result - start + line_delimiter_length(); - } - - void on_column_separator_found(const uint8_t* start, const uint8_t* separator_position) { - const uint8_t* field_start = start + _idx; - _column_separator_positions.push_back(separator_position - start); - _idx += separator_position + _column_separator_len - field_start; - } - - void on_start(const uint8_t* start) { - if (start[_idx] == _enclose) [[unlikely]] { - _state.forward_to(CsvReaderState::PRE_MATCH_ENCLOSE); - ++_idx; - } else { - _state.forward_to(CsvReaderState::NORMAL); - } - } - - void on_normal(const uint8_t* start, size_t bound) { - const size_t search_bound = _result == nullptr ? bound : _result - start; - DORIS_CHECK_LE(_idx, search_bound); - const uint8_t* separator_position = - _find_column_separator(start + _idx, search_bound - _idx, _column_separator.c_str(), - _column_separator_len); - if (separator_position != nullptr) [[likely]] { - on_column_separator_found(start, separator_position); - _state.forward_to(CsvReaderState::START); - return; - } - _idx = bound; - } - - void on_pre_match_enclose(const uint8_t* start, size_t& bound) { - do { - do { - if (_escape != _enclose && start[_idx] == _escape) [[unlikely]] { - _should_escape = !_should_escape; - } else if (_should_escape) [[unlikely]] { - _should_escape = false; - } else if (_quote_escape) { - if (start[_idx] == _enclose) { - _quote_escape = false; - } else { - _quote_escape = false; - _state.forward_to(CsvReaderState::MATCH_ENCLOSE); - return; - } - } else if (start[_idx] == _enclose) { - _quote_escape = true; - } else { - _quote_escape = false; - } - ++_idx; - } while (_idx != bound); - - if (_idx != _total_len) { - bound = update_reading_bound(start); - } else { - _result = nullptr; - break; - } - } while (true); - } - - void on_match_enclose(const uint8_t* start, size_t bound) { - const size_t search_bound = _result == nullptr ? bound : _result - start; - DORIS_CHECK_LE(_idx, search_bound); - const uint8_t* separator_position = - _find_column_separator(start + _idx, search_bound - _idx, _column_separator.c_str(), - _column_separator_len); - if (separator_position != nullptr) [[likely]] { - on_column_separator_found(start, separator_position); - _state.forward_to(CsvReaderState::START); - return; - } - _idx = bound; - } - - CsvReaderStateWrapper _state; - const char _enclose; - const char _escape; - const bool _skip_utf8_bom; - bool _first_record_prefix_checked = false; - const uint8_t* _result = nullptr; - size_t _total_len = 0; - const size_t _column_separator_len; - size_t _idx = 0; - bool _should_escape = false; - bool _quote_escape = false; - const std::string _column_separator; - std::vector _column_separator_positions; - FindDelimiterFunc _find_column_separator; -}; - namespace { bool starts_with_at(const Slice& line, size_t pos, const std::string& needle) { @@ -357,7 +136,7 @@ Status CsvReader::_create_line_reader() { } else { const size_t col_sep_num = _source_file_slot_descs.size() > 1 ? _source_file_slot_descs.size() - 1 : 0; - _enclose_reader_ctx = std::make_shared( + _enclose_reader_ctx = std::make_shared( _line_delimiter, _line_delimiter.size(), _value_separator, _value_separator.size(), col_sep_num, _enclose, _escape, _keep_cr, _start_offset == 0); diff --git a/be/src/format_v2/delimited_text/csv_reader.h b/be/src/format_v2/delimited_text/csv_reader.h index 088da8ba77cd50..dd1d77d5a34b9a 100644 --- a/be/src/format_v2/delimited_text/csv_reader.h +++ b/be/src/format_v2/delimited_text/csv_reader.h @@ -25,13 +25,12 @@ #include "util/slice.h" namespace doris { +class EncloseCsvLineReaderCtx; class SlotDescriptor; } // namespace doris namespace doris::format::csv { -class EncloseCsvLineReaderV2Ctx; - // FileScannerV2 CSV reader. // // CSV files do not carry a physical schema. FE provides the table slot descriptors plus @@ -71,7 +70,7 @@ class CsvReader final : public ::doris::format::DelimitedTextReader { bool _trim_tailing_spaces = false; bool _empty_field_as_null = false; bool _keep_cr = false; - std::shared_ptr _enclose_reader_ctx; + std::shared_ptr _enclose_reader_ctx; }; } // namespace doris::format::csv diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp index f19d12c75714b9..63486d174effe0 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -38,6 +38,7 @@ #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/string_util.h" @@ -180,7 +181,9 @@ void DelimitedTextReader::_init_profile() { return; } - ADD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, 1); + file_scan_profile::ensure_hierarchy(_profile); + _text_profile.total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, + file_scan_profile::FILE_READER, 1); _text_profile.open_file_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OpenFileTime", DELIMITED_TEXT_PROFILE, 1); _text_profile.create_line_reader_time = @@ -199,8 +202,10 @@ void DelimitedTextReader::_init_profile() { _profile, "RawLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + // RuntimeProfile counter names are global within one profile, so the format prefix preserves + // independent ownership when Parquet and delimited readers are initialized in either order. _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _profile, "DelimitedRowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -215,6 +220,7 @@ void DelimitedTextReader::_init_profile() { Status DelimitedTextReader::init(RuntimeState* state) { _init_profile(); + SCOPED_TIMER(_text_profile.total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("{} v2 reader requires scan params", _reader_name); @@ -275,6 +281,7 @@ Status DelimitedTextReader::init(RuntimeState* state) { } Status DelimitedTextReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_text_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("{} v2 file_schema is null", _reader_name); } @@ -288,6 +295,7 @@ std::unique_ptr DelimitedTextReader::create_column_mapper( } Status DelimitedTextReader::open(std::shared_ptr request) { + SCOPED_TIMER(_text_profile.total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -307,6 +315,7 @@ Status DelimitedTextReader::open(std::shared_ptr request) { } Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -358,11 +367,18 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& request, FileAggregateResult* result) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(result != nullptr); if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("{} v2 reader only supports COUNT aggregate pushdown", _reader_name); } + if (!request.columns.empty()) { + // Text files expose no NULL-count metadata, and this fast path intentionally skips field + // parsing. Returning the physical row count for COUNT(nullable_col) would be incorrect. + return Status::NotSupported("{} v2 reader cannot push down COUNT with a column argument", + _reader_name); + } if (_line_reader == nullptr) { return Status::InternalError("{} v2 reader is not open", _reader_name); } @@ -396,6 +412,7 @@ Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& req } Status DelimitedTextReader::close() { + SCOPED_TIMER(_text_profile.total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h index daea3bc90942d8..dff27980c9cd12 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.h +++ b/be/src/format_v2/delimited_text/delimited_text_reader.h @@ -59,6 +59,7 @@ class DelimitedTextReader : public FileReader { protected: struct DelimitedTextProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* open_file_time = nullptr; RuntimeProfile::Counter* create_line_reader_time = nullptr; RuntimeProfile::Counter* read_line_time = nullptr; diff --git a/be/src/format_v2/delimited_text/text_reader.cpp b/be/src/format_v2/delimited_text/text_reader.cpp index 02bd9cd92f8383..6dea7a07a4c6fa 100644 --- a/be/src/format_v2/delimited_text/text_reader.cpp +++ b/be/src/format_v2/delimited_text/text_reader.cpp @@ -25,7 +25,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type_serde/data_type_string_serde.h" #include "format/file_reader/new_plain_text_line_reader.h" -#include "format_v2/delimited_text/hive_text_util.h" +#include "format/hive_text_util.h" #include "runtime/descriptors.h" #include "util/decompressor.h" diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp index 5dd8de557c062b..efeb9d851deb22 100644 --- a/be/src/format_v2/expr/cast.cpp +++ b/be/src/format_v2/expr/cast.cpp @@ -111,8 +111,7 @@ Status Cast::_do_execute(VExprContext* context, const Block* block, const Select ColumnNumbers args(1); ColumnPtr tmp_arg_column; - RETURN_IF_ERROR(_children[0]->execute_column(context, block, const_cast(selector), - count, tmp_arg_column)); + RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, tmp_arg_column)); auto arg_type = _children[0]->execute_type(block); temp_block.insert({tmp_arg_column, arg_type, _children[0]->expr_name()}); args[0] = 0; @@ -125,6 +124,7 @@ Status Cast::_do_execute(VExprContext* context, const Block* block, const Select num_columns_without_result, count)); result_column = temp_block.get_by_position(num_columns_without_result).column; DCHECK_EQ(result_column->size(), count); + RETURN_IF_ERROR(result_column->column_self_check()); return Status::OK(); } diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 42e50e7ba07364..1d111aa7a74930 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -26,28 +26,63 @@ #include "core/assert_cast.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { bool column_value_equal(const ColumnPtr& lhs, size_t lhs_row, const ColumnPtr& rhs, size_t rhs_row) { - if (lhs->is_nullable() && rhs->is_nullable()) { - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; - } + const IColumn* lhs_data = lhs.get(); + const IColumn* rhs_data = rhs.get(); if (lhs->is_nullable()) { const auto& nullable_lhs = assert_cast(*lhs); - return !nullable_lhs.is_null_at(lhs_row) && - nullable_lhs.get_nested_column().compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + if (nullable_lhs.is_null_at(lhs_row)) { + return rhs->is_nullable() && + assert_cast(*rhs).is_null_at(rhs_row); + } + lhs_data = &nullable_lhs.get_nested_column(); } if (rhs->is_nullable()) { const auto& nullable_rhs = assert_cast(*rhs); - return !nullable_rhs.is_null_at(rhs_row) && - lhs->compare_at(lhs_row, rhs_row, nullable_rhs.get_nested_column(), -1) == 0; + if (nullable_rhs.is_null_at(rhs_row)) { + return false; + } + rhs_data = &nullable_rhs.get_nested_column(); + } + const bool lhs_binary = check_and_get_column(*lhs_data) != nullptr || + check_and_get_column(*lhs_data) != nullptr; + const bool rhs_binary = check_and_get_column(*rhs_data) != nullptr || + check_and_get_column(*rhs_data) != nullptr; + if (lhs_binary && rhs_binary) { + // Iceberg schema evolution may represent the same byte key as STRING in one file and + // VARBINARY in another. Equality-delete semantics compare bytes, not column storage classes. + return lhs_data->get_data_at(lhs_row) == rhs_data->get_data_at(rhs_row); + } + return lhs_data->compare_at(lhs_row, rhs_row, *rhs_data, -1) == 0; +} + +void update_varbinary_hashes(const ColumnWithTypeAndName& entry, uint64_t* hashes) { + const IColumn* data = entry.column.get(); + const uint8_t* null_map = nullptr; + if (entry.column->is_nullable()) { + const auto& nullable = assert_cast(*entry.column); + data = &nullable.get_nested_column(); + null_map = nullable.get_null_map_data().data(); + } + for (size_t row = 0; row < entry.column->size(); ++row) { + if (null_map != nullptr && null_map[row] != 0) { + hashes[row] = HashUtil::xxHash64NullWithSeed(hashes[row]); + continue; + } + const auto bytes = data->get_data_at(row); + hashes[row] = HashUtil::xxHash64WithSeed(bytes.data, bytes.size, hashes[row]); } - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; } } // namespace @@ -123,7 +158,7 @@ Status EqualityDeletePredicate::execute_column_impl(VExprContext* context, const Block data_key_block; for (const auto& child : _children) { ColumnPtr key_column; - RETURN_IF_ERROR(child->execute_column_impl(context, block, selector, count, key_column)); + RETURN_IF_ERROR(child->execute_column(context, block, selector, count, key_column)); // Equality comparison operates on row-addressable columns. Materialize literal constants // so nullable NULL keys and regular slot columns share the same compare_at contract. data_key_block.insert({key_column->convert_to_full_column_if_const(), @@ -155,8 +190,14 @@ ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_blo std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { std::vector hashes(block.rows(), 0); - for (const auto& column : block.get_columns()) { - column->update_hashes_with_value(hashes.data(), nullptr); + for (const auto& entry : block) { + if (remove_nullable(entry.type)->get_primitive_type() == TYPE_VARBINARY) { + // ColumnVarbinary intentionally lacks the generic column hash hook. Keep the V2 delete + // hash byte-identical to ColumnString so schema-mapped binary keys share hash buckets. + update_varbinary_hashes(entry, hashes.data()); + } else { + entry.column->update_hashes_with_value(hashes.data(), nullptr); + } } return hashes; } diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index 4f5b247c791efd..9bbf7c3a8fc066 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -53,6 +53,10 @@ std::string FileScanRequest::debug_string() const { << join_debug_strings( non_predicate_columns, [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", predicate_only_columns=" + << join_debug_strings( + predicate_only_columns, + [](LocalColumnId column_id) { return std::to_string(column_id.value()); }) << ", local_positions={"; size_t position_idx = 0; for (const auto& [column_id, block_position] : local_positions) { @@ -62,7 +66,14 @@ std::string FileScanRequest::debug_string() const { out << column_id << ":" << block_position; } out << "}, conjunct_count=" << conjuncts.size() - << ", delete_conjunct_count=" << delete_conjuncts.size() << "}"; + << ", delete_conjunct_count=" << delete_conjuncts.size() + << ", count_star_placeholder_columns={"; + const char* delimiter = ""; + for (const auto column_id : count_star_placeholder_columns) { + out << delimiter << column_id.value(); + delimiter = ","; + } + out << "}}"; return out.str(); } diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 59f684121ce1e3..1f90957064b254 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -70,13 +70,35 @@ struct FileScanRequest { // Columns read after row-level filtering. Predicate columns are also available for output and // should not be duplicated here. std::vector non_predicate_columns; + // Predicate columns introduced only to evaluate hidden filter slots. Their values are dead + // after all file-local predicates run, although the shared file block still needs row-shaped + // placeholders until TableReader finalizes projected columns. + std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; - // Row-level filters converted to file-local expressions from table-level predicates. + // Row-level filters converted to file-local expressions from table-level predicates. Readers + // must enforce these exactly on returned rows; metadata pruning alone does not transfer + // predicate ownership away from TableReader. VExprContextSPtrs conjuncts; // Delete predicates converted to file-local expressions. A TRUE result means that the row is // deleted, so readers must invert each result when building their keep filter. VExprContextSPtrs delete_conjuncts; + // File-local ids retained only because Nereids keeps a minimum-width output tuple for an + // explicit COUNT(*). These columns have no semantic value: for example, after pruning a scan + // may retain an unsupported TIME_MILLIS leaf even though COUNT(*) only needs one row per + // surviving input row. A reader may synthesize defaults instead of reading a marked column + // while it remains non-predicate. If filters or equality deletes promote the same id to + // predicate_columns, the value is semantically required and must still be validated and read. + std::vector count_star_placeholder_columns; + + bool is_count_star_placeholder(LocalColumnId column_id) const { + return std::ranges::find(count_star_placeholder_columns, column_id) != + count_star_placeholder_columns.end(); + } + + bool is_predicate_only(LocalColumnId column_id) const { + return std::ranges::find(predicate_only_columns, column_id) != predicate_only_columns.end(); + } }; // Helper for constructing the scan-column layout in FileScanRequest. diff --git a/be/src/format_v2/jni/iceberg_sys_table_reader.cpp b/be/src/format_v2/jni/iceberg_sys_table_reader.cpp index 896929925aab3f..b41d505f886d31 100644 --- a/be/src/format_v2/jni/iceberg_sys_table_reader.cpp +++ b/be/src/format_v2/jni/iceberg_sys_table_reader.cpp @@ -17,10 +17,33 @@ #include "format_v2/jni/iceberg_sys_table_reader.h" +#include + +#include "format/jni/jni_data_bridge.h" +#include "util/string_util.h" + namespace doris::format::iceberg { +namespace { + +constexpr std::string_view HADOOP_OPTION_PREFIX = "hadoop."; + +} // namespace Status IcebergSysTableJniReader::validate_scan_range(const TFileRangeDesc& range) const { - return Status::NotSupported("native Iceberg system-table splits are unavailable on branch-4.1"); + if (!range.__isset.table_format_params) { + return Status::InternalError( + "missing table_format_params for iceberg sys table jni reader"); + } + if (!range.table_format_params.__isset.iceberg_params) { + return Status::InternalError("missing iceberg_params for iceberg sys table jni reader"); + } + if (!range.table_format_params.iceberg_params.__isset.serialized_split || + range.table_format_params.iceberg_params.serialized_split.empty()) { + return Status::InternalError( + "missing serialized_split for iceberg sys table jni reader, " + "possibly caused by FE/BE protocol mismatch"); + } + return Status::OK(); } std::string IcebergSysTableJniReader::connector_class() const { @@ -31,7 +54,23 @@ Status IcebergSysTableJniReader::build_scanner_params( std::map* params) const { DORIS_CHECK(params != nullptr); params->clear(); - return Status::NotSupported("native Iceberg system-table splits are unavailable on branch-4.1"); + params->emplace("serialized_split", + _current_range.table_format_params.iceberg_params.serialized_split); + + std::vector required_types; + required_types.reserve(_projected_columns.size()); + for (const auto& column : _projected_columns) { + required_types.emplace_back(JniDataBridge::get_jni_type_with_different_string(column.type)); + } + (*params)["required_types"] = join(required_types, "#"); + + if (_scan_params != nullptr && _scan_params->__isset.properties && + !_scan_params->properties.empty()) { + for (const auto& kv : _scan_params->properties) { + (*params)[std::string(HADOOP_OPTION_PREFIX) + kv.first] = kv.second; + } + } + return Status::OK(); } } // namespace doris::format::iceberg diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp index e9d225aeb46fc5..41d3a5282b6255 100644 --- a/be/src/format_v2/jni/jdbc_reader.cpp +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -30,22 +30,57 @@ #include "exprs/function/simple_function_factory.h" #include "exprs/vexpr_context.h" #include "format_v2/table_reader.h" +#include "util/jdbc_utils.h" namespace doris::format::jdbc { +Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows) { + const auto* nullable = check_and_get_column(&result); + if (UNLIKELY(nullable == nullptr)) { + return Status::InternalError("JDBC special-type CAST did not return a nullable column"); + } + if (UNLIKELY(nullable->has_null(0, rows))) { + // CAST NULL represents invalid source data; stripping the null map would turn it into a + // valid-looking default for a NOT NULL destination. + return Status::DataQualityError( + "JDBC special-type CAST produced NULL for a non-nullable column"); + } + return Status::OK(); +} + std::string JdbcJniReader::connector_class() const { return "org/apache/doris/jdbc/JdbcJniScanner"; } Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { - return Status::NotSupported("native JDBC file splits are unavailable on branch-4.1"); + { + // End these scopes before JniTableReader enters the same counters; nested use would count + // this JDBC parameter preparation twice instead of extending the common lifecycle total. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + _jdbc_params.clear(); + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.table_format_type == "jdbc") { + _jdbc_params = std::map( + options.current_range.table_format_params.jdbc_params.begin(), + options.current_range.table_format_params.jdbc_params.end()); + } + } + return format::JniTableReader::prepare_split(options); } // need pass to the java side, so the java scanner can parse the params and construct the JDBC connection Status JdbcJniReader::build_scanner_params(std::map* params) const { DORIS_CHECK(params != nullptr); - params->clear(); - return Status::NotSupported("native JDBC file splits are unavailable on branch-4.1"); + *params = _jdbc_params; + if (params->contains("jdbc_driver_url")) { + std::string resolved; + if (JdbcUtils::resolve_driver_url((*params)["jdbc_driver_url"], &resolved).ok()) { + (*params)["jdbc_driver_url"] = resolved; + } + } + return Status::OK(); } Status JdbcJniReader::build_jni_columns( @@ -163,6 +198,7 @@ Status JdbcJniReader::_cast_string_to_special_type(const format::JniTableReader: if (target_type->is_nullable()) { output_block->replace_by_position(column.output_index, result_column); } else { + RETURN_IF_ERROR(validate_non_nullable_special_type_result(*result_column, rows)); const auto* nullable_column = assert_cast(result_column.get()); output_block->replace_by_position(column.output_index, nullable_column->get_nested_column_ptr()); diff --git a/be/src/format_v2/jni/jdbc_reader.h b/be/src/format_v2/jni/jdbc_reader.h index 91a5878cb4622f..e10f8ad97902b8 100644 --- a/be/src/format_v2/jni/jdbc_reader.h +++ b/be/src/format_v2/jni/jdbc_reader.h @@ -29,6 +29,8 @@ namespace doris::format::jdbc { +Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows); + class JdbcJniReader final : public format::JniTableReader { public: ~JdbcJniReader() override = default; diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index 3cb105193b56ae..7658a52fc86e0f 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "exprs/vexpr_context.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/string_util.h" @@ -31,17 +32,31 @@ namespace doris::format { Status JniTableReader::init(TableReadOptions&& options) { RETURN_IF_ERROR(TableReader::init(std::move(options))); - _init_profile(); + { + // Base and derived scopes must not overlap on the same counter: RuntimeProfile timers add + // deltas, so nested use would double-count instead of extending lifecycle coverage. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _init_profile(); + } + SCOPED_TIMER(_connector_total_time); return Status::OK(); } Status JniTableReader::prepare_split(const SplitReadOptions& options) { - // EOF belongs to the previous split. Keep it set after closing that split so repeated reads - // are idempotent, and clear it only when a new split is explicitly prepared. - _eof = false; - _current_range = options.current_range; - RETURN_IF_ERROR(validate_scan_range(options.current_range)); + SCOPED_TIMER(_connector_total_time); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + // EOF belongs to the previous split. Keep it set after closing that split so repeated reads + // are idempotent, and clear it only when a new split is explicitly prepared. + _eof = false; + _current_range = options.current_range; + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -63,6 +78,9 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { } Status JniTableReader::get_block(Block* output_block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.exec_timer); + SCOPED_TIMER(_connector_total_time); DORIS_CHECK(output_block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(output_block->columns() == _projected_columns.size()); @@ -109,7 +127,11 @@ Status JniTableReader::get_block(Block* output_block, bool* eos) { } Status JniTableReader::abort_split() { - RETURN_IF_ERROR(_close_jni_scanner()); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + RETURN_IF_ERROR(_close_jni_scanner()); + } return TableReader::abort_split(); } @@ -342,10 +364,16 @@ void JniTableReader::_publish_split_profile(JNIEnv* env) { } Status JniTableReader::close() { + SCOPED_TIMER(_connector_total_time); if (_closed) { return Status::OK(); } - auto close_status = _close_jni_scanner(); + Status close_status; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + close_status = _close_jni_scanner(); + } auto table_status = TableReader::close(); if (close_status.ok() && !table_status.ok()) { close_status = std::move(table_status); @@ -535,7 +563,9 @@ void JniTableReader::_init_profile() { return; } const auto connector_name = _connector_name(); - ADD_TIMER(_scanner_profile, connector_name); + file_scan_profile::ensure_hierarchy(_scanner_profile); + _connector_total_time = + ADD_CHILD_TIMER(_scanner_profile, connector_name, file_scan_profile::TABLE_READER); _open_scanner_time = ADD_CHILD_TIMER(_scanner_profile, "OpenScannerTime", connector_name); _java_scan_time = ADD_CHILD_TIMER(_scanner_profile, "JavaScanTime", connector_name); _java_append_data_time = diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index fe908d654863eb..5e48270515f996 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -23,7 +23,7 @@ #include "common/status.h" #include "core/data_type/data_type.h" -#include "format_v2/jni/jni_data_bridge.h" +#include "format/jni/jni_data_bridge.h" #include "format_v2/table_reader.h" #include "runtime/runtime_profile.h" #include "util/jni-util.h" @@ -84,6 +84,7 @@ class JniTableReader : public TableReader { virtual Status _open_jni_scanner(); bool _reserve_split_profile_publication(); const std::vector& jni_columns() const { return _jni_columns; } + RuntimeProfile::Counter* connector_total_timer() const { return _connector_total_time; } TFileRangeDesc _current_range; private: @@ -110,6 +111,7 @@ class JniTableReader : public TableReader { bool _eof = false; bool _split_profile_published = false; + RuntimeProfile::Counter* _connector_total_time = nullptr; RuntimeProfile::Counter* _open_scanner_time = nullptr; RuntimeProfile::Counter* _java_scan_time = nullptr; RuntimeProfile::Counter* _java_append_data_time = nullptr; diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.cpp b/be/src/format_v2/jni/trino_connector_jni_reader.cpp index 11c9945c5dea16..4f0a3f55c3891e 100644 --- a/be/src/format_v2/jni/trino_connector_jni_reader.cpp +++ b/be/src/format_v2/jni/trino_connector_jni_reader.cpp @@ -84,8 +84,15 @@ Status TrinoConnectorJniReader::validate_scan_range(const TFileRangeDesc& range) } Status TrinoConnectorJniReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(validate_scan_range(options.current_range)); - RETURN_IF_ERROR(_set_spi_plugins_dir()); + { + // Plugin discovery can dominate a cold split. Use non-overlapping common scopes because + // the JNI base method subsequently enters the same RuntimeProfile counters. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(_set_spi_plugins_dir()); + } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 313115e82ce744..caf4205fa61c5b 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -47,6 +47,7 @@ #include "io/fs/stream_load_pipe.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/slice.h" @@ -174,7 +175,26 @@ JsonReader::~JsonReader() { static_cast(close()); } +void JsonReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* json_profile = "JsonReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, json_profile, file_scan_profile::FILE_READER, 1); + _open_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonOpenTime", json_profile, 1); + _read_document_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonReadDocumentTime", json_profile, 1); + _parse_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonParseTime", json_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonMaterializeTime", json_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonFilterTime", json_profile, 1); +} + Status JsonReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("JSON v2 reader requires scan params"); @@ -230,6 +250,7 @@ Status JsonReader::init(RuntimeState* state) { } Status JsonReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("JSON v2 file_schema is null"); } @@ -243,14 +264,22 @@ std::unique_ptr JsonReader::create_column_mapper( } Status JsonReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); _slot_name_to_index.clear(); + _hive_slot_name_to_index.clear(); _slot_name_to_index.reserve(_requested_columns.size()); + _hive_slot_name_to_index.reserve(_requested_columns.size()); for (size_t idx = 0; idx < _requested_columns.size(); ++idx) { auto name = _requested_columns[idx].slot_desc->col_name(); - _slot_name_to_index.emplace(_is_hive_table ? lower_key(name) : name, idx); + if (_is_hive_table) { + _hive_slot_name_to_index.emplace(std::move(name), idx); + } else { + _slot_name_to_index.emplace(std::move(name), idx); + } } _previous_positions.clear(); _reader_range = _json_range(); @@ -269,6 +298,7 @@ Status JsonReader::open(std::shared_ptr request) { } Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -297,7 +327,10 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { bool is_empty_row = false; Status st = Status::OK(); try { - st = _parse_next_json(&size, &_reader_eof); + { + SCOPED_TIMER(_parse_time); + st = _parse_next_json(&size, &_reader_eof); + } if (st.ok() && !_reader_eof) { if (size == 0) { is_empty_row = true; @@ -306,6 +339,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { } } if (st.ok() && !_reader_eof && !is_empty_row) { + SCOPED_TIMER(_materialize_time); st = _append_rows_from_current_value(file_block, &is_empty_row, &_reader_eof); } } catch (simdjson::simdjson_error& e) { @@ -324,13 +358,17 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } *eof = _reader_eof && *rows == 0; _eof = *eof; return Status::OK(); } Status JsonReader::close() { + SCOPED_TIMER(_total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); @@ -489,6 +527,7 @@ Status JsonReader::_parse_jsonpath_and_json_root() { } Status JsonReader::_read_one_document(size_t* size, bool* eof) { + SCOPED_TIMER(_read_document_time); DORIS_CHECK(size != nullptr); DORIS_CHECK(eof != nullptr); *size = 0; @@ -499,7 +538,9 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) { if (*eof) { return Status::OK(); } - _document_buffer.assign(reinterpret_cast(line), *size); + // The line reader owns this span until the next read, and parsing copies it immediately + // into the padded buffer. Borrowing it avoids a redundant line-sized string copy. + _document_view = std::string_view(reinterpret_cast(line), *size); return Status::OK(); } // Non-line mode treats the split as one JSON document. This supports a single object or an @@ -525,6 +566,7 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) { Slice result(_document_buffer.data(), _document_buffer.size()); RETURN_IF_ERROR(_physical_file_reader->read_at(_current_offset, result, size, _io_ctx.get())); _document_buffer.resize(*size); + _document_view = _document_buffer; if (*size == 0) { *eof = true; } @@ -539,6 +581,7 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { DorisUniqueBufferPtr file_buf; RETURN_IF_ERROR(stream_load_pipe->read_one_message(&file_buf, read_size)); _document_buffer.assign(reinterpret_cast(file_buf.get()), *read_size); + _document_view = _document_buffer; if (!stream_load_pipe->is_chunked_transfer()) { return Status::OK(); } @@ -553,6 +596,7 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { _document_buffer.append(reinterpret_cast(next_buf.get()), next_size); *read_size += next_size; } + _document_view = _document_buffer; return Status::OK(); } @@ -561,10 +605,10 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) { if (*eof || *size == 0) { return Status::OK(); } - if (*size >= 3 && static_cast(_document_buffer[0]) == 0xEF && - static_cast(_document_buffer[1]) == 0xBB && - static_cast(_document_buffer[2]) == 0xBF) { - _document_buffer.erase(0, 3); + if (*size >= 3 && static_cast(_document_view[0]) == 0xEF && + static_cast(_document_view[1]) == 0xBB && + static_cast(_document_view[2]) == 0xBF) { + _document_view.remove_prefix(3); *size -= 3; } if (*size + simdjson::SIMDJSON_PADDING > _padded_size) { @@ -573,7 +617,7 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) { } // Ondemand values reference the input buffer. Keep the padded bytes in a member buffer until the // current document is fully materialized. - std::memcpy(_padding_buffer.data(), _document_buffer.data(), *size); + std::memcpy(_padding_buffer.data(), _document_view.data(), *size); _original_doc_size = *size; const auto error = _json_parser->iterate(std::string_view(_padding_buffer.data(), *size), _padded_size) @@ -1087,32 +1131,39 @@ void JsonReader::_pop_back_last_inserted_value(Block* block, size_t column_index } size_t JsonReader::_column_index(std::string_view key, size_t key_index) { - std::string hive_key; - std::string_view lookup_key = key; - if (_is_hive_table) { - hive_key = lower_key(key); - lookup_key = hive_key; - } if (key_index < _previous_positions.size()) { // Most JSON lines share field order. Reuse the previous line's key-position mapping before // falling back to the hash table lookup. const auto previous = _previous_positions[key_index]; if (previous < _requested_columns.size()) { const auto previous_name = _requested_columns[previous].slot_desc->col_name(); - if ((_is_hive_table ? lower_key(previous_name) : previous_name) == lookup_key) { + if ((_is_hive_table && CaseInsensitiveStringEqual {}(previous_name, key)) || + (!_is_hive_table && previous_name == key)) { return previous; } } } - const auto it = _slot_name_to_index.find(std::string(lookup_key)); - if (it == _slot_name_to_index.end()) { + // Transparent lookup keeps the common per-key path allocation-free; Hive case folding is + // performed by the map's hash/equality without constructing a lower-cased string. + const auto index = _is_hive_table ? ([&]() -> std::optional { + const auto it = _hive_slot_name_to_index.find(key); + return it == _hive_slot_name_to_index.end() ? std::nullopt + : std::optional(it->second); + })() + : ([&]() -> std::optional { + const auto it = _slot_name_to_index.find(key); + return it == _slot_name_to_index.end() + ? std::nullopt + : std::optional(it->second); + })(); + if (!index.has_value()) { return static_cast(-1); } if (key_index >= _previous_positions.size()) { _previous_positions.resize(key_index + 1, static_cast(-1)); } - _previous_positions[key_index] = it->second; - return it->second; + _previous_positions[key_index] = *index; + return *index; } bool JsonReader::_is_root_path_for_column(const RequestedColumn& column) const { diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index 52cdfad6728d64..c7346cb1d66357 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -42,6 +42,40 @@ class IColumn; namespace doris::format::json { +struct TransparentStringHash { + using is_transparent = void; + size_t operator()(std::string_view value) const { + return std::hash {}(value); + } +}; + +struct CaseInsensitiveStringHash { + using is_transparent = void; + size_t operator()(std::string_view value) const { + size_t hash = 1469598103934665603ULL; + for (const unsigned char c : value) { + const unsigned char folded = c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + hash = (hash ^ folded) * 1099511628211ULL; + } + return hash; + } +}; + +struct CaseInsensitiveStringEqual { + using is_transparent = void; + bool operator()(std::string_view lhs, std::string_view rhs) const { + if (lhs.size() != rhs.size()) return false; + for (size_t i = 0; i < lhs.size(); ++i) { + const unsigned char left = + lhs[i] >= 'A' && lhs[i] <= 'Z' ? lhs[i] + ('a' - 'A') : lhs[i]; + const unsigned char right = + rhs[i] >= 'A' && rhs[i] <= 'Z' ? rhs[i] + ('a' - 'A') : rhs[i]; + if (left != right) return false; + } + return true; + } +}; + // FileScannerV2 JSON reader. // // JSON files do not carry an embedded physical schema. The v2 table layer still needs a @@ -74,7 +108,12 @@ class JsonReader final : public FileReader { Status get_block(Block* file_block, size_t* rows, bool* eof) override; Status close() override; +#ifdef BE_TEST + size_t TEST_document_buffer_size() const { return _document_buffer.size(); } +#endif + private: + void _init_profile() override; // A requested column keeps both identities: // - `source_index`: index in FE file slots, used for jsonpaths and SerDe lookup. // - `block_position`: index in the caller's output block, used for materialization. @@ -134,9 +173,19 @@ class JsonReader final : public FileReader { TFileCompressType::type _range_compress_type = TFileCompressType::UNKNOWN; std::optional _stream_load_id; std::vector _requested_columns; - std::unordered_map _slot_name_to_index; + std::unordered_map> + _slot_name_to_index; + std::unordered_map + _hive_slot_name_to_index; std::vector _previous_positions; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_time = nullptr; + RuntimeProfile::Counter* _read_document_time = nullptr; + RuntimeProfile::Counter* _parse_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + io::FileReaderSPtr _physical_file_reader; std::unique_ptr _decompressor; std::unique_ptr _line_reader; @@ -170,6 +219,7 @@ class JsonReader final : public FileReader { simdjson::ondemand::array _array; simdjson::ondemand::array_iterator _array_iter; std::string _document_buffer; + std::string_view _document_view; std::string _padding_buffer; size_t _original_doc_size = 0; size_t _padded_size = 1024 * 1024 * 8 + simdjson::SIMDJSON_PADDING; diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp index 0e348f3de2ccf5..3d5429e0a9a262 100644 --- a/be/src/format_v2/native/native_reader.cpp +++ b/be/src/format_v2/native/native_reader.cpp @@ -29,6 +29,7 @@ #include "format_v2/materialized_reader_util.h" #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/slice.h" @@ -54,7 +55,26 @@ NativeReader::~NativeReader() { static_cast(close()); } +void NativeReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* native_profile = "NativeReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, native_profile, file_scan_profile::FILE_READER, 1); + _read_block_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeReadBlockTime", native_profile, 1); + _deserialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeDeserializeTime", native_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeMaterializeTime", native_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeFilterTime", native_profile, 1); +} + Status NativeReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_file_description == nullptr) { return Status::InvalidArgument("Native v2 reader requires file description"); @@ -65,6 +85,7 @@ Status NativeReader::init(RuntimeState* state) { } Status NativeReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("Native v2 file_schema is null"); } @@ -79,6 +100,7 @@ std::unique_ptr NativeReader::create_column_mapper( } Status NativeReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); _first_block_consumed = false; @@ -88,6 +110,7 @@ Status NativeReader::open(std::shared_ptr request) { } Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -108,6 +131,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { if (_first_block_loaded && !_first_block_consumed) { buffer = _first_block_buffer; } else { + SCOPED_TIMER(_read_block_time); RETURN_IF_ERROR(_read_next_pblock(&buffer, &local_eof)); } @@ -131,11 +155,20 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { Block source_block; size_t uncompressed_bytes = 0; int64_t decompress_time = 0; - RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); - RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + { + SCOPED_TIMER(_deserialize_time); + RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); + } + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + } *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } if (_first_block_loaded && !_first_block_consumed) { _first_block_consumed = true; @@ -149,6 +182,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { } Status NativeReader::close() { + SCOPED_TIMER(_total_time); _file_reader.reset(); _tracing_file_reader.reset(); _request.reset(); @@ -231,6 +265,16 @@ Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { return Status::OK(); } + const auto remaining_prefix_bytes = _file_size - _current_offset; + if (remaining_prefix_bytes < sizeof(uint64_t)) { + // Header-only is the one valid empty-file representation and returned above. Once any + // prefix byte exists, all eight bytes are mandatory. For example, `header + 4 bytes` is a + // truncated Native file, not EOF that FileScannerV2 may skip as an empty split. + return Status::InternalError( + "truncated native block length in file {} at offset {}, expect {}, remaining {}", + _file_description->path, _current_offset, sizeof(uint64_t), remaining_prefix_bytes); + } + uint64_t block_len = 0; Slice len_slice(reinterpret_cast(&block_len), sizeof(block_len)); size_t bytes_read = 0; @@ -247,8 +291,22 @@ Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { } _current_offset += sizeof(block_len); if (block_len == 0) { - *eof = (_current_offset >= _file_size); - return Status::OK(); + // A header-only file reaches the `_current_offset >= _file_size` branch above and is a + // valid empty Native file. Once an explicit length prefix is present, however, zero is not + // an EOF marker in the Native format. For example, `header + uint64(0)` is truncated input, + // not a zero-row split, and must not escape as EOF for FileScannerV2 to count as empty. + return Status::InternalError("zero-length native block in file {} at offset {}", + _file_description->path, _current_offset - sizeof(block_len)); + } + + const auto remaining_block_bytes = cast_set(_file_size - _current_offset); + if (block_len > remaining_block_bytes) { + // Validate before allocating `block_len` bytes. Besides reporting a precise corruption + // error, this prevents a truncated file with a damaged length prefix from requesting an + // allocation much larger than the physical file. + return Status::InternalError( + "truncated native block body in file {} at offset {}, expect {}, remaining {}", + _file_description->path, _current_offset, block_len, remaining_block_bytes); } buffer->assign(block_len, '\0'); diff --git a/be/src/format_v2/native/native_reader.h b/be/src/format_v2/native/native_reader.h index 3719a6afd6c4f5..15a52fe6f8bc87 100644 --- a/be/src/format_v2/native/native_reader.h +++ b/be/src/format_v2/native/native_reader.h @@ -49,6 +49,7 @@ class NativeReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _validate_and_consume_header(); Status _ensure_schema_loaded() const; Status _read_next_pblock(std::string* buffer, bool* eof) const; @@ -65,6 +66,11 @@ class NativeReader final : public FileReader { mutable std::string _first_block_buffer; mutable bool _first_block_loaded = false; mutable bool _first_block_consumed = false; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _read_block_time = nullptr; + RuntimeProfile::Counter* _deserialize_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; }; } // namespace doris::format::native diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index a5144d7b2017b0..df0927aac3444a 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -27,6 +27,7 @@ #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "orc/Exceptions.hh" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "util/slice.h" @@ -54,24 +55,28 @@ class OrcMergedRangeFileReader final : public io::FileReader { _size(_file_reader->size()) { _statistics.apply_bytes += _range.end_offset - _range.start_offset; if (_profile != nullptr) { - const char* profile_name = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, profile_name, 1); - _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); - _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); - _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, + const char* profile_name = "OrcMergedSmallIO"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); + // RuntimeProfile counter lookup is flat, so every child must be format-qualified; + // a unique parent alone cannot prevent aliasing with Parquet's MergeRange reader. + _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedCopyTime", profile_name, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedReadTime", profile_name, 1); + _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestIO", TUnit::UNIT, profile_name, 1); - _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT, + _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedIO", TUnit::UNIT, profile_name, 1); - _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, - profile_name, 1); - _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestBytes", + TUnit::BYTES, profile_name, 1); + _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedBytes", TUnit::BYTES, profile_name, 1); - _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES, - profile_name, 1); - _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OverReadBytes", TUnit::BYTES, - profile_name, 1); - _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ClusterNum", TUnit::UNIT, - profile_name, 1); + _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedApplyBytes", + TUnit::BYTES, profile_name, 1); + _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedOverReadBytes", + TUnit::BYTES, profile_name, 1); + _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedClusterNum", + TUnit::UNIT, profile_name, 1); } } @@ -113,6 +118,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { if (_profile == nullptr) { return; } + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -165,6 +171,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { OrcMergedRangeStatistics _statistics; RuntimeProfile::Counter* _copy_time = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; RuntimeProfile::Counter* _merged_io = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 9a2aeb538977dc..5248fcaf22dd11 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -76,6 +76,7 @@ #include "format_v2/timestamp_statistics.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/segment/condition_cache.h" @@ -776,7 +777,9 @@ void OrcReader::_init_profile() { } static const char* orc_profile = "OrcReader"; - ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1); + file_scan_profile::ensure_hierarchy(_profile); + _orc_profile.total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1); _orc_profile.reader_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -803,20 +806,22 @@ void OrcReader::_init_profile() { _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); _orc_profile.read_row_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsFiltered", - TUnit::UNIT, orc_profile, 1); + // RuntimeProfile counter names are flat; format-qualified names keep ORC ownership stable when + // one scan profile also initializes Parquet counters in either order. + _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcRowGroupsFiltered", TUnit::UNIT, orc_profile, 1); _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); - _orc_profile.read_row_groups = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsReadNum", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByGroup", - TUnit::UNIT, orc_profile, 1); + _profile, "OrcRowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcRowGroupsReadNum", + TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcFilteredRowsByGroup", TUnit::UNIT, orc_profile, 1); _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_bytes = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); + _profile, "OrcFilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFilteredBytes", + TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFileNum", TUnit::UNIT, orc_profile, 1); } void OrcReader::_collect_profile() const { @@ -874,6 +879,8 @@ format::ColumnDefinition OrcReader::row_position_column_definition() { } Status OrcReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_orc_profile.total_time); RETURN_IF_ERROR(format::FileReader::init(state)); _state = std::make_unique(); TimezoneUtils::find_cctz_time_zone(_state->timezone, _state->timezone_obj); @@ -1162,6 +1169,7 @@ Status OrcReader::_fill_map_schema_children(const ::orc::Type& type, } Status OrcReader::get_schema(std::vector* const file_schema) const { + SCOPED_TIMER(_orc_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } @@ -1195,6 +1203,7 @@ std::unique_ptr OrcReader::create_column_mapper( } Status OrcReader::open(std::shared_ptr request) { + SCOPED_TIMER(_orc_profile.total_time); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); } @@ -1877,6 +1886,7 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type } Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -1990,7 +2000,13 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { } _state->orc_lazy_selection_valid = false; } else { - RETURN_IF_ERROR(_filter_block(file_block, rows)); + auto filter_status = _filter_block(file_block, rows); + if (!filter_status.ok()) { + // V2 evaluates residual predicates after ORC returns the batch, while callers retain + // the historical nextBatch error contract used to identify row-filter failures. + filter_status.prepend("Orc row reader nextBatch failed. reason = "); + return filter_status; + } } *eof = false; return Status::OK(); @@ -2000,6 +2016,7 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { // NOLINTNEXTLINE(readability-function-size) Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(result != nullptr); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); @@ -2455,6 +2472,7 @@ void OrcReader::_filter_requested_columns(Block* file_block, const IColumn::Filt } Status OrcReader::close() { + SCOPED_TIMER(_orc_profile.total_time); _collect_profile(); if (_state != nullptr) { _state = std::make_unique(); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index adba20fbce0b10..a67dc5ad7f5e3e 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -72,6 +72,7 @@ class OrcReader final : public format::FileReader { private: struct OrcProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs RuntimeProfile::Counter* decompression_call = nullptr; // DecompressionCall diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp new file mode 100644 index 00000000000000..9eba6d8feaa55a --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -0,0 +1,874 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_desc.h" + +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "util/slice.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +static bool is_group_node(const tparquet::SchemaElement& schema) { + return schema.num_children > 0; +} + +static bool is_list_node(const tparquet::SchemaElement& schema) { + // Writers may emit only the modern logical type, so collection shape must not depend on the + // deprecated converted_type field being duplicated in the footer. + return (schema.__isset.converted_type && + schema.converted_type == tparquet::ConvertedType::LIST) || + (schema.__isset.logicalType && schema.logicalType.__isset.LIST); +} + +static bool is_map_node(const tparquet::SchemaElement& schema) { + return (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::MAP || + schema.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE)) || + (schema.__isset.logicalType && schema.logicalType.__isset.MAP); +} + +static bool has_primitive_only_annotation(const tparquet::SchemaElement& schema) { + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.DECIMAL || + logical.__isset.DATE || logical.__isset.TIME || logical.__isset.TIMESTAMP || + logical.__isset.INTEGER || logical.__isset.UNKNOWN || logical.__isset.JSON || + logical.__isset.BSON || logical.__isset.UUID || logical.__isset.FLOAT16 || + logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + return true; + } + } + if (!schema.__isset.converted_type) { + return false; + } + return schema.converted_type != tparquet::ConvertedType::MAP && + schema.converted_type != tparquet::ConvertedType::MAP_KEY_VALUE && + schema.converted_type != tparquet::ConvertedType::LIST; +} + +static bool is_repeated_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REPEATED; +} + +static bool is_required_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REQUIRED; +} + +static bool is_optional_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::OPTIONAL; +} + +static int num_children_node(const tparquet::SchemaElement& schema) { + return schema.__isset.num_children ? schema.num_children : 0; +} + +static Status validate_native_schema_structure( + const std::vector& schemas) { + if (schemas.empty()) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + const auto& root = schemas[0]; + if (root.__isset.type || !root.__isset.num_children || root.num_children < 0 || + (root.__isset.repetition_type && + root.repetition_type != tparquet::FieldRepetitionType::REQUIRED)) { + // Writers may encode the root's implicit REQUIRED repetition explicitly, and a zero-child + // root is a valid metadata-only schema. Optional or repeated roots remain ambiguous. + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + struct PendingGroup { + size_t remaining_children; + size_t depth; + }; + const auto root_children = num_children_node(schemas[0]); + if (root_children < 0 || static_cast(root_children) > schemas.size() - 1) { + return Status::InvalidArgument("Invalid parquet root child count {}", root_children); + } + std::vector pending {{static_cast(root_children), 0}}; + for (size_t pos = 1; pos < schemas.size(); ++pos) { + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (pending.empty()) { + return Status::InvalidArgument("Schema element {} is not reachable from the root", pos); + } + + const size_t depth = pending.back().depth + 1; + --pending.back().remaining_children; + const auto& schema = schemas[pos]; + if (!schema.__isset.repetition_type) { + return Status::InvalidArgument("Schema element {} has no repetition type", pos); + } + const bool has_children = schema.__isset.num_children && schema.num_children > 0; + if (schema.__isset.type == has_children) { + // Some legacy parquet-cpp files explicitly encode num_children=0 on primitive nodes. + // Only a positive count denotes a group, preserving strict rejection of real dual-kind + // nodes without dropping those otherwise valid files. + return Status::InvalidArgument("Schema element {} has ambiguous primitive/group kind", + pos); + } + if (schema.__isset.num_children && schema.num_children < 0) { + return Status::InvalidArgument("Schema element {} has a negative child count", pos); + } + const int children = num_children_node(schemas[pos]); + if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { + return Status::InvalidArgument("Invalid child count {} at schema element {}", children, + pos); + } + if (children > 0) { + // Bound the tree before any resize or recursion so an untrusted footer cannot + // turn a tiny schema into an unbounded allocation or parser stack. + if (depth > MAX_NATIVE_SCHEMA_DEPTH) { + return Status::InvalidArgument("Parquet schema depth {} exceeds limit {}", depth, + MAX_NATIVE_SCHEMA_DEPTH); + } + pending.push_back({static_cast(children), depth}); + } + } + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (!pending.empty()) { + return Status::InvalidArgument("Parquet schema ended before all children were parsed"); + } + return Status::OK(); +} + +/** + * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. + * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the + * empty or null values in ancestor node. + * + * For instance, considering an array of strings with 3 rows like the following: + * null, [], [a, b, c] + * We can store four elements in data column: null, a, b, c + * and the offsets column is: 1, 1, 4 + * and the null map is: 1, 0, 0 + * For the i-th row in array column: range from `offsets[i - 1]` until `offsets[i]` represents the elements in this row, + * so we can't store empty array/map values in doris data column. + * As a comparison, spark does not require `repeated_parent_def_level`, + * because the spark column stores empty array/map values , and use anther length column to indicate empty values. + * Please reference: https://github.com/apache/spark/blob/master/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnVector.java + * + * Furthermore, we can also avoid store null array/map values in doris data column. + * The same three rows as above, We can only store three elements in data column: a, b, c + * and the offsets column is: 0, 0, 3 + * and the null map is: 1, 0, 0 + * + * Inherit the repetition and definition level from parent node, if the parent node is repeated, + * we should set repeated_parent_def_level = definition_level, otherwise as repeated_parent_def_level. + * @param parent parent node + * @param repeated_parent_def_level the first ancestor node whose repetition_type equals REPEATED + */ +static void set_child_node_level(NativeFieldSchema* parent, int16_t repeated_parent_def_level) { + for (auto& child : parent->children) { + child.repetition_level = parent->repetition_level; + child.definition_level = parent->definition_level; + child.repeated_parent_def_level = repeated_parent_def_level; + } +} + +static bool is_struct_list_node(const tparquet::SchemaElement& schema, + const std::string& enclosing_list_name) { + // The legacy Parquet exception is exact: accepting every "*_tuple" wrapper changes a standard + // one-child LIST wrapper from ARRAY to ARRAY>. + return schema.name == "array" || schema.name == enclosing_list_name + "_tuple"; +} + +std::string NativeFieldSchema::debug_string() const { + std::stringstream ss; + ss << "NativeFieldSchema(name=" << name << ", R=" << repetition_level + << ", D=" << definition_level; + if (children.size() > 0) { + ss << ", type=" << data_type->get_name() << ", children=["; + for (int i = 0; i < children.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << children[i].debug_string(); + } + ss << "]"; + } else { + ss << ", physical_type=" << physical_type; + ss << " , doris_type=" << data_type->get_name(); + } + ss << ")"; + return ss.str(); +} + +Status NativeFieldDescriptor::parse_from_thrift( + const std::vector& t_schemas) { + _fields.clear(); + _physical_fields.clear(); + _name_to_field.clear(); + RETURN_IF_ERROR(validate_native_schema_structure(t_schemas)); + const auto& root_schema = t_schemas[0]; + _fields.resize(root_schema.num_children); + _next_schema_pos = 1; + + for (int i = 0; i < root_schema.num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &_fields[i])); + if (_name_to_field.find(_fields[i].name) != _name_to_field.end()) { + return Status::InvalidArgument("Duplicated field name: {}", _fields[i].name); + } + _name_to_field.emplace(_fields[i].name, &_fields[i]); + } + + if (_next_schema_pos != t_schemas.size()) { + return Status::InvalidArgument("Remaining {} unparsed schema elements", + t_schemas.size() - _next_schema_pos); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_node_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field) { + if (curr_pos >= t_schemas.size()) { + return Status::InvalidArgument("Out-of-bounds index of schema elements"); + } + auto& t_schema = t_schemas[curr_pos]; + if (is_group_node(t_schema)) { + // nested structure or nullable list + return parse_group_field(t_schemas, curr_pos, node_field); + } + if (is_repeated_node(t_schema)) { + // repeated (LIST) + // produce required list + node_field->repetition_level++; + node_field->definition_level++; + node_field->children.resize(1); + set_child_node_level(node_field, node_field->definition_level); + auto child = &node_field->children[0]; + parse_physical_field(t_schema, false, child); + + node_field->name = t_schema.name; + node_field->lower_case_name = to_lower(t_schema.name); + node_field->data_type = std::make_shared(make_nullable(child->data_type)); + _next_schema_pos = curr_pos + 1; + node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1; + } else { + bool is_optional = is_optional_node(t_schema); + if (is_optional) { + node_field->definition_level++; + } + parse_physical_field(t_schema, is_optional, node_field); + _next_schema_pos = curr_pos + 1; + } + return Status::OK(); +} + +void NativeFieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema, + bool is_nullable, + NativeFieldSchema* physical_field) { + physical_field->name = physical_schema.name; + physical_field->lower_case_name = to_lower(physical_field->name); + physical_field->parquet_schema = physical_schema; + physical_field->physical_type = physical_schema.type; + physical_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + _physical_fields.push_back(physical_field); + physical_field->physical_column_index = cast_set(_physical_fields.size() - 1); + std::pair type; + try { + type = get_doris_type(physical_schema, is_nullable); + } catch (const Exception& e) { + const bool is_decimal = + (physical_schema.__isset.logicalType && + physical_schema.logicalType.__isset.DECIMAL) || + (physical_schema.__isset.converted_type && + physical_schema.converted_type == tparquet::ConvertedType::DECIMAL); + // DECIMAL conversion failures carry precision/scale semantics that must not be erased by + // a raw-byte fallback. Other unknown annotations intentionally retain the established + // physical fallback, including legacy INTERVAL footers and forward-compatible types. + if (is_decimal) { + physical_field->unsupported_reason = e.what(); + } + auto fallback_schema = physical_schema; + fallback_schema.__isset.logicalType = false; + fallback_schema.__isset.converted_type = false; + type = get_doris_type(fallback_schema, is_nullable); + } + physical_field->data_type = type.first; + physical_field->is_type_compatibility = type.second; + physical_field->field_id = physical_schema.__isset.field_id ? physical_schema.field_id : -1; +} + +std::pair NativeFieldDescriptor::get_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + if (physical_schema.__isset.logicalType) { + ans = convert_to_doris_type(physical_schema.logicalType, nullable); + } else if (physical_schema.__isset.converted_type) { + ans = convert_to_doris_type(physical_schema, nullable); + } + if (ans.first->get_primitive_type() == PrimitiveType::INVALID_TYPE) { + switch (physical_schema.type) { + case tparquet::Type::BOOLEAN: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BOOLEAN, nullable); + break; + case tparquet::Type::INT32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::Type::INT64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::Type::INT96: + if (_enable_mapping_timestamp_tz) { + // treat INT96 as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, nullable, + 0, 6); + } else { + // in most cases, it's a nano timestamp + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, + 0, 6); + } + break; + case tparquet::Type::FLOAT: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + break; + case tparquet::Type::DOUBLE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, nullable); + break; + case tparquet::Type::BYTE_ARRAY: + if (_enable_mapping_varbinary) { + // if physical_schema not set logicalType and converted_type, + // we treat BYTE_ARRAY as VARBINARY by default, so that we can read all data directly. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet logicalType{}", + physical_schema.type)); + break; + } + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + tparquet::LogicalType logicalType, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + if (logicalType.__isset.STRING || logicalType.__isset.ENUM || logicalType.__isset.JSON || + logicalType.__isset.BSON) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } else if (logicalType.__isset.DECIMAL) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, + logicalType.DECIMAL.precision, + logicalType.DECIMAL.scale); + } else if (logicalType.__isset.DATE) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + } else if (logicalType.__isset.INTEGER) { + if (logicalType.INTEGER.isSigned) { + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } + } else { + is_type_compatibility = true; + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + } + } + } else if (logicalType.__isset.TIME) { + const int scale = logicalType.TIME.unit.__isset.MILLIS ? 3 : 6; + // TIME stores an integer unit, so its Doris scale must preserve the footer unit or + // sub-second values are silently truncated by the target SerDe. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, scale); + } else if (logicalType.__isset.TIMESTAMP) { + if (_enable_mapping_timestamp_tz) { + if (logicalType.TIMESTAMP.isAdjustedToUTC) { + // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_TIMESTAMPTZ, nullable, 0, + logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + return ans; + } + } + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + } else if (logicalType.__isset.UUID) { + if (_enable_mapping_varbinary) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, + -1, 16); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + } else if (logicalType.__isset.FLOAT16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else { + throw Exception(Status::InternalError("Not supported parquet logicalType")); + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + switch (physical_schema.converted_type) { + case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: + case tparquet::ConvertedType::type::JSON: + case tparquet::ConvertedType::type::BSON: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + case tparquet::ConvertedType::type::DECIMAL: + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DECIMAL128I, nullable, physical_schema.precision, physical_schema.scale); + break; + case tparquet::ConvertedType::type::DATE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + break; + case tparquet::ConvertedType::type::TIME_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIME_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::INT_8: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_8: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_16: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_16: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::ConvertedType::type::UINT_32: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_64: + is_type_compatibility = true; + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet ConvertedType: {}", + physical_schema.converted_type)); + } + return ans; +} + +Status NativeFieldDescriptor::parse_group_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field) { + auto& group_schema = t_schemas[curr_pos]; + if ((group_schema.__isset.logicalType && group_schema.logicalType.__isset.ENUM) || + (group_schema.__isset.converted_type && + group_schema.converted_type == tparquet::ConvertedType::ENUM)) { + // Preserve the established diagnostic used by compatibility tests and clients while the + // generic branch below handles every other primitive-only group annotation. + return Status::InvalidArgument("Logical type Enum cannot be applied to group node {}", + group_schema.name); + } + if (has_primitive_only_annotation(group_schema)) { + // Primitive annotations cannot change a group into a scalar. Reject them here instead of + // silently interpreting malformed metadata as an ordinary STRUCT. + return Status::InvalidArgument("Primitive annotation cannot be applied to group node {}", + group_schema.name); + } + if (is_map_node(group_schema)) { + // the map definition: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + return parse_map_field(t_schemas, curr_pos, group_field); + } + if (is_list_node(group_schema)) { + // the list definition: + // optional group (LIST) { + // repeated group [bag | list] { // hive or spark + // optional [array_element | element]; // hive or spark + // } + // } + return parse_list_field(t_schemas, curr_pos, group_field); + } + + if (is_repeated_node(group_schema)) { + group_field->repetition_level++; + group_field->definition_level++; + group_field->children.resize(1); + set_child_node_level(group_field, group_field->definition_level); + auto struct_field = &group_field->children[0]; + // the list of struct: + // repeated group (LIST) { + // optional/required ; + // ... + // } + // produce a non-null list + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field)); + + group_field->name = group_schema.name; + group_field->lower_case_name = to_lower(group_field->name); + group_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + group_field->data_type = + std::make_shared(make_nullable(struct_field->data_type)); + group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1; + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, group_field)); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_list_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, bool repeated_node_is_enclosing_list_element) { + // the list definition: + // spark and hive have three level schemas but with different schema name + // spark: - "list" - "element" + // hive: - "bag" - "array_element" + // parse three level schemas to two level primitive like: LIST, + // or nested structure like: LIST> + auto& first_level = t_schemas[curr_pos]; + if (first_level.num_children != 1) { + return Status::InvalidArgument("List element should have only one child"); + } + + if (curr_pos + 1 >= t_schemas.size()) { + return Status::InvalidArgument("List element should have the second level schema"); + } + + if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED && + !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("List element can't be a repeated schema"); + } + + // the repeated schema element + auto& second_level = t_schemas[curr_pos + 1]; + if (second_level.repetition_type != tparquet::FieldRepetitionType::REPEATED) { + return Status::InvalidArgument("The second level of list element should be repeated"); + } + + // This indicates if this list is nullable. + bool is_optional = is_optional_node(first_level); + if (is_optional) { + list_field->definition_level++; + } + list_field->repetition_level++; + list_field->definition_level++; + list_field->children.resize(1); + NativeFieldSchema* list_child = &list_field->children[0]; + + size_t num_children = num_children_node(second_level); + if (num_children > 0) { + const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); + const auto& only_child = t_schemas[curr_pos + 2]; + const bool single_key_set_wrapper = + second_level.__isset.converted_type && + second_level.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE && + !is_repeated_node(only_child); + const bool nested_collection_annotation = + is_list_node(second_level) || + (is_map_node(second_level) && !single_key_set_wrapper); + if (num_children == 1 && !structural_wrapper && nested_collection_annotation) { + // The repeated node is already the outer LIST element. Preserve its own LIST/MAP + // annotation. A MAP_KEY_VALUE node with a repeated child is the legacy uncontained + // MAP shape; only its direct, non-repeated child denotes the enclosing SET key. + set_child_node_level(list_field, list_field->definition_level); + if (is_list_node(second_level)) { + RETURN_IF_ERROR(parse_list_field(t_schemas, curr_pos + 1, list_child, true)); + } else if (is_map_node(second_level)) { + RETURN_IF_ERROR(parse_map_field(t_schemas, curr_pos + 1, list_child, true)); + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 1 && !structural_wrapper && !is_repeated_node(only_child)) { + // optional field, and the third level element is the nested structure in list + // produce nested structure like: LIST, LIST, LIST> + // skip bag/list, it's a repeated element. + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_node_field(t_schemas, curr_pos + 2, list_child)); + } else { + // required field, produce the list of struct + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 0) { + // required two level list, for compatibility reason. + set_child_node_level(list_field, list_field->definition_level); + parse_physical_field(second_level, false, list_child); + _next_schema_pos = curr_pos + 2; + } + + list_field->name = first_level.name; + list_field->lower_case_name = to_lower(first_level.name); + list_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + list_field->data_type = + std::make_shared(make_nullable(list_field->children[0].data_type)); + if (is_optional) { + list_field->data_type = make_nullable(list_field->data_type); + } + list_field->field_id = first_level.__isset.field_id ? first_level.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element) { + // the map definition in parquet: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + // Map value can be optional, the map without values is a SET + if (curr_pos + 2 >= t_schemas.size()) { + return Status::InvalidArgument("Map element should have at least three levels"); + } + auto& map_schema = t_schemas[curr_pos]; + if (map_schema.num_children != 1) { + return Status::InvalidArgument( + "Map element should have only one child(name='map', type='MAP_KEY_VALUE')"); + } + if (is_repeated_node(map_schema) && !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("Map element can't be a repeated schema"); + } + auto& map_key_value = t_schemas[curr_pos + 1]; + if (!is_group_node(map_key_value) || !is_repeated_node(map_key_value)) { + return Status::InvalidArgument( + "the second level in map must be a repeated group(key and value)"); + } + auto& map_key = t_schemas[curr_pos + 2]; + if (!is_required_node(map_key)) { + LOG(WARNING) << "Filed " << map_schema.name << " is map type, but with nullable key column"; + } + + if (map_key_value.num_children == 1) { + // The map with three levels is a SET + return parse_list_field(t_schemas, curr_pos, map_field, + repeated_node_is_enclosing_list_element); + } + if (map_key_value.num_children != 2) { + // A standard map should have four levels + return Status::InvalidArgument( + "the second level in map(MAP_KEY_VALUE) should have two children"); + } + // standard map + bool is_optional = is_optional_node(map_schema); + if (is_optional) { + map_field->definition_level++; + } + map_field->repetition_level++; + map_field->definition_level++; + + // Directly create key and value children instead of intermediate key_value node + map_field->children.resize(2); + // map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level` + set_child_node_level(map_field, map_field->definition_level); + + auto key_field = &map_field->children[0]; + auto value_field = &map_field->children[1]; + + // Parse key and value fields directly from the key_value group's children + _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field)); + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field)); + + map_field->name = map_schema.name; + map_field->lower_case_name = to_lower(map_field->name); + map_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + map_field->data_type = std::make_shared(make_nullable(key_field->data_type), + make_nullable(value_field->data_type)); + if (is_optional) { + map_field->data_type = make_nullable(map_field->data_type); + } + map_field->field_id = map_schema.__isset.field_id ? map_schema.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_struct_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* struct_field) { + // the nested column in parquet, parse group to struct. + auto& struct_schema = t_schemas[curr_pos]; + bool is_optional = is_optional_node(struct_schema); + if (is_optional) { + struct_field->definition_level++; + } + auto num_children = struct_schema.num_children; + struct_field->children.resize(num_children); + set_child_node_level(struct_field, struct_field->repeated_parent_def_level); + _next_schema_pos = curr_pos + 1; + for (int i = 0; i < num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i])); + } + struct_field->name = struct_schema.name; + struct_field->lower_case_name = to_lower(struct_field->name); + struct_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + + struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1; + DataTypes res_data_types; + std::vector names; + for (int i = 0; i < num_children; ++i) { + res_data_types.push_back(make_nullable(struct_field->children[i].data_type)); + names.push_back(struct_field->children[i].name); + } + struct_field->data_type = std::make_shared(res_data_types, names); + if (is_optional) { + struct_field->data_type = make_nullable(struct_field->data_type); + } + return Status::OK(); +} + +int NativeFieldDescriptor::get_column_index(const std::string& column) const { + for (int32_t i = 0; i < _fields.size(); i++) { + if (_fields[i].name == column) { + return i; + } + } + return -1; +} + +NativeFieldSchema* NativeFieldDescriptor::get_column(const std::string& name) const { + auto it = _name_to_field.find(name); + if (it != _name_to_field.end()) { + return it->second; + } + throw Exception(Status::InternalError("Name {} not found in NativeFieldDescriptor!", name)); + return nullptr; +} + +void NativeFieldDescriptor::get_column_names(std::unordered_set* names) const { + names->clear(); + for (const NativeFieldSchema& f : _fields) { + names->emplace(f.name); + } +} + +std::string NativeFieldDescriptor::debug_string() const { + std::stringstream ss; + ss << "fields=["; + for (int i = 0; i < _fields.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << _fields[i].debug_string(); + } + ss << "]"; + return ss.str(); +} + +void NativeFieldDescriptor::assign_ids() { + uint64_t next_id = 1; + for (auto& field : _fields) { + field.assign_ids(next_id); + } +} + +const NativeFieldSchema* NativeFieldDescriptor::find_column_by_id(uint64_t column_id) const { + for (const auto& field : _fields) { + if (auto result = field.find_column_by_id(column_id)) { + return result; + } + } + return nullptr; +} + +void NativeFieldSchema::assign_ids(uint64_t& next_id) { + column_id = next_id++; + + for (auto& child : children) { + child.assign_ids(next_id); + } + + max_column_id = next_id - 1; +} + +const NativeFieldSchema* NativeFieldSchema::find_column_by_id(uint64_t target_id) const { + if (column_id == target_id) { + return this; + } + + for (const auto& child : children) { + if (auto result = child.find_column_by_id(target_id)) { + return result; + } + } + + return nullptr; +} + +uint64_t NativeFieldSchema::get_column_id() const { + return column_id; +} + +void NativeFieldSchema::set_column_id(uint64_t id) { + column_id = id; +} + +uint64_t NativeFieldSchema::get_max_column_id() const { + return max_column_id; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h new file mode 100644 index 00000000000000..dfb669559fc71a --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nothing.h" +#include "util/slice.h" + +namespace doris::format::parquet { + +// Constant for unassigned column IDs +constexpr uint64_t NATIVE_UNASSIGNED_COLUMN_ID = UINT64_MAX; +constexpr size_t MAX_NATIVE_SCHEMA_DEPTH = 100; + +struct NativeFieldSchema { + std::string name; + std::string lower_case_name; // for hms column name case insensitive match + // the referenced parquet schema element + tparquet::SchemaElement parquet_schema; + + // Used to identify whether this field is a nested field. + DataTypePtr data_type; + // Schema construction keeps a physical fallback so unprojected columns and metadata-only + // queries remain readable, while projection validation reports the original logical failure. + std::string unsupported_reason; + + // Only valid when this field is a leaf node + tparquet::Type::type physical_type; + // The index order in NativeFieldDescriptor._physical_fields + int physical_column_index = -1; + int16_t definition_level = 0; + int16_t repetition_level = 0; + int16_t repeated_parent_def_level = 0; + std::vector children; + + //For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128. + bool is_type_compatibility = false; + + NativeFieldSchema() + : data_type(std::make_shared()), + column_id(NATIVE_UNASSIGNED_COLUMN_ID) {} + ~NativeFieldSchema() = default; + NativeFieldSchema(const NativeFieldSchema& fieldSchema) = default; + std::string debug_string() const; + + int32_t field_id = -1; + uint64_t column_id = NATIVE_UNASSIGNED_COLUMN_ID; + uint64_t max_column_id = 0; // Maximum column ID for this field and its children + + // Column ID assignment and lookup methods + void assign_ids(uint64_t& next_id); + const NativeFieldSchema* find_column_by_id(uint64_t target_id) const; + uint64_t get_column_id() const; + void set_column_id(uint64_t id); + uint64_t get_max_column_id() const; +}; + +// V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path. +class NativeFieldDescriptor { +private: + // Only the schema elements at the first level + std::vector _fields; + // The leaf node of schema elements + std::vector _physical_fields; + // Name to _fields, not all schema elements + std::unordered_map _name_to_field; + // Used in from_thrift, marking the next schema position that should be parsed + size_t _next_schema_pos; + // useful for parse_node_field to decide whether to convert byte_array to VARBINARY type + bool _enable_mapping_varbinary = false; + bool _enable_mapping_timestamp_tz = false; + +private: + void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, + NativeFieldSchema* physical_field); + + Status parse_list_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_map_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_struct_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* struct_field); + + Status parse_group_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field); + + Status parse_node_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field); + + std::pair convert_to_doris_type(tparquet::LogicalType logicalType, + bool nullable); + std::pair convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable); + std::pair get_doris_type(const tparquet::SchemaElement& physical_schema, + bool nullable); + +public: + NativeFieldDescriptor() = default; + ~NativeFieldDescriptor() = default; + + /** + * Parse NativeFieldDescriptor from parquet thrift FileMetaData. + * @param t_schemas list of schema elements + */ + Status parse_from_thrift(const std::vector& t_schemas); + + int get_column_index(const std::string& column) const; + + /** + * Get the column(the first level schema element, maybe nested field) by index. + * @param index Column index in _fields + */ + const NativeFieldSchema* get_column(size_t index) const { return &_fields[index]; } + + /** + * Get the column(the first level schema element, maybe nested field) by name. + * @param name Column name + * @return NativeFieldSchema or nullptr if not exists + */ + NativeFieldSchema* get_column(const std::string& name) const; + + void get_column_names(std::unordered_set* names) const; + + std::string debug_string() const; + + int32_t size() const { return cast_set(_fields.size()); } + + size_t physical_fields_size() const { return _physical_fields.size(); } + + const NativeFieldSchema* get_physical_field(size_t index) const { + return _physical_fields[index]; + } + + const std::vector& get_fields_schema() const { return _fields; } + + /** + * Assign stable column IDs to schema fields. + * + * This uses an ORC-compatible encoding so that the results of + * create_column_ids() are consistent across formats. IDs start from 1 + * and are assigned in a pre-order traversal (parent before children). + * After calling this, each NativeFieldSchema will have column_id and + * max_column_id populated. + */ + void assign_ids(); + + const NativeFieldSchema* find_column_by_id(uint64_t column_id) const; + void set_enable_mapping_varbinary(bool enable) { _enable_mapping_varbinary = enable; } + void set_enable_mapping_timestamp_tz(bool enable) { _enable_mapping_timestamp_tz = enable; } +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.cpp b/be/src/format_v2/parquet/native_schema_node.cpp new file mode 100644 index 00000000000000..052df93e4b4143 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.cpp @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_node.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +void NativeStructSchemaNode::add_child(std::string table_name, std::string file_name, + std::shared_ptr node) { + _children.emplace(std::move(table_name), Child {std::move(file_name), std::move(node)}); +} + +void NativeStructSchemaNode::add_missing_child(std::string table_name) { + _children.emplace(std::move(table_name), Child {}); +} + +std::shared_ptr NativeStructSchemaNode::child( + const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? nullptr : it->second.node; +} + +std::string NativeStructSchemaNode::file_child_name(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? std::string {} : it->second.file_name; +} + +bool NativeStructSchemaNode::has_child(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it != _children.end() && it->second.node != nullptr; +} + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result) { + if (projected_type == nullptr || result == nullptr) { + return Status::InvalidArgument("Native Parquet schema mapping input is null"); + } + const auto type = remove_nullable(projected_type); + switch (type->get_primitive_type()) { + case TYPE_STRUCT: { + if (file_schema.kind != ParquetColumnSchemaKind::STRUCT) { + return Status::Corruption("Parquet column {} is not a STRUCT", file_schema.name); + } + const auto* struct_type = assert_cast(type.get()); + auto node = std::make_shared(); + for (size_t i = 0; i < struct_type->get_elements().size(); ++i) { + const auto& table_name = struct_type->get_element_name(i); + const ParquetColumnSchema* file_child = nullptr; + for (const auto& child : file_schema.children) { + if (child->name == table_name) { + file_child = child.get(); + break; + } + } + if (file_child == nullptr) { + for (const auto& child : file_schema.children) { + if (to_lower(child->name) != to_lower(table_name)) { + continue; + } + if (UNLIKELY(file_child != nullptr)) { + // Exact writer identity is authoritative; normalized fallback is only safe + // when the requested field has one physical candidate. + return Status::Corruption( + "Parquet STRUCT {} has ambiguous case-insensitive child name {}", + file_schema.name, table_name); + } + file_child = child.get(); + } + } + if (file_child == nullptr) { + node->add_missing_child(table_name); + continue; + } + std::shared_ptr child_node; + RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *file_child, + &child_node)); + node->add_child(table_name, file_child->name, std::move(child_node)); + } + *result = std::move(node); + return Status::OK(); + } + case TYPE_ARRAY: { + if (file_schema.kind != ParquetColumnSchemaKind::LIST || file_schema.children.size() != 1) { + return Status::Corruption("Parquet column {} is not an ARRAY", file_schema.name); + } + const auto* array_type = assert_cast(type.get()); + std::shared_ptr element; + RETURN_IF_ERROR(build_native_schema_node(array_type->get_nested_type(), + *file_schema.children[0], &element)); + *result = std::make_shared(std::move(element)); + return Status::OK(); + } + case TYPE_MAP: { + if (file_schema.kind != ParquetColumnSchemaKind::MAP || file_schema.children.size() != 2) { + return Status::Corruption("Parquet column {} is not a MAP", file_schema.name); + } + const auto* map_type = assert_cast(type.get()); + std::shared_ptr key; + std::shared_ptr value; + RETURN_IF_ERROR( + build_native_schema_node(map_type->get_key_type(), *file_schema.children[0], &key)); + RETURN_IF_ERROR(build_native_schema_node(map_type->get_value_type(), + *file_schema.children[1], &value)); + *result = std::make_shared(std::move(key), std::move(value)); + return Status::OK(); + } + default: + if (file_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { + return Status::Corruption("Parquet column {} is not a scalar", file_schema.name); + } + *result = std::make_shared(); + return Status::OK(); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.h b/be/src/format_v2/parquet/native_schema_node.h new file mode 100644 index 00000000000000..25c42ad4b2c6d4 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" + +namespace doris::format::parquet { + +struct ParquetColumnSchema; + +// V2-owned semantic mapping consumed by the native decoder. It deliberately contains no V1 +// reader/schema-helper state, so FileScannerV2 can build the mapping from its native metadata tree. +class NativeSchemaNode { +public: + virtual ~NativeSchemaNode() = default; + + virtual std::shared_ptr child(const std::string&) const { return nullptr; } + virtual std::string file_child_name(const std::string&) const { return {}; } + virtual bool has_child(const std::string&) const { return false; } + virtual std::shared_ptr element() const { return nullptr; } + virtual std::shared_ptr key() const { return nullptr; } + virtual std::shared_ptr value() const { return nullptr; } +}; + +class NativeScalarSchemaNode final : public NativeSchemaNode {}; + +class NativeStructSchemaNode final : public NativeSchemaNode { +public: + void add_child(std::string table_name, std::string file_name, + std::shared_ptr node); + void add_missing_child(std::string table_name); + + std::shared_ptr child(const std::string& table_name) const override; + std::string file_child_name(const std::string& table_name) const override; + bool has_child(const std::string& table_name) const override; + +private: + struct Child { + std::string file_name; + std::shared_ptr node; + }; + std::map _children; +}; + +class NativeArraySchemaNode final : public NativeSchemaNode { +public: + explicit NativeArraySchemaNode(std::shared_ptr element) + : _element(std::move(element)) {} + std::shared_ptr element() const override { return _element; } + +private: + std::shared_ptr _element; +}; + +class NativeMapSchemaNode final : public NativeSchemaNode { +public: + NativeMapSchemaNode(std::shared_ptr key, + std::shared_ptr value) + : _key(std::move(key)), _value(std::move(value)) {} + std::shared_ptr key() const override { return _key; } + std::shared_ptr value() const override { return _value; } + +private: + std::shared_ptr _key; + std::shared_ptr _value; +}; + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index b42d47987a54cb..7e541b57da2780 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -15,476 +15,215 @@ #include "format_v2/parquet/parquet_column_schema.h" -#include - #include #include #include #include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_type.h" namespace doris::format::parquet { namespace { -struct SchemaBuildContext { - int32_t local_id = -1; // child ordinal in the parent node - int16_t definition_level = 0; // accumulated optional/repeated level count - int16_t repetition_level = 0; // accumulated repeated level count - int16_t nullable_definition_level = 0; // definition level of the nearest optional node - int16_t repeated_repetition_level = 0; // repetition level of the nearest repeated node - int16_t repeated_ancestor_definition_level = 0; // definition level of the nearest repeated node -}; - -enum class SchemaBuildMode { - // Normal recursive schema build. Bare repeated fields are exposed as Doris ARRAY for - // protobuf/legacy Parquet compatibility, while repeated LIST/MAP annotated groups are rejected - // because Parquet LIST/MAP outer groups are not allowed to be repeated at a top-level or struct - // field boundary. - NORMAL, - // Build the current repeated node as the already-selected element of an enclosing LIST. This - // is the compatibility path for Arrow/parquet-format legacy two-level LIST encodings where the - // repeated node itself is the array element instead of a wrapper that should be stripped. - REPEATED_NODE_AS_LIST_ELEMENT, - // Build the current repeated group as a STRUCT element of an enclosing LIST, ignoring LIST/MAP - // annotations on the repeated group itself. This keeps compatibility with the old Doris - // Parquet schema parser for Hive/legacy wrappers named "array" or "_tuple". - REPEATED_NODE_AS_STRUCT_ELEMENT, -}; - -// Result of applying Parquet LIST backward compatibility rules to the single repeated child of a -// LIST-annotated group. The repeated child can either be a physical wrapper whose only child is the -// element, or the element node itself. -struct ListElementResolution { - // Parquet node that should be exposed as Doris ARRAY element. - const ::parquet::schema::Node* element_node = nullptr; - // Level state after consuming the LIST repeated child. The parent ARRAY schema keeps this state - // to materialize offsets, empty arrays and null arrays. - SchemaBuildContext repeated_context; - // Level state used to build element_node. This equals repeated_context when the repeated child - // itself is the element, and includes the wrapper's only child when standard 3-level LIST - // encoding is stripped. - SchemaBuildContext element_context; - // Build mode for element_node. Non-NORMAL modes mean element_node is the repeated child itself, - // and the repeated level must not be interpreted as a second unrelated array at the same - // boundary. - SchemaBuildMode element_build_mode = SchemaBuildMode::NORMAL; -}; - -// Resolved repeated entry group of a MAP-annotated group. The entry wrapper is a physical Parquet -// encoding detail; Doris folds it into the parent MAP schema and exposes only direct [key, value] -// children. -struct MapEntryResolution { - const ::parquet::schema::GroupNode* entry_group = nullptr; - // Level state after consuming the repeated entry group. The parent MAP schema keeps this state - // to materialize offsets, empty maps and null maps. - SchemaBuildContext entry_context; -}; - -bool is_list_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::LIST || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_list()); -} - -bool is_map_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::MAP || - node.converted_type() == ::parquet::ConvertedType::MAP_KEY_VALUE || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_map()); -} - -bool has_logical_annotation(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return (node.converted_type() != ::parquet::ConvertedType::NONE && - node.converted_type() != ::parquet::ConvertedType::UNDEFINED) || - (logical_type != nullptr && logical_type->is_valid() && !logical_type->is_none()); -} - -bool has_structural_list_name(const std::string& list_name, const std::string& repeated_name) { - return repeated_name == "array" || repeated_name == list_name + "_tuple"; -} - -bool should_build_repeated_field_as_list(const ::parquet::schema::Node& node) { - return node.is_repeated() && !is_list_node(node) && !is_map_node(node); -} - -DataTypePtr nullable_if_needed(DataTypePtr type, const ::parquet::schema::Node& node) { - return node.is_optional() ? make_nullable(type) : type; -} - -void inherit_common_schema_state(const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - column_schema->local_id = context.local_id; - column_schema->parquet_field_id = node.field_id(); - column_schema->name = node.name(); - column_schema->max_definition_level = context.definition_level; - column_schema->max_repetition_level = context.repetition_level; - column_schema->nullable_definition_level = context.nullable_definition_level; - column_schema->definition_level = context.definition_level; - column_schema->repetition_level = context.repetition_level; - column_schema->repeated_ancestor_definition_level = context.repeated_ancestor_definition_level; - column_schema->repeated_repetition_level = context.repeated_repetition_level; -} - -SchemaBuildContext child_context(const SchemaBuildContext& parent, - const ::parquet::schema::Node& child_node, int32_t child_idx) { - SchemaBuildContext result = parent; - result.local_id = child_idx; - if (child_node.repetition() == ::parquet::Repetition::OPTIONAL) { - result.definition_level++; - result.nullable_definition_level = result.definition_level; +ParquetTimeUnit native_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; } - if (child_node.is_repeated()) { - result.repetition_level++; - result.definition_level++; - result.repeated_repetition_level = result.repetition_level; - result.repeated_ancestor_definition_level = result.definition_level; + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; } - return result; -} - -void propagate_child_levels(ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - for (const auto& child : column_schema->children) { - column_schema->max_definition_level = - std::max(column_schema->max_definition_level, child->max_definition_level); - column_schema->max_repetition_level = - std::max(column_schema->max_repetition_level, child->max_repetition_level); + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; } + return ParquetTimeUnit::UNKNOWN; } -// Mirrors Arrow's ResolveList() compatibility rules, but only decides which Parquet node is the -// logical LIST element. The caller still builds Doris' semantic LIST->[element] schema tree. -// Important cases: -// - repeated primitive: the primitive itself is the element (legacy two-level LIST). -// - repeated group with multiple children: the group itself is a STRUCT element. -// - repeated group named "array" or "_tuple": the group itself is a STRUCT element per -// Parquet backward compatibility rules, even when it has one child or its own logical annotation. -// This also keeps v2 file-local schema aligned with Doris' old schema parser used by HDFS TVF. -// - other repeated group with a logical annotation, or whose only child is repeated: the group -// itself is the element. This preserves nested LIST/MAP and repeated fields inside struct -// elements. -// - otherwise, strip the one-child repeated wrapper as standard three-level LIST encoding. -Status resolve_list_element_node(const ::parquet::schema::GroupNode& list_group, - const SchemaBuildContext& list_context, - ListElementResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (list_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - const auto& repeated_node = *list_group.field(0); - if (!repeated_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - result->repeated_context = child_context(list_context, repeated_node, 0); - if (repeated_node.is_primitive()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& repeated_group = static_cast(repeated_node); - if (repeated_group.field_count() == 0) { - return Status::NotSupported("Unsupported parquet LIST element layout for column {}", - list_group.name()); - } - const bool repeated_group_has_logical_annotation = has_logical_annotation(repeated_group); - if (repeated_group.field_count() > 1 || - has_structural_list_name(list_group.name(), repeated_group.name())) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT; - return Status::OK(); - } - if (repeated_group_has_logical_annotation) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& only_child = *repeated_group.field(0); - if (only_child.is_repeated()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); +ParquetExtraTypeInfo native_time_extra(ParquetTimeUnit unit) { + switch (unit) { + case ParquetTimeUnit::MILLIS: + return ParquetExtraTypeInfo::UNIT_MS; + case ParquetTimeUnit::MICROS: + return ParquetExtraTypeInfo::UNIT_MICROS; + case ParquetTimeUnit::NANOS: + return ParquetExtraTypeInfo::UNIT_NS; + case ParquetTimeUnit::UNKNOWN: + default: + return ParquetExtraTypeInfo::NONE; } - - result->element_node = &only_child; - result->element_context = child_context(result->repeated_context, only_child, 0); - return Status::OK(); -} - -// Resolves the repeated entry group of a MAP/MAP_KEY_VALUE node. Unlike LIST, MAP has no supported -// two-level form in this reader: Doris requires a repeated group with exactly key and value -// children, then folds that physical entry group out of ParquetColumnSchema. Some external writers -// emit optional MAP keys even though standard Parquet MAP keys are required; keep the key's -// definition levels and expose it as nullable for compatibility with the old reader. -Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, - const SchemaBuildContext& map_context, MapEntryResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (map_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - const auto& entry_node = *map_group.field(0); - if (!entry_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - if (entry_node.is_primitive()) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - const auto& entry_group = static_cast(entry_node); - if (entry_group.field_count() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - // The Parquet logical MAP spec requires key to be REQUIRED. Some legacy/Hive-written files - // still mark the key field OPTIONAL even when all actual keys are non-null, for example: - // optional group t_map_varchar (MAP) { - // repeated group key_value { - // optional binary key (STRING); - // optional binary value (STRING); - // } - // } - // Accept that schema here so compatible files can be read. MapColumnReader validates the - // materialized key column and rejects data that really contains null map keys. - result->entry_group = &entry_group; - result->entry_context = child_context(map_context, entry_node, 0); - return Status::OK(); -} - -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode); - -// Builds a semantic ARRAY schema for a bare repeated field. Arrow handles this in -// NodeToSchemaField()/GroupToSchemaField(); Doris needs the same compatibility behavior because -// protobuf and old parquet writers often encode repeated fields without a LIST annotation. -// Example: -// optional group event { -// repeated group links { -// optional binary url (UTF8); -// optional int32 rank; -// } -// } -// Doris exposes event.links as ARRAY>, not STRUCT. This keeps v2's -// file-local schema aligned with the old schema parser used by HDFS TVF schema fetching. -// When the repeated field appears inside an already resolved LIST element, only the nested repeated -// child should be wrapped: -// optional group a (LIST) { -// repeated group element { -// repeated int32 items; -// } -// } -// The outer LIST element is the repeated "element" group, and its repeated "items" child should be -// represented as a field of type ARRAY inside the struct element. -Status build_repeated_field_as_list_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& repeated_node, - const SchemaBuildContext& repeated_context, - std::unique_ptr* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - auto list_schema = std::make_unique(); - inherit_common_schema_state(repeated_node, repeated_context, list_schema.get()); - list_schema->kind = ParquetColumnSchemaKind::LIST; - list_schema->definition_level = repeated_context.definition_level; - list_schema->repetition_level = repeated_context.repetition_level; - list_schema->repeated_repetition_level = repeated_context.repeated_repetition_level; - - std::unique_ptr element_child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, repeated_node, repeated_context, - &element_child, - SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT)); - element_child->name = "element"; - list_schema->type = std::make_shared(element_child->type); - list_schema->children.push_back(std::move(element_child)); - propagate_child_levels(list_schema.get()); - *result = std::move(list_schema); - return Status::OK(); } -// Recursively builds ParquetColumnSchema for the given schema node and its children in Parquet -// file's metadata. NORMAL mode exposes bare repeated fields as ARRAY for legacy compatibility. -// REPEATED_NODE_AS_LIST_ELEMENT mode means the current repeated node was already selected as an -// enclosing LIST element, so only its nested bare repeated children should be wrapped. -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (mode == SchemaBuildMode::NORMAL && should_build_repeated_field_as_list(node)) { - return build_repeated_field_as_list_schema(schema, node, context, result); - } - - auto column_schema = std::make_unique(); - inherit_common_schema_state(node, context, column_schema.get()); - - if (node.is_primitive()) { - const int leaf_column_id = schema.ColumnIndex(node); - if (leaf_column_id < 0) { - return Status::InvalidArgument("Cannot find leaf column id for parquet column {}", - node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::PRIMITIVE; - column_schema->leaf_column_id = leaf_column_id; - column_schema->descriptor = schema.Column(leaf_column_id); - if (column_schema->descriptor != nullptr) { - column_schema->max_definition_level = column_schema->descriptor->max_definition_level(); - column_schema->max_repetition_level = column_schema->descriptor->max_repetition_level(); - } - column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); - column_schema->type = column_schema->type_descriptor.doris_type; - if (column_schema->type == nullptr) { - if (!column_schema->type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", node.name(), - column_schema->type_descriptor.unsupported_reason); +void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDescriptor* result) { + DORIS_CHECK(result != nullptr); + const auto& schema = field.parquet_schema; + result->doris_type = field.data_type; + result->unsupported_reason = field.unsupported_reason; + result->physical_type = static_cast(field.physical_type); + result->fixed_length = schema.__isset.type_length ? schema.type_length : -1; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.DECIMAL) { + result->is_decimal = true; + result->decimal_precision = logical.DECIMAL.precision; + result->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.INTEGER) { + result->integer_bit_width = logical.INTEGER.bitWidth; + result->is_unsigned_integer = !logical.INTEGER.isSigned; + } else if (logical.__isset.TIME) { + result->time_unit = native_time_unit(logical.TIME.unit); + result->extra_type_info = native_time_extra(result->time_unit); + if (logical.TIME.isAdjustedToUTC) { + result->unsupported_reason = + "Parquet TIME with isAdjustedToUTC=true is not supported"; } - return Status::NotSupported("Unsupported parquet column type for column {}", - node.name()); + } else if (logical.__isset.TIMESTAMP) { + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + result->time_unit = native_time_unit(logical.TIMESTAMP.unit); + result->extra_type_info = native_time_extra(result->time_unit); + } else if (logical.__isset.FLOAT16) { + result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; } - column_schema->type = node.is_optional() - ? make_nullable(remove_nullable(column_schema->type)) - : remove_nullable(column_schema->type); - *result = std::move(column_schema); - return Status::OK(); - } - - const auto& group = static_cast(node); - if (is_list_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet LIST column {}", node.name()); + } else if (schema.__isset.converted_type) { + switch (schema.converted_type) { + case tparquet::ConvertedType::DECIMAL: + result->is_decimal = true; + result->decimal_precision = schema.__isset.precision ? schema.precision : -1; + result->decimal_scale = schema.__isset.scale ? schema.scale : -1; + break; + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::UINT_8: + result->integer_bit_width = 8; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_8; + break; + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::UINT_16: + result->integer_bit_width = 16; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_16; + break; + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::UINT_32: + result->integer_bit_width = 32; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_32; + break; + case tparquet::ConvertedType::INT_64: + case tparquet::ConvertedType::UINT_64: + result->integer_bit_width = 64; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_64; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS + ? ParquetTimeUnit::MILLIS + : ParquetTimeUnit::MICROS; + result->extra_type_info = native_time_extra(result->time_unit); + break; + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::TIME_MICROS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + break; + default: + break; } - column_schema->kind = ParquetColumnSchemaKind::LIST; - ListElementResolution list_element; - RETURN_IF_ERROR(resolve_list_element_node(group, context, &list_element)); - column_schema->definition_level = list_element.repeated_context.definition_level; - column_schema->repetition_level = list_element.repeated_context.repetition_level; - column_schema->repeated_repetition_level = - list_element.repeated_context.repeated_repetition_level; - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, *list_element.element_node, - list_element.element_context, &child, - list_element.element_build_mode)); - child->name = "element"; - column_schema->type = - nullable_if_needed(std::make_shared(child->type), node); - column_schema->children.push_back(std::move(child)); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } - if (is_map_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet MAP column {}", node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::MAP; - MapEntryResolution map_entry; - RETURN_IF_ERROR(resolve_map_entry_group(group, context, &map_entry)); - column_schema->definition_level = map_entry.entry_context.definition_level; - column_schema->repetition_level = map_entry.entry_context.repetition_level; - column_schema->repeated_repetition_level = - map_entry.entry_context.repeated_repetition_level; - for (int child_idx = 0; child_idx < map_entry.entry_group->field_count(); ++child_idx) { - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode( - schema, *map_entry.entry_group->field(child_idx), - child_context(map_entry.entry_context, *map_entry.entry_group->field(child_idx), - child_idx), - &child, SchemaBuildMode::NORMAL)); - child->name = child_idx == 0 ? "key" : "value"; - column_schema->children.push_back(std::move(child)); + if (result->is_decimal) { + switch (result->physical_type) { + case tparquet::Type::INT32: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; + break; + case tparquet::Type::INT64: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; + break; + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; + break; + default: + break; } - if (column_schema->children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - node.name()); - } - auto key_type = make_nullable(column_schema->children[0]->type); - auto value_type = make_nullable(column_schema->children[1]->type); - column_schema->type = - nullable_if_needed(std::make_shared(key_type, value_type), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); - } + } else if (result->physical_type == tparquet::Type::INT96) { + result->is_timestamp = true; + result->extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; + } + result->is_string_like = !result->is_decimal && + result->extra_type_info != ParquetExtraTypeInfo::FLOAT16 && + (result->physical_type == tparquet::Type::BYTE_ARRAY || + result->physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY); +} - column_schema->kind = ParquetColumnSchemaKind::STRUCT; - DataTypes child_types; - Strings child_names; - child_types.reserve(group.field_count()); - child_names.reserve(group.field_count()); - for (int child_idx = 0; child_idx < group.field_count(); ++child_idx) { - const auto& child_node = *group.field(child_idx); - std::unique_ptr child; - const auto child_ctx = child_context(context, child_node, child_idx); - if (should_build_repeated_field_as_list(child_node)) { - RETURN_IF_ERROR( - build_repeated_field_as_list_schema(schema, child_node, child_ctx, &child)); - } else { - RETURN_IF_ERROR(build_node_schema_with_mode(schema, child_node, child_ctx, &child, - SchemaBuildMode::NORMAL)); - } - child_types.push_back(make_nullable(child->type)); - child_names.push_back(child->name); - column_schema->children.push_back(std::move(child)); +void propagate_native_max_levels(ParquetColumnSchema* schema) { + DORIS_CHECK(schema != nullptr); + for (const auto& child : schema->children) { + DORIS_CHECK(child != nullptr); + propagate_native_max_levels(child.get()); + schema->max_definition_level = + std::max(schema->max_definition_level, child->max_definition_level); + schema->max_repetition_level = + std::max(schema->max_repetition_level, child->max_repetition_level); } - column_schema->type = - nullable_if_needed(std::make_shared(child_types, child_names), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } -Status build_node_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, const SchemaBuildContext& context, - std::unique_ptr* result) { - return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); +std::unique_ptr build_native_node_schema(const NativeFieldSchema& field, + int32_t local_id) { + auto result = std::make_unique(); + result->local_id = local_id; + result->parquet_field_id = field.field_id; + result->name = field.name; + result->type = field.data_type; + result->definition_level = field.definition_level; + result->repetition_level = field.repetition_level; + result->max_definition_level = field.definition_level; + result->max_repetition_level = field.repetition_level; + result->nullable_definition_level = field.data_type != nullptr && field.data_type->is_nullable() + ? field.definition_level - field.repetition_level + : 0; + result->repeated_ancestor_definition_level = field.repeated_parent_def_level; + result->repeated_repetition_level = field.repetition_level; + + const auto primitive_type = remove_nullable(field.data_type)->get_primitive_type(); + if (field.children.empty()) { + result->kind = ParquetColumnSchemaKind::PRIMITIVE; + result->leaf_column_id = field.physical_column_index; + fill_native_type_descriptor(field, &result->type_descriptor); + return result; + } + if (primitive_type == TYPE_ARRAY) { + result->kind = ParquetColumnSchemaKind::LIST; + } else if (primitive_type == TYPE_MAP) { + result->kind = ParquetColumnSchemaKind::MAP; + } else { + result->kind = ParquetColumnSchemaKind::STRUCT; + } + result->children.reserve(field.children.size()); + for (size_t child_idx = 0; child_idx < field.children.size(); ++child_idx) { + result->children.push_back( + build_native_node_schema(field.children[child_idx], cast_set(child_idx))); + } + propagate_native_max_levels(result.get()); + return result; } } // namespace -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields) { if (fields == nullptr) { return Status::InvalidArgument("fields is null"); } fields->clear(); - const auto* root = schema.group_node(); - if (root == nullptr) { - return Status::InvalidArgument("Parquet schema root is null"); - } - fields->reserve(root->field_count()); - for (int field_idx = 0; field_idx < root->field_count(); ++field_idx) { - std::unique_ptr field; - SchemaBuildContext context; - RETURN_IF_ERROR(build_node_schema( - schema, *root->field(field_idx), - child_context(context, *root->field(field_idx), field_idx), &field)); - fields->push_back(std::move(field)); + const auto& native_fields = schema.get_fields_schema(); + fields->reserve(native_fields.size()); + for (size_t field_idx = 0; field_idx < native_fields.size(); ++field_idx) { + // Unsupported logical leaves stay in the file schema so request-level validation can + // ignore unprojected fields and COUNT(*) placeholders without weakening real projections. + // The scan projection and native readers must share one tree; rebuilding wrappers through + // Arrow changes legacy LIST/STRUCT boundaries and makes valid nested values look absent. + fields->push_back( + build_native_node_schema(native_fields[field_idx], cast_set(field_idx))); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index 1fb7262aabde6f..697bcd498382b1 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -23,18 +23,15 @@ #include "core/data_type/data_type.h" #include "format_v2/parquet/parquet_type.h" -namespace parquet { -class ColumnDescriptor; -class SchemaDescriptor; -} // namespace parquet - namespace doris::format::parquet { +class NativeFieldDescriptor; + enum class ParquetColumnSchemaKind { - PRIMITIVE, // primitive leaf -> ScalarColumnReader - STRUCT, // struct -> StructColumnReader - LIST, // array -> ListColumnReader - MAP, // map -> MapColumnReader + PRIMITIVE, // physical primitive leaf + STRUCT, // Parquet group with STRUCT semantics + LIST, // Parquet group with LIST semantics + MAP, // Parquet group with MAP semantics }; // ============================================================================ @@ -55,8 +52,6 @@ struct ParquetColumnSchema { ParquetColumnSchemaKind kind = ParquetColumnSchemaKind::PRIMITIVE; - const ::parquet::ColumnDescriptor* descriptor = nullptr; - // ======== Dremel Levels ======== int16_t max_definition_level = 0; @@ -74,7 +69,7 @@ struct ParquetColumnSchema { std::vector> children {}; }; -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 52151c7942af2b..c178e491b8ac21 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -15,86 +15,139 @@ #include "format_v2/parquet/parquet_file_context.h" -#include -#include #include -#include -#include #include #include #include #include -#include #include -#include #include +#include "common/cast_set.h" #include "common/check.h" #include "common/config.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" -#include "storage/cache/page_cache.h" +#include "runtime/exec_env.h" +#include "util/coding.h" #include "util/slice.h" +#include "util/thrift_util.h" +#include "util/time.h" namespace doris::format::parquet { +constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; + +NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) + : _metadata(std::move(metadata)), _parsed_size(parsed_size) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); +} + +NativeParquetMetadata::~NativeParquetMetadata() { + ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size()); +} + +Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); + _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); + RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); + // Native readers address projected leaves by stable DFS IDs. Assign them only on the private + // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. + _schema.assign_ids(); + for (size_t row_group_idx = 0; row_group_idx < _metadata.row_groups.size(); ++row_group_idx) { + const auto& row_group = _metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.columns.size() != _schema.physical_fields_size()) { + // All v2 planners index chunks by the native DFS leaf order, so validate cardinality + // once before any projection, prefetch, or decoder can perform indexed access. + return Status::Corruption( + "Parquet row group {} has {} column chunks but schema has {} physical fields", + row_group_idx, row_group.columns.size(), _schema.physical_fields_size()); + } + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + const auto* physical_field = _schema.get_physical_field(column_idx); + if (chunk.meta_data.type != physical_field->physical_type) { + return Status::Corruption( + "Parquet row group {} column {} physical type {} does not match schema {}", + row_group_idx, column_idx, tparquet::to_string(chunk.meta_data.type), + tparquet::to_string(physical_field->physical_type)); + } + if (chunk.meta_data.num_values < 0) { + return Status::Corruption( + "Parquet row group {} column {} has negative value count {}", row_group_idx, + column_idx, chunk.meta_data.num_values); + } + if (physical_field->repetition_level == 0 && + chunk.meta_data.num_values != row_group.num_rows) { + // A flat leaf contributes one definition-level slot per row, including NULLs. + // Reject a mismatched footer before metadata COUNT can treat it as cardinality. + return Status::Corruption( + "Parquet row group {} flat column {} has {} values but {} rows", + row_group_idx, column_idx, chunk.meta_data.num_values, row_group.num_rows); + } + } + } + return Status::OK(); +} + namespace detail { -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges) { - if (position < 0 || nbytes <= 0) { - return {}; +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit) { + if (file_size < V2_PARQUET_FOOTER_SIZE || + serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, + file_size); + } + if (serialized_size > metadata_size_limit) { + return Status::Corruption("Parquet v2 footer size {} exceeds metadata limit {}", + serialized_size, metadata_size_limit); } + return Status::OK(); +} - std::vector ranges; - ranges.reserve(cached_ranges.size()); - const int64_t request_end = position + nbytes; - for (const auto& range : cached_ranges) { - if (range.size > 0 && range.offset < request_end && position < range.end_offset()) { - ranges.push_back(range); - } +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable) { + const int64_t mtime = description_mtime != 0 ? description_mtime : reader_mtime; + if (mtime == 0 && !is_immutable) { + // Unknown version is not a stable identity: an overwrite can preserve path and size while + // changing both footer semantics and page bytes. + return {}; } - std::sort(ranges.begin(), ranges.end(), [](const auto& lhs, const auto& rhs) { - if (lhs.offset != rhs.offset) { - return lhs.offset < rhs.offset; - } - return lhs.size > rhs.size; - }); - - std::vector plan; - int64_t cursor = position; - while (cursor < request_end) { - // At each cursor position, choose the cached range that already covers the cursor and - // extends farthest to the right. This handles both adjacent ranges and overlapping - // ranges. If no range covers the current cursor, there is a gap and the request must - // miss as a whole. - auto best = ranges.end(); - int64_t best_end = cursor; - for (auto it = ranges.begin(); it != ranges.end(); ++it) { - const int64_t cached_end = it->end_offset(); - if (it->offset <= cursor && cursor < cached_end && cached_end > best_end) { - best = it; - best_end = cached_end; - } - } - if (best == ranges.end()) { - return {}; - } - const int64_t copy_size = std::min(best_end, request_end) - cursor; - ParquetPageCacheReadPlanEntry entry; - entry.cached_range = *best; - entry.copy_offset_in_cache = cursor - best->offset; - entry.output_offset = cursor - position; - entry.copy_size = copy_size; - plan.push_back(entry); - cursor += copy_size; + const int64_t file_size = description_file_size >= 0 ? description_file_size : reader_file_size; + return fmt::format("{}::{}::mtime={}::size={}", fs_name, path, mtime, file_size); +} + +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length) { + if (offset < 0 || length <= 0 || length > MAX_SERIALIZED_PARQUET_INDEX_BYTES || + static_cast(offset) > file_size) { + return false; } - return plan; + return static_cast(length) <= file_size - static_cast(offset); +} + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end) { + return span_offset >= 0 && span_end >= span_offset && + span_end - span_offset <= MAX_SERIALIZED_PARQUET_INDEX_BYTES; } std::vector valid_prefetch_ranges( @@ -129,489 +182,426 @@ bool should_use_merge_range_reader(const std::vector& ran avg_io_size < io::MergeRangeFileReader::SMALL_IO; } +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size) { + return file_size <= in_memory_file_size && + (path.starts_with("http://") || path.starts_with("https://")); +} + } // namespace detail namespace { -// StoragePageCache only supports exact-key lookup. Keep lightweight range metadata here so later -// Arrow ReadAt requests can reuse cached bytes when their requested ranges are subsets of, or are -// fully covered by, previously cached ranges. Stale metadata is pruned on lookup. -std::mutex cached_page_range_index_mutex; -std::unordered_map> cached_page_range_index; -constexpr size_t MAX_CACHED_PAGE_RANGE_FILES = 4096; -constexpr size_t MAX_CACHED_PAGE_RANGES_PER_FILE = 65536; - -void register_cached_page_range(const std::string& file_key, int64_t position, int64_t nbytes) { - DORIS_CHECK(nbytes > 0); - std::lock_guard lock(cached_page_range_index_mutex); - if (cached_page_range_index.find(file_key) == cached_page_range_index.end() && - cached_page_range_index.size() >= MAX_CACHED_PAGE_RANGE_FILES) { - cached_page_range_index.erase(cached_page_range_index.begin()); - } - auto& ranges = cached_page_range_index[file_key]; - auto it = std::find_if(ranges.begin(), ranges.end(), [&](const ParquetPageCacheRange& range) { - return range.offset == position && range.size == nbytes; - }); - if (it == ranges.end()) { - if (ranges.size() >= MAX_CACHED_PAGE_RANGES_PER_FILE) { - ranges.erase(ranges.begin()); - } - ranges.push_back(ParquetPageCacheRange {position, nbytes}); - } -} - -void unregister_cached_page_range(const std::string& file_key, - const ParquetPageCacheRange& stale_range) { - std::lock_guard lock(cached_page_range_index_mutex); - auto it = cached_page_range_index.find(file_key); - if (it == cached_page_range_index.end()) { - return; - } - auto& ranges = it->second; - ranges.erase(std::remove_if(ranges.begin(), ranges.end(), - [&](const ParquetPageCacheRange& range) { - return range.offset == stale_range.offset && - range.size == stale_range.size; - }), - ranges.end()); - if (ranges.empty()) { - cached_page_range_index.erase(it); - } -} - -std::vector cached_page_ranges_for_file(const std::string& file_key) { - std::lock_guard lock(cached_page_range_index_mutex); - auto it = cached_page_range_index.find(file_key); - if (it == cached_page_range_index.end()) { - return {}; - } - return it->second; +constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; +constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; + +Status parse_native_parquet_footer(io::FileReaderSPtr file, + std::unique_ptr* metadata, + size_t* footer_size, io::IOContext* io_ctx, + bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(footer_size != nullptr); + const size_t file_size = file->size(); + if (file_size < V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 file is too small for a footer: {}", file_size); + } + + const size_t tail_size = std::min(file_size, V2_INITIAL_FOOTER_READ_SIZE); + std::vector tail(tail_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - tail_size, Slice(tail.data(), tail.size()), + &bytes_read, io_ctx)); + if (bytes_read != tail.size()) { + return Status::Corruption("Short Parquet v2 footer read: expected {}, got {}", tail.size(), + bytes_read); + } + const auto* magic = tail.data() + tail.size() - sizeof(V2_PARQUET_MAGIC); + if (memcmp(magic, V2_PARQUET_MAGIC, sizeof(V2_PARQUET_MAGIC)) != 0) { + return Status::Corruption("Invalid Parquet v2 footer magic in {}", file->path().native()); + } + + const uint32_t serialized_size = + decode_fixed32_le(tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE); + // The configured Thrift message ceiling also bounds this file-controlled allocation. Keep the + // check before both allocation and the optional second read so a sparse file cannot force a + // process-sized metadata buffer merely by advertising a large footer. + const size_t metadata_size_limit = + static_cast(std::max(config::thrift_max_message_size, 0)); + RETURN_IF_ERROR( + detail::validate_native_footer_size(serialized_size, file_size, metadata_size_limit)); + std::vector serialized_metadata(serialized_size); + if (serialized_size <= tail.size() - V2_PARQUET_FOOTER_SIZE) { + const auto* metadata_start = + tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE - serialized_size; + memcpy(serialized_metadata.data(), metadata_start, serialized_size); + } else { + bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - V2_PARQUET_FOOTER_SIZE - serialized_size, + Slice(serialized_metadata.data(), serialized_metadata.size()), + &bytes_read, io_ctx)); + if (bytes_read != serialized_metadata.size()) { + return Status::Corruption("Short Parquet v2 metadata read: expected {}, got {}", + serialized_metadata.size(), bytes_read); + } + } + + uint32_t thrift_size = serialized_size; + tparquet::FileMetaData thrift_metadata; + RETURN_IF_ERROR(deserialize_thrift_msg(serialized_metadata.data(), &thrift_size, true, + &thrift_metadata)); + auto parsed = + std::make_unique(std::move(thrift_metadata), serialized_size); + RETURN_IF_ERROR(parsed->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); + *footer_size = V2_PARQUET_FOOTER_SIZE + serialized_size; + *metadata = std::move(parsed); + return Status::OK(); } std::string build_page_cache_file_key(const io::FileReader& file_reader, const io::FileDescription& file_description) { - const int64_t mtime = - file_description.mtime != 0 ? file_description.mtime : file_reader.mtime(); - if (mtime == 0 && !file_description.is_immutable) { - // mtime == 0 means "unknown version", not the Unix epoch. V1 historically caches such a - // file under path::0, but copying that behavior for every V2 file is unsafe: a mutable file - // can be overwritten with different bytes while retaining both its path and size, causing - // process-global page cache entries to return stale data. Only callers that explicitly - // guarantee path immutability may use the mtime=0 cache key below. - return {}; - } - const int64_t file_size = file_description.file_size >= 0 - ? file_description.file_size - : static_cast(file_reader.size()); - return fmt::format("{}::{}::mtime={}::size={}", file_description.fs_name, - file_reader.path().native(), mtime, file_size); + return detail::build_native_file_cache_key( + file_description.fs_name, file_reader.path().native(), file_description.mtime, + file_reader.mtime(), file_description.file_size, + static_cast(file_reader.size()), file_description.is_immutable); } -class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { -public: - DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) - : _file_reader(std::move(file_reader)), - _base_file_reader(_file_reader), - _io_ctx(io_ctx), - _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)) { - DORIS_CHECK(_file_reader != nullptr); - if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { - _file_reader_stats = tracing_reader->stats(); - _base_file_reader = tracing_reader->inner_reader(); - } - DORIS_CHECK(_base_file_reader != nullptr); - set_mode(arrow::io::FileMode::READ); - } - - arrow::Status Close() override { - if (!_closed) { - collect_active_merge_range_profile(); - _closed = true; - } - return arrow::Status::OK(); - } - - bool closed() const override { return _closed; } - - arrow::Result Tell() const override { return _pos; } - - arrow::Status Seek(int64_t position) override { - if (position < 0) { - return arrow::Status::Invalid("negative seek position"); - } - _pos = position; - return arrow::Status::OK(); - } - - arrow::Result GetSize() override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - return static_cast(_file_reader->size()); - } - - arrow::Result Read(int64_t nbytes, void* out) override { - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(_pos, nbytes, out)); - _pos += bytes_read; - return bytes_read; - } - - arrow::Result> Read(int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, Read(nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; - } +} // namespace - arrow::Result ReadAt(int64_t position, int64_t nbytes, void* out) override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - if (position < 0 || nbytes < 0) { - return arrow::Status::Invalid("negative read position or length"); - } - if (try_read_from_page_cache(position, nbytes, out)) { - return nbytes; - } - size_t bytes_read = 0; - Status st = _file_reader->read_at( - static_cast(position), - Slice(static_cast(out), static_cast(nbytes)), &bytes_read, - _io_ctx); - if (!st.ok()) { - return arrow::Status::IOError(st.to_string_no_stack()); - } - insert_page_cache(position, nbytes, out, bytes_read); - return static_cast(bytes_read); - } +Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, + bool enable_page_cache, const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { + DORIS_CHECK(input_file_reader != nullptr); + if (detail::should_stage_small_http_file(input_file_reader->path().native(), + input_file_reader->size(), + config::in_memory_file_size)) { + // A metadata-cache hit can make the first physical read start inside a tiny HTTP file. + // Read it from byte zero once so EOF-range quirks cannot make warm scans less reliable + // than cold scans, while keeping this compatibility policy entirely inside v2. + native_file = std::make_shared(std::move(input_file_reader)); + } else { + native_file = std::move(input_file_reader); + } + native_io_ctx = io_ctx; + + // Footer and page bytes must use the same stable identity. In particular, fs_name separates + // identical HDFS paths from different nameservices, while an unknown mutable version bypasses + // both caches rather than reusing an overwritten file of the same size. + auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + auto meta_cache_key = build_page_cache_file_key(*native_file, file_description); + const bool has_stable_meta_cache_identity = !meta_cache_key.empty(); + if (has_stable_meta_cache_identity) { + // The discriminator prevents a v1 metadata value from being cast as the v2-owned type; + // schema mapping flags participate because they change the cached native schema tree. + meta_cache_key.append("\0v2", 3); + meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); + meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); + } + size_t native_footer_size = 0; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && + meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { + native_metadata = native_meta_cache_handle.data(); + ++native_footer_cache_hits; + } else { + RETURN_IF_ERROR(parse_native_parquet_footer( + native_file, &native_metadata_owner, &native_footer_size, io_ctx, + enable_mapping_varbinary, enable_mapping_timestamp_tz)); + ++native_footer_read_calls; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { + meta_cache->insert(meta_cache_key, native_metadata_owner.release(), + &native_meta_cache_handle); + native_metadata = native_meta_cache_handle.data(); + } else { + native_metadata = native_metadata_owner.get(); + } + } + DORIS_CHECK(native_metadata != nullptr); + + auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); + native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); + // Native page readers use the FileDescription-derived immutable identity directly. + native_page_cache_file_key = page_cache_file_key; + return Status::OK(); +} - arrow::Result> ReadAt(int64_t position, - int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(position, nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; +Status ParquetFileContext::load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const { + DORIS_CHECK(offset_indexes != nullptr); + offset_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - void register_page_cache_ranges(std::vector ranges) { - std::lock_guard lock(_page_cache_mutex); - _page_cache_ranges = std::move(ranges); + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for OffsetIndex", row_group_id); } - - void prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - auto cached_reader = cached_remote_file_reader(); - if (cached_reader == nullptr) { - return; - } - const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : _io_ctx; - for (const auto& range : ranges) { - if (range.offset < 0 || range.size <= 0) { + const auto& native_row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + try { + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(native_row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for OffsetIndex", + leaf_column_id); + } + const auto& column_chunk = native_row_group.columns[leaf_column_id]; + if (!column_chunk.__isset.offset_index_offset || + !column_chunk.__isset.offset_index_length || + column_chunk.offset_index_length <= 0) { continue; } - cached_reader->prefetch_range(static_cast(range.offset), - static_cast(range.size), prefetch_io_ctx); - } - } - - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - reset_active_file_reader(); - const auto valid_ranges = detail::valid_prefetch_ranges(ranges); - if (!detail::should_use_merge_range_reader( - valid_ranges, avg_io_size, - typeid_cast(_base_file_reader.get()) != nullptr)) { - return false; - } - - std::vector random_access_ranges; - random_access_ranges.reserve(valid_ranges.size()); - for (const auto& range : valid_ranges) { - random_access_ranges.emplace_back(static_cast(range.offset), - static_cast(range.end_offset())); - } - - // This mirrors the v1 parquet reader: when projected column chunks in a row group are - // small random IOs, make the actual ReadAt path range-aware. Arrow still drives decoding, - // but every page read below this point sees MergeRangeFileReader instead of the raw remote - // reader, so adjacent small requests can be coalesced and served from merge buffers. - // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later - // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once - // and satisfy the following ReadAt calls from its boxes, reducing remote request count. - _merge_range_active = true; - set_active_file_reader(std::make_shared( - profile, _base_file_reader, random_access_ranges, merge_read_slice_size)); - return true; - } - - void reset_random_access_ranges() { reset_active_file_reader(); } - - ParquetPageCacheStats page_cache_stats() const { - std::lock_guard lock(_page_cache_mutex); - return _page_cache_stats; - } - -private: - bool page_cache_enabled() const { - return _enable_page_cache && !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty(); - } - - bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { - if (nbytes <= 0) { - return false; - } - const int64_t end = position + nbytes; - for (const auto& range : _page_cache_ranges) { - const int64_t range_end = range.offset + range.size; - if (position >= range.offset && end <= range_end) { - return true; + const int64_t index_offset = column_chunk.offset_index_offset; + const int64_t index_length = column_chunk.offset_index_length; + if (!detail::is_serialized_index_range_safe(native_file->size(), index_offset, + index_length)) { + // OffsetIndex is optional. A malformed range must not allocate from untrusted + // footer values or redirect the native reader outside the file. + continue; + } + std::vector serialized_index(static_cast(index_length)); + Slice index_slice(serialized_index.data(), serialized_index.size()); + size_t bytes_read = 0; + if (!native_file->read_at(index_offset, index_slice, &bytes_read, native_io_ctx).ok() || + bytes_read != serialized_index.size()) { + continue; + } + uint32_t thrift_length = static_cast(serialized_index.size()); + tparquet::OffsetIndex native_index; + if (!deserialize_thrift_msg(serialized_index.data(), &thrift_length, true, + &native_index) + .ok() || + native_index.page_locations.empty()) { + continue; + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + native_row_group.columns[leaf_column_id].meta_data, native_file->size(), + compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index( + native_index, chunk_range, + native_row_group.columns[leaf_column_id].meta_data.data_page_offset, + native_row_group.num_rows)) { + // OffsetIndex is optional. Reject the complete index instead of letting one bad + // location redirect an indexed reader outside its owning column chunk. + continue; } + offset_indexes->emplace(leaf_column_id, std::move(native_index)); } - return false; + } catch (const std::exception&) { + // OffsetIndex is optional. Selected logical ranges still enforce correctness, while the + // native reader conservatively falls back to sequential page traversal. + offset_indexes->clear(); } + return Status::OK(); +} - StoragePageCache::CacheKey page_cache_key(int64_t position, int64_t nbytes) const { - return StoragePageCache::CacheKey(_page_cache_file_key, - static_cast(position + nbytes), position); +Status ParquetFileContext::load_native_page_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, int64_t* read_time, + int64_t* parse_time) const { + DORIS_CHECK(page_indexes != nullptr); + page_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - bool copy_cached_range(const ParquetPageCacheRange& cached_range, int64_t copy_position, - int64_t copy_size, void* out, int64_t output_offset) { - PageCacheHandle handle; - if (!StoragePageCache::instance()->lookup( - page_cache_key(cached_range.offset, cached_range.size), &handle, - segment_v2::DATA_PAGE)) { - unregister_cached_page_range(_page_cache_file_key, cached_range); - return false; + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for PageIndex", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + + struct SerializedIndexRange { + int leaf_column_id; + int64_t offset; + int64_t length; + }; + struct PendingPageIndex { + NativeParquetPageIndex indexes; + bool has_column_index = false; + bool has_offset_index = false; + }; + std::vector column_index_ranges; + std::vector offset_index_ranges; + std::unordered_map pending_indexes; + + auto valid_index_range = [&](int64_t offset, int64_t length) { + return detail::is_serialized_index_range_safe(native_file->size(), offset, length); + }; + + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for PageIndex", leaf_column_id); + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (!chunk.__isset.column_index_offset || !chunk.__isset.column_index_length || + !chunk.__isset.offset_index_offset || !chunk.__isset.offset_index_length) { + continue; } - Slice cached = handle.data(); - const int64_t cache_offset = copy_position - cached_range.offset; - DORIS_CHECK(cache_offset >= 0); - DORIS_CHECK(cached.size >= static_cast(cache_offset + copy_size)); - memcpy(static_cast(out) + output_offset, cached.data + cache_offset, - static_cast(copy_size)); - return true; - } - - bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { - auto plan = detail::plan_page_cache_range_read( - position, nbytes, cached_page_ranges_for_file(_page_cache_file_key)); - if (plan.empty()) { - return false; + if (!valid_index_range(chunk.column_index_offset, chunk.column_index_length) || + !valid_index_range(chunk.offset_index_offset, chunk.offset_index_length)) { + continue; } - for (const auto& entry : plan) { - if (!copy_cached_range(entry.cached_range, - entry.cached_range.offset + entry.copy_offset_in_cache, - entry.copy_size, out, entry.output_offset)) { - return false; + column_index_ranges.push_back( + {leaf_column_id, chunk.column_index_offset, chunk.column_index_length}); + offset_index_ranges.push_back( + {leaf_column_id, chunk.offset_index_offset, chunk.offset_index_length}); + pending_indexes.try_emplace(leaf_column_id); + } + + auto read_coalesced_indexes = [&](std::vector* ranges, + bool column_index) { + std::sort(ranges->begin(), ranges->end(), + [](const auto& lhs, const auto& rhs) { return lhs.offset < rhs.offset; }); + size_t range_begin = 0; + while (range_begin < ranges->size()) { + size_t range_end = range_begin + 1; + int64_t span_end = (*ranges)[range_begin].offset + (*ranges)[range_begin].length; + while (range_end < ranges->size() && (*ranges)[range_end].offset <= span_end) { + span_end = std::max(span_end, + (*ranges)[range_end].offset + (*ranges)[range_end].length); + ++range_end; } - } - return true; - } - bool try_read_from_page_cache(int64_t position, int64_t nbytes, void* out) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes)) { - return false; - } - ++_page_cache_stats.read_count; - // Fast path: Arrow issues the same ReadAt(offset, size) again, so the exact - // StoragePageCache key matches. - // Fallback path: Arrow may read a different but related byte range on another scan. - // Examples: - // - Current request [120, 150) can be served from cached [100, 200) by copying the - // 30-byte subset starting at cached offset 20. - // - Current request [100, 260) can be served by stitching cached [100, 180) and - // [180, 260). If any middle span is missing, it is a miss and the file reader fills - // the whole request from storage. - if (!copy_cached_range(ParquetPageCacheRange {position, nbytes}, position, nbytes, out, - 0) && - !try_read_from_cached_ranges(position, nbytes, out)) { - ++_page_cache_stats.miss_count; - return false; + const int64_t span_offset = (*ranges)[range_begin].offset; + if (!detail::is_serialized_index_span_safe(span_offset, span_end)) { + // Optional indexes share one allocation per contiguous footer block. Skipping the + // whole block prevents many individually small ranges from bypassing the budget. + range_begin = range_end; + continue; + } + const int64_t span_length = span_end - span_offset; + std::vector serialized(static_cast(span_length)); + Slice slice(serialized.data(), serialized.size()); + size_t bytes_read = 0; + Status read_status; + int64_t read_time_sink = 0; + { + SCOPED_RAW_TIMER(read_time == nullptr ? &read_time_sink : read_time); + read_status = native_file->read_at(span_offset, slice, &bytes_read, native_io_ctx); + } + if (read_status.ok() && bytes_read == serialized.size()) { + for (size_t i = range_begin; i < range_end; ++i) { + const auto& range = (*ranges)[i]; + auto pending = pending_indexes.find(range.leaf_column_id); + if (pending == pending_indexes.end()) { + continue; + } + uint32_t thrift_length = static_cast(range.length); + const auto* thrift_data = + serialized.data() + static_cast(range.offset - span_offset); + int64_t parse_time_sink = 0; + SCOPED_RAW_TIMER(parse_time == nullptr ? &parse_time_sink : parse_time); + if (column_index) { + pending->second.has_column_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.column_index) + .ok(); + } else { + pending->second.has_offset_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.offset_index) + .ok(); + } + } + } + range_begin = range_end; } - ++_page_cache_stats.hit_count; - ++_page_cache_stats.compressed_hit_count; - return true; - } + }; - void insert_page_cache(int64_t position, int64_t nbytes, const void* data, size_t bytes_read) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes) || - bytes_read != static_cast(nbytes)) { - return; - } - auto* page = new DataPage(bytes_read, true, segment_v2::DATA_PAGE); - memcpy(page->data(), data, bytes_read); - PageCacheHandle handle; - StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, - segment_v2::DATA_PAGE); - register_cached_page_range(_page_cache_file_key, position, nbytes); - ++_page_cache_stats.write_count; - ++_page_cache_stats.compressed_write_count; - } + // Parquet writers place each index kind in a contiguous block. Reading overlapping/adjacent + // ranges as one span keeps cold small-file planning from paying two remote round trips per + // projected leaf, while refusing gaps avoids amplifying reads from untrusted footer offsets. + read_coalesced_indexes(&column_index_ranges, true); + read_coalesced_indexes(&offset_index_ranges, false); - void set_active_file_reader(io::FileReaderSPtr reader) { - DORIS_CHECK(reader != nullptr); - _file_reader = _file_reader_stats != nullptr - ? std::make_shared(std::move(reader), - _file_reader_stats) - : std::move(reader); - } - - void reset_active_file_reader() { - collect_active_merge_range_profile(); - _merge_range_active = false; - set_active_file_reader(_base_file_reader); - } - - void collect_active_merge_range_profile() { - if (_merge_range_active && _file_reader != nullptr) { - // MergeRangeFileReader writes its MergedSmallIO counters only from - // collect_profile_before_close(). v2 replaces the active reader for every row group, - // so collect before overwriting it; Close() handles the final row group. Example: - // RG0 installs a merge reader, RG1 calls set_random_access_ranges() and resets the - // active reader first, so RG0's RequestIO/MergedIO counters must be flushed here. - _file_reader->collect_profile_before_close(); - } - } - - std::shared_ptr cached_remote_file_reader() { - if (_merge_range_active) { - return nullptr; - } - auto reader = _file_reader; - if (reader == nullptr) { - return nullptr; + for (auto& [leaf_column_id, pending] : pending_indexes) { + const auto& chunk = row_group.columns[leaf_column_id]; + auto& indexes = pending.indexes; + if (!pending.has_column_index || !pending.has_offset_index || + indexes.column_index.null_pages.size() != indexes.offset_index.page_locations.size()) { + continue; } - // FileReader::init wraps the physical reader with TracingFileReader when scan IO stats are - // enabled. Prefetch should target the physical cached reader below that tracing wrapper, - // otherwise v2 scans with profiling would silently lose prefetch. - if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { - reader = tracing_reader->inner_reader(); + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, native_file->size(), compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index(indexes.offset_index, chunk_range, + chunk.meta_data.data_page_offset, row_group.num_rows)) { + continue; } - return std::dynamic_pointer_cast(reader); + page_indexes->emplace(leaf_column_id, std::move(indexes)); } + return Status::OK(); +} - io::FileReaderSPtr _file_reader; - io::FileReaderSPtr _base_file_reader; - io::FileReaderStats* _file_reader_stats = nullptr; - io::IOContext* _io_ctx = nullptr; - int64_t _pos = 0; - bool _closed = false; - bool _enable_page_cache = false; - bool _merge_range_active = false; - std::string _page_cache_file_key; - mutable std::mutex _page_cache_mutex; - std::vector _page_cache_ranges; - ParquetPageCacheStats _page_cache_stats; -}; - -} // namespace - -Status arrow_status_to_doris_status(const arrow::Status& status) { - if (status.ok()) { - return Status::OK(); +void ParquetFileContext::prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { + io::FileReaderSPtr reader = native_file; + if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { + reader = tracing_reader->inner_reader(); } - if (status.IsIOError()) { - return Status::IOError(status.ToString()); + auto cached_reader = std::dynamic_pointer_cast(reader); + if (cached_reader == nullptr) { + return; } - if (status.IsInvalid()) { - return Status::InvalidArgument(status.ToString()); + const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : native_io_ctx; + for (const auto& range : detail::valid_prefetch_ranges(ranges)) { + cached_reader->prefetch_range(cast_set(range.offset), cast_set(range.size), + prefetch_io_ctx); } - return Status::InternalError(status.ToString()); } -Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { - DORIS_CHECK(input_file_reader != nullptr); - auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); - try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); - metadata = this->file_reader->metadata(); - schema = metadata != nullptr ? metadata->schema() : nullptr; - } catch (const ::parquet::ParquetException& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet file: {}", e.what()); - } catch (const std::exception& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet file: {}", e.what()); +bool ParquetFileContext::set_native_random_access_ranges( + const std::vector& ranges, size_t avg_io_size, + RuntimeProfile* profile, int64_t merge_read_slice_size) { + DORIS_CHECK(native_file != nullptr); + if (!detail::should_use_merge_range_reader( + ranges, avg_io_size, + typeid_cast(native_file.get()) != nullptr)) { + native_row_group_file = native_file; + return false; } - if (metadata == nullptr || schema == nullptr) { - return Status::Corruption("Failed to read parquet metadata"); + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + std::vector native_ranges; + native_ranges.reserve(valid_ranges.size()); + for (const auto& range : valid_ranges) { + native_ranges.emplace_back(cast_set(range.offset), + cast_set(range.end_offset())); } - return Status::OK(); -} - -void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get()) - ->register_page_cache_ranges(std::move(ranges)); -} - -void ParquetFileContext::prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->prefetch_ranges(ranges, io_ctx); -} - -bool ParquetFileContext::set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - DORIS_CHECK(arrow_file != nullptr); - return static_cast(arrow_file.get()) - ->set_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); + std::ranges::sort(native_ranges, {}, &io::PrefetchRange::start_offset); + native_row_group_file = std::make_shared( + profile, native_file, native_ranges, merge_read_slice_size); + return true; } void ParquetFileContext::reset_random_access_ranges() { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->reset_random_access_ranges(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); } ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { - if (arrow_file == nullptr) { - return {}; - } - return static_cast(arrow_file.get())->page_cache_stats(); + return {}; } Status ParquetFileContext::close() { - if (file_reader != nullptr) { - try { - file_reader->Close(); - } catch (const std::exception&) { - } - } - if (arrow_file != nullptr) { - static_cast(arrow_status_to_doris_status(arrow_file->Close())); - } - file_reader.reset(); - arrow_file.reset(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); + native_metadata = nullptr; + native_metadata_owner.reset(); + native_meta_cache_handle = {}; + native_file.reset(); + native_io_ctx = nullptr; + native_page_cache_enabled = false; + native_page_cache_file_key.clear(); return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 0dca52244957d7..0cd413e10557ac 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -15,16 +15,21 @@ #pragma once -#include -#include +#include #include #include #include +#include +#include +#include +#include #include #include "common/status.h" +#include "format_v2/parquet/native_schema_desc.h" #include "io/fs/file_reader.h" +#include "util/obj_lru_cache.h" namespace doris::io { struct FileDescription; @@ -37,6 +42,26 @@ class RuntimeProfile; namespace doris::format::parquet { +struct NativeParquetPageIndex; + +// V2-owned footer/schema tree. Production planning and decoding consume this object directly; +// Arrow metadata is intentionally not materialized from the serialized footer. +class NativeParquetMetadata { +public: + NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size); + ~NativeParquetMetadata(); + + Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); + const tparquet::FileMetaData& to_thrift() const { return _metadata; } + const NativeFieldDescriptor& schema() const { return _schema; } + size_t get_mem_size() const { return _parsed_size; } + +private: + tparquet::FileMetaData _metadata; + NativeFieldDescriptor _schema; + size_t _parsed_size = 0; +}; + struct ParquetPageCacheRange { int64_t offset = 0; int64_t size = 0; @@ -44,17 +69,6 @@ struct ParquetPageCacheRange { int64_t end_offset() const { return offset + size; } }; -struct ParquetPageCacheReadPlanEntry { - // The exact cached StoragePageCache entry. The final cache key is still exact-range based: - // file key + cached_range.end_offset() + cached_range.offset. - ParquetPageCacheRange cached_range; - // Byte offset inside cached_range to start copying from. - int64_t copy_offset_in_cache = 0; - // Byte offset inside the current ReadAt output buffer to start writing to. - int64_t output_offset = 0; - int64_t copy_size = 0; -}; - struct ParquetPageCacheStats { int64_t read_count = 0; int64_t write_count = 0; @@ -66,21 +80,19 @@ struct ParquetPageCacheStats { namespace detail { -// Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of -// previously cached entries. -// StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The -// caller therefore keeps lightweight cached range metadata and uses this function to decide -// which exact cache entries to fetch and which byte spans to copy. -// Examples: -// 1. Subset hit: -// request [120, 150), cached [100, 200) -> copy 30 bytes from cached offset 20. -// 2. Superset hit covered by multiple cached entries: -// request [100, 260), cached [100, 180) and [180, 260) -// -> two copies: [100, 180) to output offset 0, [180, 260) to output offset 80. -// 3. Partial overlap is a miss: -// request [100, 260), cached [100, 180) only -> empty plan, caller reads from file. -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges); +inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; + +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit); + +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable); + +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length); + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); // Keep only byte ranges that are safe to hand to FileReader implementations. Parquet metadata is // expected to contain non-negative offsets and positive compressed sizes, but tests and corrupted @@ -89,54 +101,74 @@ std::vector plan_page_cache_range_read( std::vector valid_prefetch_ranges( const std::vector& ranges); -// Average projected column-chunk size for one row group. The v1 parquet path uses this value to -// decide whether a row group is dominated by small random IOs; v2 uses the same signal before -// installing MergeRangeFileReader. Example: chunks of 512KB and 1MB average below SMALL_IO and are -// good merge-reader candidates, while two 8MB chunks should stay on the raw random-access reader. +// Average projected column-chunk size for one row group. V2 uses this signal to decide whether the +// row group is dominated by small random IOs before installing MergeRangeFileReader. Example: +// chunks of 512KB and 1MB average below SMALL_IO and are good merge-reader candidates, while two +// 8MB chunks should stay on the raw random-access reader. size_t average_prefetch_range_size(const std::vector& ranges); -// Decide whether Arrow ReadAt() should be routed through MergeRangeFileReader for the current row -// group. This is intentionally stricter than the background warm-up path: +// Decide whether native data-page ReadAt() should be routed through MergeRangeFileReader for the +// current row group. This is intentionally stricter than the background warm-up path: // - no valid projected chunks -> nothing to merge; // - in-memory file readers already avoid remote random IO; // - average chunk size >= MergeRangeFileReader::SMALL_IO would make merged reading wasteful. bool should_use_merge_range_reader(const std::vector& ranges, size_t avg_io_size, bool is_in_memory_reader); +// HTTP range servers may reject an overlong range near EOF even after accepting the capability +// probe. Staging a bounded small HTTP object also turns all native page reads into memory copies. +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size); + } // namespace detail struct ParquetFileContext { - std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader - std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser - std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) - const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema + // Native metadata, index, and data-page paths share Doris' FileReader without transferring + // ownership to an external metadata tree. + io::FileReaderSPtr native_file; + // Row-group-scoped view of native_file. Small projected chunks use MergeRangeFileReader; + // large chunks and in-memory files keep native_file. + io::FileReaderSPtr native_row_group_file; + io::IOContext* native_io_ctx = nullptr; + // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is + // owned by native_meta_cache_handle; a miss without cache is owned by native_metadata_owner. + const NativeParquetMetadata* native_metadata = nullptr; + std::unique_ptr native_metadata_owner; + ObjLRUCache::CacheHandle native_meta_cache_handle; + int64_t native_footer_read_calls = 0; + int64_t native_footer_cache_hits = 0; + bool native_page_cache_enabled = false; + std::string native_page_cache_file_key; Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); - // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page - // decoding, so v2 caches the serialized bytes read inside these ranges and excludes - // footer/metadata reads that happen before registration. - void register_page_cache_ranges(std::vector ranges); + const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); + Status load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const; + Status load_native_page_indexes(int row_group_id, + const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, + int64_t* read_time = nullptr, + int64_t* parse_time = nullptr) const; // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Switch the active reader used by Arrow ReadAt() to v1's MergeRangeFileReader when the current - // row group's projected column chunks are small random IOs. This is the real v1-compatible - // prefetch path: subsequent Arrow page reads go through the merged reader instead of merely - // warming file cache in the background. Returns true when merge-range reading is active. - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size); - // Restore Arrow ReadAt() to the base Doris file reader and flush any active merge-reader - // counters. Row-group setup uses this before dictionary-page probes, because those probes are - // a separate pass over the column chunk from the later Arrow RecordReader data-page stream. + // Install the row-group-scoped MergeRangeFileReader on the native data-page path. Dictionary + // probes must run before this method because their native ReadAt order is independent of the + // sequential projected chunk ranges consumed by MergeRangeFileReader. + bool set_native_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size); + const io::FileReaderSPtr& native_data_file() const { + return native_row_group_file != nullptr ? native_row_group_file : native_file; + } + // Restore native ReadAt() to the base Doris file reader and flush merge-reader counters. void reset_random_access_ranges(); ParquetPageCacheStats page_cache_stats() const; Status close(); }; -Status arrow_status_to_doris_status(const arrow::Status& status); - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 29783e14a05071..b56a57d419e0f3 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/parquet_statistics.h" +#include "runtime/file_scan_profile.h" namespace doris::format::parquet { @@ -26,9 +27,13 @@ void ParquetProfile::init(RuntimeProfile* profile) { return; } + file_scan_profile::ensure_hierarchy(profile); static const char* parquet_profile = "ParquetReader"; - ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); + total_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); + // Row-group counters are part of the long-standing ParquetReader profile contract. Keep them + // below the format node so profile parsers and operators can attribute pruning to Parquet. filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -55,10 +60,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); selected_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); + // Keep every Parquet scan metric below the format node: profile consumers extract that + // subtree and otherwise silently lose this counter even though filtering happened. rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", TUnit::UNIT, parquet_profile, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); + dense_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DenseBatches", TUnit::UNIT, parquet_profile, 1); + selected_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedBatches", TUnit::UNIT, + parquet_profile, 1); empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", TUnit::UNIT, parquet_profile, 1); range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", @@ -69,14 +80,30 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, parquet_profile, 1); - arrow_read_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); - arrow_skip_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowSkipRecordsTime", parquet_profile, 1); + level_only_read_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlyReadTime", parquet_profile, 1); + level_only_skip_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + hybrid_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionBatches", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionRanges", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_null_fallback_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "HybridSelectionNullFallbackBatches", TUnit::UNIT, parquet_profile, 1); + native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, + parquet_profile, 1); + native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", + TUnit::UNIT, parquet_profile, 1); + page_crossing_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCrossingBatches", + TUnit::UNIT, parquet_profile, 1); + nested_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1); + // Format-specific counters stay below ParquetReader so subtree consumers cannot silently + // attribute them to a different file format initialized on the same RuntimeProfile. filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1); raw_rows_read = @@ -88,7 +115,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); - page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); + page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, + parquet_profile, 1); page_index_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); read_page_index_time = @@ -103,8 +131,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); row_group_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); - file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); - file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", + TUnit::UNIT, parquet_profile, 1); + file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, + parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); @@ -139,6 +169,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); predicate_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); + predicate_compaction_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateCompactionTime", parquet_profile, 1); + predicate_compaction_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionBytes", + TUnit::BYTES, parquet_profile, 1); + predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", + TUnit::UNIT, parquet_profile, 1); + fixed_width_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FixedWidthPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); + fixed_width_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FixedWidthPredicateDirectRows", TUnit::UNIT, parquet_profile, 1); dict_filter_rewrite_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); dict_filter_expr_rewrite_time = @@ -157,7 +197,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); - convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); bloom_filter_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); } @@ -176,12 +215,42 @@ void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_sta COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); + COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); + COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); + COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); + COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); + COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); + COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); + COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); + COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); +} + +void ParquetProfile::update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const { + const int64_t filtered = selected ? 0 : 1; + COUNTER_UPDATE(filtered_row_groups, filtered); + COUNTER_UPDATE(filtered_row_groups_by_dictionary, + pruning_stats.filtered_row_groups_by_dictionary); + COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, + pruning_stats.filtered_row_groups_by_bloom_filter); + COUNTER_UPDATE(filtered_row_groups_by_page_index, + pruning_stats.filtered_row_groups_by_page_index); + // Initial footer planning counts every surviving candidate as readable. Lazy probes correct + // that estimate only when a later dictionary, Bloom, or page-index check removes the group. + COUNTER_UPDATE(to_read_row_groups, -filtered); + COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); + COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); @@ -199,9 +268,35 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .reader_read_rows = reader_read_rows, .reader_skip_rows = reader_skip_rows, .reader_select_rows = reader_select_rows, - .arrow_read_records_time = arrow_read_records_time, - .arrow_skip_records_time = arrow_skip_records_time, + .level_only_read_time = level_only_read_time, + .level_only_skip_time = level_only_skip_time, .materialization_time = materialization_time, + .hybrid_selection_batches = hybrid_selection_batches, + .hybrid_selection_ranges = hybrid_selection_ranges, + .hybrid_selection_null_fallback_batches = hybrid_selection_null_fallback_batches, + .decompress_time = decompress_time, + .decompress_count = decompress_cnt, + .decode_header_time = decode_header_time, + .decode_value_time = decode_value_time, + .decode_dictionary_time = decode_dict_time, + .decode_level_time = decode_level_time, + .decode_null_map_time = decode_null_map_time, + .page_index_read_calls = page_index_read_calls, + .skip_page_header_count = skip_page_header_num, + .parse_page_header_count = parse_page_header_num, + .read_page_header_time = read_page_header_time, + .page_read_count = page_read_counter, + .page_cache_write_count = page_cache_write_counter, + .page_cache_compressed_write_count = page_cache_compressed_write_counter, + .page_cache_decompressed_write_count = page_cache_decompressed_write_counter, + .page_cache_hit_count = page_cache_hit_counter, + .page_cache_miss_count = page_cache_missing_counter, + .page_cache_compressed_hit_count = page_cache_compressed_hit_counter, + .page_cache_decompressed_hit_count = page_cache_decompressed_hit_counter, + .native_read_calls = native_read_calls, + .native_page_fragments = native_page_fragments, + .page_crossing_batches = page_crossing_batches, + .nested_batches = nested_batches, }; } @@ -212,10 +307,17 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .rows_filtered_by_conjunct = rows_filtered_by_conjunct, .lazy_read_filtered_rows = lazy_read_filtered_rows, .total_batches = total_batches, + .dense_batches = dense_batches, + .selected_batches = selected_batches, .empty_selection_batches = empty_selection_batches, .range_gap_skipped_rows = range_gap_skipped_rows, .column_read_time = column_read_time, .predicate_filter_time = predicate_filter_time, + .predicate_compaction_time = predicate_compaction_time, + .predicate_compaction_bytes = predicate_compaction_bytes, + .predicate_compaction_count = predicate_compaction_count, + .fixed_width_predicate_direct_batches = fixed_width_predicate_direct_batches, + .fixed_width_predicate_direct_rows = fixed_width_predicate_direct_rows, .dict_filter_rewrite_time = dict_filter_rewrite_time, .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, .dict_filter_read_dict_time = dict_filter_read_dict_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 827cac45d94df0..170f14b56c5d14 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -31,28 +31,65 @@ struct ParquetPageSkipProfile { // ============================================================================ // ============================================================================ struct ParquetColumnReaderProfile { - RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() - RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() - RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() - RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; // Arrow SkipRecords time (ns) - RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() + RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() + RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() + // COUNT(nullable_col) shape-only path; ordinary scans keep both counters zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; + RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + // Native page/encoding reader internals. These counters keep page IO, decompression, levels, + // value decode and conversion attributable to separate stages. + RuntimeProfile::Counter* decompress_time = nullptr; + RuntimeProfile::Counter* decompress_count = nullptr; + RuntimeProfile::Counter* decode_header_time = nullptr; + RuntimeProfile::Counter* decode_value_time = nullptr; + RuntimeProfile::Counter* decode_dictionary_time = nullptr; + RuntimeProfile::Counter* decode_level_time = nullptr; + RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* page_index_read_calls = nullptr; + RuntimeProfile::Counter* skip_page_header_count = nullptr; + RuntimeProfile::Counter* parse_page_header_count = nullptr; + RuntimeProfile::Counter* read_page_header_time = nullptr; + RuntimeProfile::Counter* page_read_count = nullptr; + RuntimeProfile::Counter* page_cache_write_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_miss_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_hit_count = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; // native column-reader calls + RuntimeProfile::Counter* native_page_fragments = nullptr; // page-bounded read fragments + RuntimeProfile::Counter* page_crossing_batches = nullptr; // batches spanning multiple pages + RuntimeProfile::Counter* nested_batches = nullptr; // complex-column read batches }; // ============================================================================ // ============================================================================ struct ParquetScanProfile { - RuntimeProfile::Counter* raw_rows_read = nullptr; // raw rows read from RecordReader + RuntimeProfile::Counter* raw_rows_read = nullptr; // logical rows consumed before filtering RuntimeProfile::Counter* selected_rows = nullptr; // rows selected after conjunct filtering RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; // rows filtered by conjuncts RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; // rows avoided by late materialization RuntimeProfile::Counter* total_batches = nullptr; // total batch count + RuntimeProfile::Counter* dense_batches = nullptr; // batches retaining every physical row + RuntimeProfile::Counter* selected_batches = + nullptr; // non-empty batches compacted by predicates RuntimeProfile::Counter* empty_selection_batches = nullptr; // empty batches after full filtering RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps RuntimeProfile::Counter* column_read_time = nullptr; // column read time (ns) RuntimeProfile::Counter* predicate_filter_time = nullptr; // predicate filter time (ns) + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; // expression/residual rewrite time (ns) @@ -73,11 +110,15 @@ struct ParquetScanProfile { struct ParquetProfile { void init(RuntimeProfile* profile); void update_pruning_stats(const ParquetPruningStats& pruning_stats) const; + void update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const; ParquetPageSkipProfile page_skip_profile() const; ParquetColumnReaderProfile column_reader_profile() const; ParquetScanProfile scan_profile() const; + RuntimeProfile::Counter* total_time = nullptr; + RuntimeProfile::Counter* filtered_row_groups = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_dictionary = nullptr; @@ -96,6 +137,8 @@ struct ParquetProfile { RuntimeProfile::Counter* selected_rows = nullptr; RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; RuntimeProfile::Counter* total_batches = nullptr; + RuntimeProfile::Counter* dense_batches = nullptr; + RuntimeProfile::Counter* selected_batches = nullptr; RuntimeProfile::Counter* empty_selection_batches = nullptr; RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; @@ -103,9 +146,17 @@ struct ParquetProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; RuntimeProfile::Counter* reader_skip_rows = nullptr; RuntimeProfile::Counter* reader_select_rows = nullptr; - RuntimeProfile::Counter* arrow_read_records_time = nullptr; - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; + // COUNT(nullable_col) shape-only path; ordinary scans keep these zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; + RuntimeProfile::Counter* native_page_fragments = nullptr; + RuntimeProfile::Counter* page_crossing_batches = nullptr; + RuntimeProfile::Counter* nested_batches = nullptr; RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* filtered_bytes = nullptr; @@ -149,6 +200,11 @@ struct ParquetProfile { RuntimeProfile::Counter* parse_page_header_num = nullptr; RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; @@ -158,7 +214,6 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; - RuntimeProfile::Counter* convert_time = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 753b3628bfa19b..4a80bc0403df8d 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -39,7 +39,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" -#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/count_column_reader.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -57,34 +57,26 @@ struct ParquetReaderScanState { bool enable_strict_mode = false; }; -int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); -} - -void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); +Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (column_schema.leaf_column_id >= 0) { - leaf_column_ids->insert(column_schema.leaf_column_id); + if (!column_schema.type_descriptor.unsupported_reason.empty()) { + return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, + column_schema.type_descriptor.unsupported_reason); } - return; + return Status::OK(); } for (const auto& child : column_schema.children) { DORIS_CHECK(child != nullptr); - collect_all_leaf_column_ids(*child, leaf_column_ids); + RETURN_IF_ERROR(validate_all_projected_leaves_supported(*child)); } + return Status::OK(); } -void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex& projection, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (projection.project_all_children || projection.children.empty()) { - collect_all_leaf_column_ids(column_schema, leaf_column_ids); - return; +Status validate_projected_leaves_supported(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE || + projection.project_all_children || projection.children.empty()) { + return validate_all_projected_leaves_supported(column_schema); } for (const auto& child_projection : projection.children) { const auto child_it = @@ -92,57 +84,35 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, return child_schema->local_id == child_projection.local_id(); }); DORIS_CHECK(child_it != column_schema.children.end()); - collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); + RETURN_IF_ERROR(validate_projected_leaves_supported(**child_it, child_projection)); } + return Status::OK(); } -void collect_request_leaf_column_ids( +Status validate_requested_columns_supported( const std::vector>& file_schema, - const format::FileScanRequest& request, std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - auto collect_scan_column = [&](const format::LocalColumnIndex& projection) { + const format::FileScanRequest& request) { + // Validate the projected native-schema leaves before pruning: checking a physical carrier at + // a later read site can let an unsupported logical type silently pass when every row is pruned. + auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { const auto local_id = projection.local_id(); if (local_id == format::ROW_POSITION_COLUMN_ID || local_id == format::GLOBAL_ROWID_COLUMN_ID) { - return; + return Status::OK(); } DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); DORIS_CHECK(file_schema[local_id] != nullptr); - collect_projected_leaf_column_ids(*file_schema[local_id], projection, leaf_column_ids); + return validate_projected_leaves_supported(*file_schema[local_id], projection); }; for (const auto& column : request.predicate_columns) { - collect_scan_column(column); + RETURN_IF_ERROR(validate_scan_column(column)); } for (const auto& column : request.non_predicate_columns) { - collect_scan_column(column); - } -} - -std::vector build_page_cache_ranges( - const ::parquet::FileMetaData& metadata, - const std::vector>& file_schema, - const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { - std::unordered_set leaf_column_ids; - collect_request_leaf_column_ids(file_schema, request, &leaf_column_ids); - std::vector ranges; - ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); - for (const auto& row_group_plan_item : row_group_plan.row_groups) { - auto row_group_metadata = metadata.RowGroup(row_group_plan_item.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - for (const auto leaf_column_id : leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_chunk_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - DORIS_CHECK(size >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); - } + if (!request.is_count_star_placeholder(column.column_id())) { + RETURN_IF_ERROR(validate_scan_column(column)); } } - return ranges; + return Status::OK(); } const ParquetColumnSchema& projected_root_schema( @@ -155,11 +125,10 @@ const ParquetColumnSchema& projected_root_schema( } int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, - const ParquetColumnReader& shape_reader, - int64_t expected_rows) { - const auto& def_levels = shape_reader.nested_definition_levels(); - const auto& rep_levels = shape_reader.nested_repetition_levels(); - const int64_t levels_written = shape_reader.nested_levels_written(); + const CountColumnReader& shape_reader, int64_t expected_rows) { + const auto& def_levels = shape_reader.definition_levels(); + const auto& rep_levels = shape_reader.repetition_levels(); + const int64_t levels_written = shape_reader.levels_written(); DORIS_CHECK(levels_written >= expected_rows); if (root_schema.max_repetition_level == 0) { DORIS_CHECK(levels_written == expected_rows); @@ -207,7 +176,7 @@ int timestamp_tz_scale(const ParquetTypeDescriptor& type_descriptor) { bool should_map_to_timestamp_tz(const ParquetColumnSchema& column_schema) { const auto& type_descriptor = column_schema.type_descriptor; - return type_descriptor.physical_type == ::parquet::Type::INT96 || + return type_descriptor.physical_type == tparquet::Type::INT96 || (type_descriptor.is_timestamp && type_descriptor.timestamp_is_adjusted_to_utc); } @@ -288,10 +257,9 @@ static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schem } static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { - DORIS_CHECK(column_schema.descriptor != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be // truncated bounds rather than values present in the file, so they are safe for pruning // but cannot be returned as exact aggregate results. @@ -328,14 +296,17 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _enable_mapping_varbinary(enable_mapping_varbinary) {} ParquetReader::~ParquetReader() = default; Status ParquetReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_parquet_profile.total_time); if (_io_ctx != nullptr && _io_ctx->should_stop) { return Status::EndOfFile("stop"); } @@ -356,6 +327,7 @@ Status ParquetReader::init(RuntimeState* state) { _state->enable_strict_mode = state->enable_strict_mode(); _state->scheduler.set_timezone(&state->timezone_obj()); _state->scheduler.set_enable_strict_mode(_state->enable_strict_mode); + _state->scheduler.set_runtime_state(state); } int64_t merge_read_slice_size = -1; if (state != nullptr && state->query_options().__isset.merge_read_slice_size) { @@ -363,16 +335,30 @@ Status ParquetReader::init(RuntimeState* state) { } _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); - // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + // Opening the file parses the footer before any row group can be scheduled. Keep this timer + // around the whole operation so footer/cache latency cannot disappear from a slow profile. + { + SCOPED_TIMER(_parquet_profile.parse_footer_time); + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz, _enable_mapping_varbinary)); + } + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _state->file_context.native_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _state->file_context.native_footer_cache_hits); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier - RETURN_IF_ERROR( - build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); - if (_enable_mapping_timestamp_tz) { - for (auto& column_schema : _state->file_schema) { - apply_timestamp_tz_mapping(column_schema.get()); + { + SCOPED_TIMER(_parquet_profile.parse_meta_time); + RETURN_IF_ERROR(build_parquet_column_schema(_state->file_context.native_metadata->schema(), + &_state->file_schema)); + if (_enable_mapping_timestamp_tz) { + for (auto& column_schema : _state->file_schema) { + apply_timestamp_tz_mapping(column_schema.get()); + } } } return Status::OK(); @@ -386,11 +372,12 @@ void ParquetReader::set_batch_size(size_t batch_size) { } Status ParquetReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_parquet_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } file_schema->clear(); - if (_state == nullptr || _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } @@ -413,8 +400,8 @@ std::unique_ptr ParquetReader::create_column_mapper( } Status ParquetReader::open(std::shared_ptr request) { - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } auto request_snapshot = request; @@ -454,6 +441,12 @@ Status ParquetReader::open(std::shared_ptr request) { DORIS_CHECK(local_id >= 0 && local_id < num_fields); } + // Reject requested unsupported logical leaves before row-group statistics, dictionaries, + // bloom filters or page indexes inspect their physical fallback type. For example, a predicate + // on TIME_MILLIS must fail here even when its INT32 statistics would prune every row group; + // otherwise the same unsupported SELECT could fail or silently succeed depending on data. + RETURN_IF_ERROR(validate_requested_columns_supported(_state->file_schema, *request_snapshot)); + RowGroupScanPlan row_group_plan; ParquetScanRange scan_range; scan_range.start_offset = _file_description->range_start_offset; @@ -461,19 +454,20 @@ Status ParquetReader::open(std::shared_ptr request) { scan_range.file_size = _file_description->file_size; // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). RETURN_IF_ERROR(plan_parquet_row_groups( - *_state->file_context.metadata, _state->file_context.file_reader.get(), - _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, - &row_group_plan, _state->timezone, _state->runtime_state)); + *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, + scan_range, _state->enable_bloom_filter, &row_group_plan, _state->timezone, + _state->runtime_state, &_state->file_context, + _parquet_profile.column_reader_profile())); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } - if (_state->enable_page_cache) { - _state->file_context.register_page_cache_ranges( - build_page_cache_ranges(*_state->file_context.metadata, _state->file_schema, - *request_snapshot, row_group_plan)); - } + // Native page readers admit exact validated page payloads to cache. Do not pre-register whole + // column chunks here: footer offsets are untrusted and this obsolete range map is not consumed. _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); + if (_profile != nullptr) { + _state->scheduler.set_pruning_profile(&_parquet_profile); + } _state->scheduler.set_global_rowid_context(_global_rowid_context); _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); _state->scheduler.set_plan(std::move(row_group_plan)); @@ -482,8 +476,8 @@ Status ParquetReader::open(std::shared_ptr request) { } Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { - if (_state == nullptr || _state->file_context.file_reader == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } *rows = 0; @@ -582,9 +576,9 @@ int64_t ParquetReader::get_total_rows() const { Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_parquet_profile.total_time); DORIS_CHECK(result != nullptr); - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } if (_should_stop()) { @@ -598,12 +592,32 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r request.agg_type); } + // Aggregate pushdown bypasses the scheduler but still requires the exact pruned row-group set. + // Finish lazy remote probes here; normal scans keep them at current-row-group granularity. + RETURN_IF_ERROR(finalize_parquet_row_group_plans( + *_state->file_context.native_metadata, _state->file_schema, *_request, + _state->enable_bloom_filter, &_state->scan_plan, _state->timezone, + _state->runtime_state, &_state->file_context, _parquet_profile.column_reader_profile(), + _profile == nullptr ? nullptr : &_parquet_profile)); + + for (const auto& aggregate_column : request.columns) { + const auto local_id = aggregate_column.projection.local_id(); + if (local_id < 0 || local_id >= static_cast(_state->file_schema.size())) { + return Status::InvalidArgument("Invalid parquet aggregate column id {}", local_id); + } + DORIS_CHECK(_state->file_schema[local_id] != nullptr); + // Aggregate pushdown can return directly from footer statistics without constructing a + // column reader. Validate first so MIN/MAX(TIME_MILLIS), or an all-pruned COUNT request, + // cannot expose the physical INT32 fallback as a supported logical value. + RETURN_IF_ERROR(validate_projected_leaves_supported(*_state->file_schema[local_id], + aggregate_column.projection)); + } + // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - result->count += row_group_metadata->num_rows(); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + result->count += row_group_metadata.num_rows; } if (request.agg_type == TPushAggOp::type::COUNT) { if (request.columns.empty()) { @@ -614,33 +628,26 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } const auto& count_projection = request.columns[0].projection; const auto& root_schema = projected_root_schema(_state->file_schema, count_projection); + // A required primitive COUNT(col) still carries its projection so the unsupported-type + // validation above cannot be bypassed. Once validated, its definition level proves that + // every selected row is non-NULL, so preserve the already-computed footer row count and + // avoid reading definition levels merely to rediscover COUNT(col) == COUNT(*). Complex + // roots continue through the shape reader because their count semantics and read-row + // accounting are derived from nested levels. + if (root_schema.kind == ParquetColumnSchemaKind::PRIMITIVE && + root_schema.max_definition_level == 0) { + return Status::OK(); + } result->count = 0; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - std::shared_ptr<::parquet::RowGroupReader> row_group; - try { - row_group = _state->file_context.file_reader->RowGroup(row_group_plan.row_group_id); - } catch (const ::parquet::ParquetException& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } catch (const std::exception& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } - - ParquetColumnReaderFactory column_reader_factory( - row_group, _state->file_context.schema->num_columns(), - &row_group_plan.page_skip_plans, _parquet_profile.page_skip_profile(), - _state->timezone, _state->enable_strict_mode, - _parquet_profile.scan_profile().column_reader_profile); - std::unique_ptr shape_reader; - RETURN_IF_ERROR(column_reader_factory.create_count_shape_reader( - root_schema, &count_projection, &shape_reader)); + std::unique_ptr shape_reader; + RETURN_IF_ERROR(CountColumnReader::create( + _state->file_context.native_data_file(), _state->file_context.native_metadata, + row_group_plan.row_group_id, root_schema, &count_projection, + _state->file_context.native_io_ctx, + _state->file_context.native_page_cache_enabled, + _state->file_context.native_page_cache_file_key, + _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); int64_t row_group_cursor = 0; @@ -654,18 +661,19 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r while (range_rows_read < selected_range.length) { const int64_t batch_rows = std::min(_batch_size, selected_range.length - range_rows_read); - // COUNT(col) only needs the top-level NULL state. The shape reader loads - // def/rep levels from one representative leaf and does not build value_indices - // or values_column. MAP chooses the key leaf; ARRAY/STRUCT may choose a string - // leaf, but the levels-only protocol still avoids Doris-side string - // materialization for that leaf. + int64_t rows_read = 0; RETURN_IF_ERROR(_stop_status_if_requested( - shape_reader->load_nested_levels_batch(batch_rows))); - _record_scan_rows(batch_rows); + shape_reader->read_levels(batch_rows, &rows_read))); + if (rows_read != batch_rows) { + return Status::Corruption( + "Parquet COUNT reader returned {} rows, expected {}", rows_read, + batch_rows); + } + _record_scan_rows(rows_read); result->count += - count_loaded_non_null_values(root_schema, *shape_reader, batch_rows); - range_rows_read += batch_rows; - row_group_cursor += batch_rows; + count_loaded_non_null_values(root_schema, *shape_reader, rows_read); + range_rows_read += rows_read; + row_group_cursor += rows_read; } } } @@ -692,13 +700,30 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r auto& aggregate_column = result->columns[request_column_idx]; aggregate_column.projection = request.columns[request_column_idx].projection; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - auto column_chunk = row_group_metadata->ColumnChunk(leaf_schema->leaf_column_id); - DORIS_CHECK(column_chunk != nullptr); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + DORIS_CHECK(leaf_schema->leaf_column_id >= 0 && + leaf_schema->leaf_column_id < + static_cast(row_group_metadata.columns.size())); + const auto& column_chunk = row_group_metadata.columns[leaf_schema->leaf_column_id]; + DORIS_CHECK(column_chunk.__isset.meta_data); + const auto& column_metadata = column_chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics && + detail::can_use_native_footer_min_max( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id))) { + safe_statistics = detail::sanitize_native_footer_statistics( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id)); + } const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( - *leaf_schema, column_chunk->statistics(), _state->timezone); + *leaf_schema, safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, _state->timezone); if (!statistics.has_min_max) { return Status::NotSupported("Missing parquet min/max statistics for column {}", leaf_schema->name); @@ -720,7 +745,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } Status ParquetReader::close() { + SCOPED_TIMER(_parquet_profile.total_time); if (_state != nullptr) { + _state->scheduler.close(); _sync_page_cache_profile(); RETURN_IF_ERROR(_state->file_context.close()); } diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..da6135b81ae838 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -46,7 +46,7 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +89,7 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 1abddc777aa442..b92a3c27054ad1 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -15,11 +15,11 @@ #include "format_v2/parquet/parquet_scan.h" -#include - #include +#include #include #include +#include #include #include #include @@ -35,37 +35,93 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" +#include "util/defer_op.h" +#include "util/time.h" namespace doris::format::parquet { -namespace { +namespace detail { + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats) { + if (std::ranges::any_of(positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return positions; + } + auto ordered = positions; + std::stable_sort(ordered.begin(), ordered.end(), [&](size_t left, size_t right) { + const auto score = [&](size_t position) { + const auto& sample = stats.at(position); + return sample.cost_per_input_row_ns / std::max(1.0 - sample.survival_ratio, 0.01); + }; + return score(left) < score(right); + }); + return ordered; +} + +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability) { + if (std::ranges::any_of(ordered_positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return ordered_positions; + } + std::vector result; + double reach_probability = 1; + for (const size_t position : ordered_positions) { + if (!result.empty() && reach_probability < minimum_reach_probability) { + break; + } + result.push_back(position); + reach_probability *= stats.at(position).survival_ratio; + } + return result; +} -int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { + constexpr size_t WARMUP_SAMPLES = 8; + constexpr size_t STEADY_STATE_INTERVAL = 16; + return samples < WARMUP_SAMPLES || batch_sequence % STEADY_STATE_INTERVAL == 0; } -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; +} // namespace detail + +namespace { + +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( + const format::FileScanRequest& request); + +bool is_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; } -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +bool is_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; } -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; +bool is_data_page_type(tparquet::PageType::type page_type) { + return page_type == tparquet::PageType::DATA_PAGE || + page_type == tparquet::PageType::DATA_PAGE_V2; } -bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { +bool is_fully_dictionary_encoded_chunk(const tparquet::ColumnMetaData& column_metadata) { + if (!column_metadata.__isset.dictionary_page_offset || + column_metadata.dictionary_page_offset < 0) { return false; } - const auto& encoding_stats = column_metadata.encoding_stats(); + const auto& encoding_stats = column_metadata.encoding_stats; if (!encoding_stats.empty()) { bool has_dictionary_data_page = false; for (const auto& encoding_stat : encoding_stats) { @@ -81,7 +137,7 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { + for (const auto encoding : column_metadata.encodings) { if (is_dictionary_data_encoding(encoding)) { has_dictionary_encoding = true; continue; @@ -94,19 +150,39 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr || + const tparquet::ColumnMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || column_schema.type == nullptr || column_schema.max_repetition_level > 0) { return false; } - if (!column_schema.type_descriptor.is_string_like || - column_metadata.type() != ::parquet::Type::BYTE_ARRAY) { + bool is_supported_physical_type = false; + switch (column_metadata.type) { + case tparquet::Type::BYTE_ARRAY: + is_supported_physical_type = column_schema.type_descriptor.is_string_like; + break; + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + is_supported_physical_type = true; + break; + case tparquet::Type::BOOLEAN: + // Parquet booleans are PLAIN encoded and cannot have a dictionary page. + break; + } + if (!is_supported_physical_type) { return false; } - // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact - // only when every data page is dictionary encoded. Mixed dictionary/plain chunks are left on - // the normal decoded-value path, matching the safety rule used by StarRocks and Doris v1. + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_VARBINARY) { + // A table STRING predicate can be rewritten through a raw VARBINARY file slot. Evaluating + // it on dictionary Fields before the mapping expression is neither type-safe nor exact. + return false; + } + // The row filter consumes dictionary ids rather than decoded values, so a plain data page + // cannot resume this reader without changing its output domain. Keep mixed chunks on the + // normal decoded-value path to preserve one representation for the complete column chunk. return is_fully_dictionary_encoded_chunk(column_metadata); } @@ -143,49 +219,57 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, } } -bool is_row_group_outside_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, int row_group_idx) { - if (scan_range.size < 0) { - return false; - } - const int64_t range_start_offset = scan_range.start_offset; - const int64_t range_end_offset = range_start_offset + scan_range.size; - DORIS_CHECK(range_start_offset >= 0); - DORIS_CHECK(range_end_offset >= range_start_offset); - if (range_start_offset == 0 && - (scan_range.file_size < 0 || range_end_offset >= scan_range.file_size)) { - return false; - } - - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - DORIS_CHECK(row_group_metadata->num_columns() > 0); - const auto first_column = row_group_metadata->ColumnChunk(0); - const auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); - DORIS_CHECK(first_column != nullptr); - DORIS_CHECK(last_column != nullptr); - const int64_t row_group_start_offset = column_start_offset(*first_column); - const int64_t row_group_end_offset = - column_start_offset(*last_column) + last_column->total_compressed_size(); - const int64_t row_group_mid_offset = - row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; - return row_group_mid_offset < range_start_offset || row_group_mid_offset >= range_end_offset; -} - std::vector request_scan_columns(const format::FileScanRequest& request) { std::vector scan_columns; scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); scan_columns.insert(scan_columns.end(), request.predicate_columns.begin(), request.predicate_columns.end()); - scan_columns.insert(scan_columns.end(), request.non_predicate_columns.begin(), - request.non_predicate_columns.end()); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + scan_columns.push_back(column); + } + } return scan_columns; } -std::vector build_row_group_prefetch_ranges( - const ::parquet::FileMetaData& metadata, +std::vector physical_non_predicate_columns( + const format::FileScanRequest& request) { + std::vector columns; + columns.reserve(request.non_predicate_columns.size()); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + columns.push_back(column); + } + } + return columns; +} + +void materialize_count_star_placeholders(const format::FileScanRequest& request, size_t rows, + Block* file_block) { + DORIS_CHECK(file_block != nullptr); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + continue; + } + const auto block_position = request.local_positions.at(column.column_id()).value(); + auto placeholder = file_block->get_by_position(block_position).column->assert_mutable(); + DCHECK(placeholder->empty()); + placeholder->insert_many_defaults(rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + } +} + +} // namespace + +namespace detail { + +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, - const std::vector& scan_columns, int row_group_idx) { + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); std::unordered_set leaf_column_ids; for (const auto& projection : scan_columns) { const auto local_id = projection.local_id(); @@ -193,92 +277,233 @@ std::vector build_row_group_prefetch_ranges( local_id == format::GLOBAL_ROWID_COLUMN_ID) { continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return Status::Corruption("Invalid Parquet projected column id {}", local_id); + } // Prefetch and merge-reader ranges must be physical leaf chunks, not Doris logical slots. // Example: for a struct column s, projecting only s.a should include only // the Parquet leaf chunk of a. Projecting the whole struct includes both a and b. collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); } - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group index {}", row_group_idx); + } + const auto& row_group_metadata = metadata.row_groups[row_group_idx]; std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); std::ranges::sort(ordered_leaf_column_ids); - std::vector ranges; - ranges.reserve(ordered_leaf_column_ids.size()); + ranges->reserve(ordered_leaf_column_ids.size()); for (const auto leaf_column_id : ordered_leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(row_group_metadata.columns.size())) { + return Status::Corruption("Invalid Parquet leaf column id {}", leaf_column_id); + } + const auto& chunk = row_group_metadata.columns[leaf_column_id]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet leaf column {} has no chunk metadata", + leaf_column_id); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range(chunk.meta_data, file_size, + parquet_816_padding, &chunk_range)); + if (chunk_range.length > 0) { + if (chunk_range.offset > static_cast(std::numeric_limits::max()) || + chunk_range.length > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet column chunk range exceeds int64 coordinates"); + } + // Prefetch must use the same checked chunk extent as the decoder, including the + // PARQUET-816 compatibility padding, so warm-up cannot target different bytes. + ranges->push_back( + ParquetPageCacheRange {.offset = static_cast(chunk_range.offset), + .size = static_cast(chunk_range.length)}); } } - return ranges; + return Status::OK(); } -Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, - std::vector* row_group_first_rows, - std::vector* selected_row_groups) { - DORIS_CHECK(row_group_first_rows != nullptr); - DORIS_CHECK(selected_row_groups != nullptr); - row_group_first_rows->assign(metadata.num_row_groups(), 0); +} // namespace detail + +namespace detail { + +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); + if (scan_range.start_offset < 0 || scan_range.size < -1 || + (scan_range.size >= 0 && + scan_range.start_offset > std::numeric_limits::max() - scan_range.size)) { + return Status::Corruption("Invalid Parquet scan range [{}, {})", scan_range.start_offset, + scan_range.size); + } + const uint64_t range_start = static_cast(scan_range.start_offset); + const uint64_t range_end = scan_range.size < 0 + ? std::numeric_limits::max() + : range_start + static_cast(scan_range.size); + const size_t file_size = scan_range.file_size < 0 ? std::numeric_limits::max() + : static_cast(scan_range.file_size); + const bool full_file_range = + scan_range.size < 0 || (range_start == 0 && scan_range.file_size >= 0 && + range_end >= static_cast(scan_range.file_size)); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + row_group_first_rows->assign(metadata.row_groups.size(), 0); selected_row_groups->clear(); - selected_row_groups->reserve(metadata.num_row_groups()); - int64_t next_row_group_first_row = 0; - for (int row_group_idx = 0; row_group_idx < metadata.num_row_groups(); ++row_group_idx) { - (*row_group_first_rows)[row_group_idx] = next_row_group_first_row; - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows < 0) { + selected_row_groups->reserve(metadata.row_groups.size()); + int64_t next_first_row = 0; + for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_first_row; + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { return Status::Corruption("Invalid negative row count in parquet row group {}", row_group_idx); } - next_row_group_first_row += row_group_rows; - if (!is_row_group_outside_range(metadata, scan_range, row_group_idx)) { - selected_row_groups->push_back(row_group_idx); + if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } + next_first_row += row_group.num_rows; + bool selected = full_file_range; + if (!full_file_range) { + if (row_group.columns.empty()) { + return Status::Corruption("Parquet row group {} has no column chunks", + row_group_idx); + } + size_t group_start = std::numeric_limits::max(); + size_t group_end = 0; + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, file_size, compat.parquet_816_padding, &chunk_range)); + group_start = std::min(group_start, chunk_range.offset); + group_end = std::max(group_end, chunk_range.offset + chunk_range.length); + } + // Checked chunk ranges make end >= start; this midpoint form cannot overflow even + // when footer offsets are close to the host coordinate limit. + const uint64_t group_mid = + static_cast(group_start) + (group_end - group_start) / 2; + selected = group_mid >= range_start && group_mid < range_end; + } + if (selected) { + selected_row_groups->push_back(cast_set(row_group_idx)); } } return Status::OK(); } -Status build_row_group_read_plans( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +} // namespace detail + +namespace { + +std::vector intersect_row_ranges(const std::vector& left, + const std::vector& right) { + std::vector result; + size_t left_idx = 0; + size_t right_idx = 0; + while (left_idx < left.size() && right_idx < right.size()) { + const int64_t left_end = left[left_idx].start + left[left_idx].length; + const int64_t right_end = right[right_idx].start + right[right_idx].length; + const int64_t start = std::max(left[left_idx].start, right[right_idx].start); + const int64_t end = std::min(left_end, right_end); + if (start < end) { + result.push_back({.start = start, .length = end - start}); + } + if (left_end < right_end) { + ++left_idx; + } else { + ++right_idx; + } + } + return result; +} + +Status finalize_native_row_group_read_plan( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, + RowGroupReadPlan* row_group_plan, ParquetPruningStats* pruning_stats, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + bool* selected) { + DORIS_CHECK(row_group_plan != nullptr && pruning_stats != nullptr && file_context != nullptr && + selected != nullptr); + *selected = true; + if (!row_group_plan->expensive_pruning_pending) { + return Status::OK(); + } + row_group_plan->expensive_pruning_pending = false; + const auto& thrift = metadata.to_thrift(); + const std::vector candidate {row_group_plan->row_group_id}; + std::vector metadata_selected; + RETURN_IF_ERROR(select_row_groups_by_metadata( + thrift, file_schema, request, &candidate, &metadata_selected, enable_bloom_filter, + pruning_stats, timezone, runtime_state, file_context, column_reader_profile, + ParquetMetadataProbeMode::EXPENSIVE_ONLY)); + if (metadata_selected.empty()) { + *selected = false; + return Status::OK(); + } + + std::unordered_set requested_leaf_ids; + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; + } + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &requested_leaf_ids); + } + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_plan->row_group_id, requested_leaf_ids, &page_indexes, + &pruning_stats->read_page_index_time, &pruning_stats->parse_page_index_time)); + } + std::vector page_selected_ranges; + std::map page_skip_plans; + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + thrift, page_indexes, file_schema, request, row_group_plan->row_group_rows, + &page_selected_ranges, &page_skip_plans, pruning_stats, timezone, runtime_state)); + row_group_plan->selected_ranges = + intersect_row_ranges(row_group_plan->selected_ranges, page_selected_ranges); + row_group_plan->page_skip_plans = std::move(page_skip_plans); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan->offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); + } + if (row_group_plan->selected_ranges.empty()) { + *selected = false; + return Status::OK(); + } + pruning_stats->selected_row_ranges += row_group_plan->selected_ranges.size(); + return Status::OK(); +} + +Status build_native_row_group_read_plans( + const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector& selected_row_groups, const std::vector& row_group_first_rows, RowGroupScanPlan* plan, - const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + const auto& thrift = metadata.to_thrift(); plan->row_groups.reserve(selected_row_groups.size()); - for (const auto row_group_idx : selected_row_groups) { - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows == 0) { + for (const int row_group_idx : selected_row_groups) { + const auto& row_group = thrift.row_groups[row_group_idx]; + if (row_group.num_rows == 0) { continue; } - RowGroupReadPlan row_group_plan; row_group_plan.row_group_id = row_group_idx; row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; - row_group_plan.row_group_rows = row_group_rows; - RETURN_IF_ERROR(select_row_group_ranges_by_page_index( - file_reader, file_schema, request, row_group_idx, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - if (row_group_plan.selected_ranges.empty()) { - continue; - } - plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + row_group_plan.row_group_rows = row_group.num_rows; + row_group_plan.selected_ranges = {{.start = 0, .length = row_group.num_rows}}; + row_group_plan.expensive_pruning_pending = true; plan->row_groups.push_back(std::move(row_group_plan)); } return Status::OK(); @@ -286,52 +511,58 @@ Status build_row_group_read_plans( } // namespace -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); plan->row_groups.clear(); - plan->pruning_stats = ParquetPruningStats {}; - - // Row-group planning flow: - // - // parquet footer row groups - // | - // v - // split byte-range candidates - // | - // v - // row-group metadata pruning - // statistics/ZoneMap -> dictionary -> bloom filter - // | - // v - // page-index pruning per selected row group - // | - // v - // RowGroupReadPlan with selected row ranges - // - // Metadata pruning removes whole row groups before readers are opened. Page index pruning runs - // only for remaining row groups and produces selected row ranges; the scan scheduler later skips - // gaps between those ranges, while row-level VExpr conjuncts still run on loaded batches for - // correctness. + plan->pruning_stats = {}; + plan->enable_bloom_filter = enable_bloom_filter; std::vector row_group_first_rows; - std::vector scan_range_selected_row_groups; - RETURN_IF_ERROR(select_row_groups_by_scan_range(metadata, scan_range, &row_group_first_rows, - &scan_range_selected_row_groups)); - - std::vector metadata_selected_row_groups; + std::vector scan_range_selected; + RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( + metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); + std::vector metadata_selected; RETURN_IF_ERROR(select_row_groups_by_metadata( - metadata, file_reader, file_schema, request, &scan_range_selected_row_groups, - &metadata_selected_row_groups, enable_bloom_filter, &plan->pruning_stats, timezone, - runtime_state)); + metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, + enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context, + column_reader_profile, ParquetMetadataProbeMode::FOOTER_ONLY)); + RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, + metadata_selected, row_group_first_rows, plan, + timezone, runtime_state, file_context)); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} - RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, - metadata_selected_row_groups, row_group_first_rows, - plan, timezone, runtime_state)); +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + std::vector selected_plans; + selected_plans.reserve(plan->row_groups.size()); + for (auto& row_group_plan : plan->row_groups) { + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + metadata, file_schema, request, enable_bloom_filter, &row_group_plan, + &deferred_stats, timezone, runtime_state, file_context, column_reader_profile, + &selected)); + if (parquet_profile != nullptr) { + parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (selected) { + selected_plans.push_back(std::move(row_group_plan)); + } + } + plan->row_groups = std::move(selected_plans); plan->pruning_stats.selected_row_groups = plan->row_groups.size(); return Status::OK(); } @@ -421,11 +652,12 @@ Status execute_compact_delete_conjuncts(const VExprContextSPtrs& delete_conjunct *can_filter_all = false; for (const auto& delete_conjunct : delete_conjuncts) { DORIS_CHECK(delete_conjunct != nullptr); + const size_t original_columns = file_block->columns(); int result_column_id = -1; RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, &result_column_id)); - DORIS_CHECK(result_column_id >= 0 && - result_column_id < static_cast(file_block->columns())); + RETURN_IF_ERROR(detail::validate_ephemeral_expr_result_column( + original_columns, result_column_id, file_block->columns())); const auto& delete_filter = assert_cast( *file_block->get_by_position(result_column_id).column) .get_data(); @@ -471,11 +703,12 @@ Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t break; } DORIS_CHECK(delete_conjunct != nullptr); + const size_t original_columns = file_block->columns(); int result_column_id = -1; RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, &result_column_id)); - DORIS_CHECK(result_column_id >= 0 && - result_column_id < static_cast(file_block->columns())); + RETURN_IF_ERROR(detail::validate_ephemeral_expr_result_column( + original_columns, result_column_id, file_block->columns())); const auto& delete_filter = assert_cast( *file_block->get_by_position(result_column_id).column) .get_data(); @@ -496,6 +729,19 @@ Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t } // namespace +Status detail::validate_ephemeral_expr_result_column(size_t original_columns, int result_column_id, + size_t current_columns) { + // Delete predicates may erase only a temporary expression result. A bare SlotRef returns an + // input column id, which must remain in the block for later predicates and materialization. + if (UNLIKELY(result_column_id < 0 || static_cast(result_column_id) < original_columns || + static_cast(result_column_id) >= current_columns)) { + return Status::InternalError( + "Delete conjunct result column {} is not ephemeral (original={}, current={})", + result_column_id, original_columns, current_columns); + } + return Status::OK(); +} + uint16_t apply_compact_filter_to_selection(const IColumn::Filter& filter, SelectionVector* selection, uint16_t selected_rows) { DORIS_CHECK(selection != nullptr); @@ -593,6 +839,7 @@ std::vector filter_ranges_by_condition_cache(const std::vectorflush_profile(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + reader->flush_profile(); + } +} + +bool ParquetScanScheduler::finish_current_reader_batch_profiles() { + bool crossed_page = false; + // A scheduler batch is counted once even when several projected leaves cross page boundaries. + for (const auto& reader : _current_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + return crossed_page; +} + +const detail::PredicateConjunctSchedule& ParquetScanScheduler::predicate_conjunct_schedule( + const format::FileScanRequest& request) { + if (_predicate_schedule_request == &request) { + return _predicate_schedule; + } + + // FileScanRequest is frozen by ParquetReader::open(). Its address therefore identifies both + // the conjunct set and local-position mapping for the scheduler lifetime. + _predicate_schedule = build_predicate_conjunct_schedule(request); + _predicate_schedule_request = &request; + _predicate_positions_scratch.clear(); + _predicate_indices_by_position_scratch.clear(); + _predicate_positions_scratch.reserve(request.predicate_columns.size()); + _predicate_indices_by_position_scratch.reserve(request.predicate_columns.size()); + for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + const auto position_it = + request.local_positions.find(request.predicate_columns[idx].column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + _predicate_positions_scratch.push_back(position); + _predicate_indices_by_position_scratch.emplace(position, idx); + } + return _predicate_schedule; +} + +std::vector ParquetScanScheduler::adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const { + std::vector positions; + std::unordered_map columns_by_position; + positions.reserve(request.predicate_columns.size()); + columns_by_position.reserve(request.predicate_columns.size()); + for (const auto& column : request.predicate_columns) { + const auto position_it = request.local_positions.find(column.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + positions.push_back(position); + columns_by_position.emplace(position, &column); + } + auto ordered = detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + ordered = detail::adaptive_prefetch_prefix(ordered, _predicate_runtime_stats, 0.25); + std::vector result; + result.reserve(ordered.size()); + for (const size_t position : ordered) { + result.push_back(*columns_by_position.at(position)); + } + return result; +} + Status ParquetScanScheduler::open_next_row_group( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, bool* has_row_group) { *has_row_group = false; - if (_next_row_group_plan_idx >= _row_group_plans.size()) { + RowGroupReadPlan* selected_plan = nullptr; + while (_next_row_group_plan_idx < _row_group_plans.size()) { + RowGroupReadPlan& candidate_plan = _row_group_plans[_next_row_group_plan_idx++]; + // Probe only the row group about to execute. This keeps LIMIT/cancellation latency + // independent of the number of later remote row groups while preserving eager footer + // statistics pruning during open. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + *file_context.native_metadata, file_schema, request, _enable_bloom_filter, + &candidate_plan, &deferred_stats, _timezone, _runtime_state, &file_context, + _scan_profile.column_reader_profile, &selected)); + if (_parquet_profile != nullptr) { + _parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (!selected) { + continue; + } + selected_plan = &candidate_plan; + break; + } + if (selected_plan == nullptr) { + // The last row group's native readers have already been released by + // reset_current_row_group(). Flush the shared merge reader now so its counters are visible + // when EOF is returned and its bounded scratch does not survive until file close. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; return Status::OK(); } - const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + RowGroupReadPlan& row_group_plan = *selected_plan; const int row_group_idx = row_group_plan.row_group_id; - // Row-level dictionary filters read dictionary pages before Arrow RecordReaders are created. - // Keep that probe on the base reader: MergeRangeFileReader expects each registered range to be - // consumed as one forward pass, while the later RecordReader opens the same column chunk again - // for the data-page stream. + // Dictionary probes and data-page readers share the native metadata tree. Reset the previous + // row-group merge reader before probing because dictionary-page offsets are not scan ordered. file_context.reset_random_access_ranges(); _current_merge_range_active = false; - try { - _current_row_group = file_context.file_reader->RowGroup(row_group_idx); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } - - auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - _current_row_group_rows = row_group_metadata->num_rows(); + + const auto& row_group_metadata = + file_context.native_metadata->to_thrift().row_groups[row_group_idx]; + _current_row_group_rows = row_group_metadata.num_rows; DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); DORIS_CHECK(_current_row_group_rows > 0); _current_row_group_id = row_group_idx; + _has_current_row_group = true; DORIS_CHECK(!row_group_plan.selected_ranges.empty()); _current_row_group_first_row = row_group_plan.first_file_row; _current_row_group_rows_read = 0; _current_selected_ranges = row_group_plan.selected_ranges; + _current_offset_indexes = std::move(row_group_plan.offset_indexes); + // Condition Cache and split planning can narrow logical ranges without a physical OffsetIndex. + // Native readers must keep the sequential level/value cursor path valid in that case; only a + // PageIndex-derived skip plan requires the transferred indexes below. + for (const auto& [leaf_column_id, skip_plan] : row_group_plan.page_skip_plans) { + if (!_current_offset_indexes.contains(leaf_column_id)) { + continue; + } + for (size_t page = 0; page < skip_plan.skipped_pages.size(); ++page) { + if (!skip_plan.should_skip_page(page)) { + continue; + } + if (_page_skip_profile.skipped_pages != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); + } + if (_page_skip_profile.skipped_bytes != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_bytes, + skip_plan.skipped_page_compressed_size(page)); + } + } + } _current_range_idx = 0; _current_range_rows_read = 0; _current_predicate_columns.clear(); _current_non_predicate_columns.clear(); _current_dictionary_filters.clear(); RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, - row_group_idx, *row_group_metadata)); - _current_merge_range_active = - prepare_current_row_group_reader(file_context, file_schema, request, row_group_idx); - - ParquetColumnReaderFactory column_reader_factory( - _current_row_group, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, - _page_skip_profile, _timezone, _enable_strict_mode, - _scan_profile.column_reader_profile); + row_group_idx, row_group_metadata)); + // Dictionary probing is complete, so the native data-page readers can now share the same + // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a + // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy + // materialization for wide schemas. + const auto& thrift_metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : std::string {}); + std::vector native_ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + thrift_metadata, file_schema, request_scan_columns(request), row_group_idx, + file_context.native_file->size(), compat.parquet_816_padding, &native_ranges)); + _current_merge_range_active = file_context.set_native_random_access_ranges( + native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, + _merge_read_slice_size); + for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + const auto local_id = col.column_id(); + if (_current_predicate_columns.contains(local_id)) { continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); + continue; + } + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR( - column_reader_factory.create(*column_schema, &col, &column_reader, - _current_dictionary_filters.contains(local_id))); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, + _current_dictionary_filters.contains(local_id), _scan_profile.column_reader_profile, + &column_reader)); _current_predicate_columns[local_id] = std::move(column_reader); } - // Start warming filter-column chunks as soon as their row group is selected. Parquet v2 still - // reads through Arrow's random-access reader; this prefetch only warms Doris file cache blocks - // in the background and never changes the row/column materialization order. + // Start warming filter-column chunks as soon as their row group is selected. The native + // BufferedFileStreamReader later consumes the same Doris file-cache blocks; prefetch never + // changes row/column materialization order. if (!_current_merge_range_active) { - prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, - &_current_predicate_prefetched); + const auto prefetch_columns = adaptive_predicate_prefetch_columns(request); + RETURN_IF_ERROR(prefetch_current_row_group_columns( + file_context, file_schema, prefetch_columns, &_current_predicate_prefetched)); } for (const auto& col : request.non_predicate_columns) { - const auto local_id = col.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_non_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + const auto local_id = col.column_id(); + if (request.is_count_star_placeholder(col.column_id())) { continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_non_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); + continue; + } + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_non_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_non_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR(column_reader_factory.create(*column_schema, &col, &column_reader)); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, false, _scan_profile.column_reader_profile, + &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } - if (!_current_merge_range_active && request.conjuncts.empty() && - request.delete_conjuncts.empty()) { + if (!_current_merge_range_active && + ((request.conjuncts.empty() && request.delete_conjuncts.empty()) || + _predicate_survival_ratio >= 0.8)) { // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. - prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } *has_row_group = true; return Status::OK(); @@ -815,12 +1214,7 @@ Status ParquetScanScheduler::flush_pending_non_predicate_skip_rows() { namespace { -struct PredicateConjunctSchedule { - std::map single_column_conjuncts; - VExprContextSPtrs remaining_conjuncts; -}; - -PredicateConjunctSchedule build_predicate_conjunct_schedule( +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request) { std::unordered_set predicate_block_positions; predicate_block_positions.reserve(request.predicate_columns.size()); @@ -830,7 +1224,7 @@ PredicateConjunctSchedule build_predicate_conjunct_schedule( predicate_block_positions.insert(position_it->second.value()); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; for (const auto& conjunct : request.conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); @@ -928,21 +1322,11 @@ uint16_t count_selected_rows(const IColumn::Filter& filter) { return selected_rows; } -Status filter_read_predicate_columns(Block* file_block, const std::vector& positions, - const IColumn::Filter& compact_filter) { - if (positions.empty()) { - return Status::OK(); - } - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, positions, compact_filter)); - return Status::OK(); -} - IColumn::Filter build_dictionary_entry_filter(size_t block_position, const ParquetColumnSchema& column_schema, const VExprContextSPtrs& conjuncts, - const ParquetDictionaryWords& dict_words) { - auto fields = dictionary_fields_from_words(dict_words); - IColumn::Filter dictionary_filter(fields.size(), 1); + const IColumn& dictionary) { + IColumn::Filter dictionary_filter(dictionary.size(), 1); DictionaryEvalContext ctx; auto& slot = ctx.slots .emplace(static_cast(block_position), @@ -950,14 +1334,15 @@ IColumn::Filter build_dictionary_entry_filter(size_t block_position, .data_type = column_schema.type, .values = {}}) .first->second; slot.values.reserve(1); - - for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) { + for (size_t dictionary_id = 0; dictionary_id < dictionary.size(); ++dictionary_id) { + Field value; + dictionary.get(dictionary_id, value); slot.values.clear(); - slot.values.push_back(fields[dict_idx]); - dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch - ? 0 - : 1; + slot.values.push_back(std::move(value)); + dictionary_filter[dictionary_id] = VExprContext::evaluate_dictionary_filter( + conjuncts, ctx) == ZoneMapFilterResult::kNoMatch + ? 0 + : 1; } return dictionary_filter; } @@ -968,16 +1353,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata) { + const tparquet::RowGroup& row_group_metadata) { _current_dictionary_filters.clear(); _current_dictionary_residual_conjuncts.clear(); if (request.conjuncts.empty()) { return Status::OK(); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; { SCOPED_TIMER(_scan_profile.dict_filter_expr_rewrite_time); - schedule = build_predicate_conjunct_schedule(request); + schedule = predicate_conjunct_schedule(request); } if (schedule.single_column_conjuncts.empty()) { return Status::OK(); @@ -985,8 +1370,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( SCOPED_TIMER(_scan_profile.dict_filter_rewrite_time); for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + const auto local_id = col.column_id(); + if (!local_id.is_valid() || local_id.value() >= static_cast(file_schema.size())) { continue; } const auto position_it = request.local_positions.find(col.column_id()); @@ -1002,29 +1387,38 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // This optimization is deliberately limited to single-column predicates with a dictionary // evaluable part. Mixed AND predicates are split so dictionary-covered children run as a // dict-id prefilter and residual children keep the normal row-level expression path. - const auto& column_schema = file_schema[local_id]; + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); if (column_schema->leaf_column_id < 0 || - column_schema->leaf_column_id >= row_group_metadata.num_columns()) { + column_schema->leaf_column_id >= static_cast(row_group_metadata.columns.size())) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - auto column_chunk = row_group_metadata.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_row_level_dictionary_filter(*column_schema, *column_chunk)) { + const auto& column_chunk = row_group_metadata.columns[column_schema->leaf_column_id]; + if (!column_chunk.__isset.meta_data || + !supports_row_level_dictionary_filter(*column_schema, column_chunk.meta_data)) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - ParquetDictionaryWords dict_words; + std::unique_ptr column_reader; + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_file, file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, true, _scan_profile.column_reader_profile, + &column_reader)); + MutableColumnPtr dictionary_values; { SCOPED_TIMER(_scan_profile.dict_filter_read_dict_time); - if (!read_dictionary_words(file_context.file_reader.get(), row_group_idx, - column_schema->leaf_column_id, *column_schema, - &dict_words)) { + auto dictionary_result = column_reader->dictionary_values(); + if (!dictionary_result.has_value()) { update_counter_if_not_null(_scan_profile.dict_filter_read_failures, 1); + // Dictionary filtering is optional: a probe failure must not reject a file that + // the normal native read path can still decode. continue; } + dictionary_values = std::move(dictionary_result).value(); } // Build a safe dictionary prefilter from the dictionary-filter interface instead of @@ -1035,15 +1429,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( DictionaryResidualConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema, - conjunct_it->second, dict_words); + dictionary_filter = build_dictionary_entry_filter( + block_position, *column_schema, conjunct_it->second, *dictionary_values); residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } // The bitmap is keyed by Parquet dictionary id. Later data-page reads evaluate the - // predicate with an integer lookup and only materialize STRING values for surviving rows. + // predicate with an integer lookup and materialize typed values only for surviving rows. _current_dictionary_filters.emplace(local_id, std::move(dictionary_filter)); _current_dictionary_residual_conjuncts.emplace(local_id, std::move(residual_conjuncts)); + _current_predicate_columns.emplace(local_id, std::move(column_reader)); update_counter_if_not_null(_scan_profile.dict_filter_columns, 1); } return Status::OK(); @@ -1060,16 +1455,122 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, if (!request.conjuncts.empty() || !request.delete_conjuncts.empty()) { selection->resize(static_cast(batch_rows)); } - const auto schedule = build_predicate_conjunct_schedule(request); + const auto& schedule = predicate_conjunct_schedule(request); + std::unordered_set residual_predicate_positions; + auto remember_residual_positions = [&](const VExprContextSPtrs& conjuncts) { + for (const auto& conjunct : conjuncts) { + std::set positions; + conjunct->root()->collect_slot_column_ids(positions); + for (const int position : positions) { + if (position >= 0) { + residual_predicate_positions.insert(cast_set(position)); + } + } + } + }; + remember_residual_positions(schedule.remaining_conjuncts); + remember_residual_positions(request.delete_conjuncts); + const size_t predicate_batch_sequence = _predicate_batch_sequence++; const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); - std::vector read_column_positions; + auto& read_column_positions = _read_column_positions_scratch; + read_column_positions.clear(); read_column_positions.reserve(request.predicate_columns.size()); + for (auto& rows : _predicate_column_selection_scratch | std::views::values) { + rows.clear(); + } - auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, - ColumnId local_id, bool* used_dictionary_filter) -> Status { + auto remember_column_selection = [&](uint32_t position) { + auto& rows = _predicate_column_selection_scratch[position]; + rows.resize(*selected_rows); + for (uint16_t row = 0; row < *selected_rows; ++row) { + // SelectionVector and the scanner batch contract both bound row ordinals to uint16_t; + // keep the checked conversion explicit when persisting the coordinate mapping. + rows[row] = cast_set(selection->get_index(row)); + } + }; + + auto compact_predicate_columns = [&](bool discard_predicate_only_payload) -> Status { + bool compacted = false; + int64_t compacted_bytes = 0; + for (const uint32_t position : read_column_positions) { + auto& source_rows = _predicate_column_selection_scratch[position]; + const auto& old_column = file_block->get_by_position(position).column; + if (old_column->size() != source_rows.size()) { + return Status::Corruption( + "Predicate column {} has {} values but {} remembered source rows", position, + old_column->size(), source_rows.size()); + } + bool predicate_only = false; + if (discard_predicate_only_payload) { + predicate_only = std::ranges::any_of( + request.predicate_only_columns, [&](format::LocalColumnId local_id) { + const auto position_it = request.local_positions.find(local_id); + return position_it != request.local_positions.end() && + position_it->second.value() == position; + }); + } + if (predicate_only) { + auto placeholder = old_column->clone_empty(); + // Hidden predicate values are dead after the last filter, but every file-block + // column must retain the selected row count until TableReader drops hidden slots. + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(position, std::move(placeholder)); + remember_column_selection(position); + continue; + } + bool already_compact = source_rows.size() == *selected_rows && + old_column->size() == static_cast(*selected_rows); + for (uint16_t row = 0; already_compact && row < *selected_rows; ++row) { + already_compact = source_rows[row] == selection->get_index(row); + } + if (already_compact) { + continue; + } + auto& filter = _predicate_compaction_filter_scratch; + // resize_fill() preserves bytes when the next predicate column is smaller. Clear the + // whole reusable mask so survivors from an earlier coordinate space cannot reappear. + filter.resize(source_rows.size()); + std::ranges::fill(filter, 0); + size_t source_idx = 0; + uint16_t selected_idx = 0; + while (source_idx < source_rows.size() && selected_idx < *selected_rows) { + const auto source_row = source_rows[source_idx]; + const auto selected_row = selection->get_index(selected_idx); + if (source_row < selected_row) { + ++source_idx; + continue; + } + DORIS_CHECK_EQ(source_row, selected_row); + filter[source_idx++] = 1; + ++selected_idx; + } + DORIS_CHECK_EQ(selected_idx, *selected_rows); + compacted_bytes += static_cast(old_column->byte_size()); + RETURN_IF_CATCH_EXCEPTION(file_block->replace_by_position( + position, old_column->filter(filter, *selected_rows))); + remember_column_selection(position); + compacted = true; + } + if (compacted) { + update_counter_if_not_null(_scan_profile.predicate_compaction_bytes, compacted_bytes); + update_counter_if_not_null(_scan_profile.predicate_compaction_count, 1); + } + // The output path must not apply a batch-coordinate filter to columns that now use compact + // coordinates. The loop above establishes this invariant even when no bytes moved because + // every column was already aligned. + *predicate_columns_filtered = !read_column_positions.empty(); + return Status::OK(); + }; + + auto read_predicate_column = + [&](ParquetColumnReader* column_reader, size_t block_position, + format::LocalColumnId local_id, const VExprContextSPtrs* single_column_conjuncts, + bool* used_dictionary_filter, bool* used_fixed_width_filter) -> Status { DORIS_CHECK(used_dictionary_filter != nullptr); + DORIS_CHECK(used_fixed_width_filter != nullptr); *used_dictionary_filter = false; + *used_fixed_width_filter = false; DCHECK(remove_nullable(column_reader->type()) ->equals(*remove_nullable(file_block->get_by_position(block_position).type))) << column_reader->type()->get_name() << " " @@ -1096,22 +1597,72 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, update_counter_if_not_null(_scan_profile.rows_filtered_by_dict_filter, filtered_rows); if (new_selected_rows != selected_rows_before) { - // The dictionary reader has already appended only surviving values for the - // current column. Apply the compact row filter only to columns read before this - // one, then update the shared selection for later predicate/lazy columns. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); + // The dictionary reader already appended only survivors for this column. Keep + // older predicate columns in their original coordinate spaces and compact all + // of them once at the expression/output boundary below. *selected_rows = apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); *used_dictionary_filter = true; return Status::OK(); } } + if (single_column_conjuncts != nullptr && + !residual_predicate_positions.contains(block_position)) { + VExprSPtrs direct_conjuncts; + direct_conjuncts.reserve(single_column_conjuncts->size()); + std::ranges::transform(*single_column_conjuncts, std::back_inserter(direct_conjuncts), + [](const auto& context) { return context->root(); }); + if (!direct_conjuncts.empty()) { + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool used_filter = false; + const bool predicate_only = request.is_predicate_only(local_id); + // The raw decoder cannot rewind after evaluating encoded fixed-width values. + // Project survivors in that pass when output still needs the predicate column. + IColumn* projected_column = predicate_only ? nullptr : column.get(); + RETURN_IF_ERROR(column_reader->select_with_fixed_width_filter( + *selection, *selected_rows, batch_rows, direct_conjuncts, + cast_set(block_position), projected_column, &compact_filter, + &used_filter)); + if (used_filter) { + DORIS_CHECK_EQ(compact_filter.size(), selected_rows_before); + update_counter_if_not_null(_scan_profile.fixed_width_predicate_direct_batches, + 1); + update_counter_if_not_null(_scan_profile.fixed_width_predicate_direct_rows, + selected_rows_before); + const uint16_t new_selected_rows = count_selected_rows(compact_filter); + const auto filtered_rows = static_cast(selected_rows_before) - + static_cast(new_selected_rows); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += filtered_rows; + } + if (new_selected_rows != selected_rows_before) { + *selected_rows = apply_compact_filter_to_selection( + compact_filter, selection, selected_rows_before); + } + if (predicate_only) { + // This slot is absent from every residual/delete conjunct, so no later + // expression can observe its payload. Keep only the block row-shape contract. + auto placeholder = column->clone_empty(); + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + } else { + file_block->replace_by_position(block_position, std::move(column)); + } + read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); + *predicate_columns_filtered = true; + *used_fixed_width_filter = true; + return Status::OK(); + } + } + } + if (*selected_rows == batch_rows) { int64_t column_rows = 0; RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); @@ -1132,6 +1683,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); return Status::OK(); }; @@ -1153,16 +1705,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // All columns read so far are already compacted to the current selection. Apply the - // compact filter to those columns and the selection vector together, so later predicate - // columns can read only rows that survived previous predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1186,16 +1732,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // Dictionary-covered children have already reduced the compact block. Apply only the - // residual child filters here, then keep the same compacted-column invariant as the - // normal conjunct path for later predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1232,24 +1772,23 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, compact_filter.resize_fill(selected_rows_before, 0); } if (can_filter_all || count_selected_rows(compact_filter) != selected_rows_before) { - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; auto read_all_predicate_columns = [&]() -> Status { for (const auto& [fid, column_reader] : _current_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); bool used_dictionary_filter = false; + bool used_fixed_width_filter = false; RETURN_IF_ERROR(read_predicate_column(column_reader.get(), position_it->second.value(), - fid, &used_dictionary_filter)); + fid, nullptr, &used_dictionary_filter, + &used_fixed_width_filter)); } return Status::OK(); }; @@ -1269,45 +1808,69 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, // Single-column conjuncts can be evaluated immediately after their column is read. Once // selection shrinks, later predicate columns use ParquetColumnReader::select() so the // reader skips rows already rejected by earlier predicates instead of materializing them. - for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + _ordered_predicate_positions_scratch = detail::order_adaptive_predicates( + _predicate_positions_scratch, _predicate_runtime_stats); + const auto& ordered_positions = _ordered_predicate_positions_scratch; + for (size_t order_idx = 0; order_idx < ordered_positions.size(); ++order_idx) { + const size_t position = ordered_positions[order_idx]; + const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; - const auto fid = col.local_id(); + const auto fid = col.column_id(); auto reader_it = _current_predicate_columns.find(fid); DORIS_CHECK(reader_it != _current_predicate_columns.end()); auto position_it = request.local_positions.find(col.column_id()); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); + const uint16_t rows_before = *selected_rows; + auto& stats = _predicate_runtime_stats[position]; + const bool sample = detail::should_sample_adaptive_predicate(stats.samples, + predicate_batch_sequence); + const int64_t start_ns = sample ? MonotonicNanos() : 0; bool used_dictionary_filter = false; + bool used_fixed_width_filter = false; + const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + const VExprContextSPtrs* column_conjuncts = + conjunct_it == schedule.single_column_conjuncts.end() ? nullptr + : &conjunct_it->second; RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, - &used_dictionary_filter)); - if (*selected_rows == 0) { - for (size_t remaining_idx = idx + 1; - remaining_idx < request.predicate_columns.size(); ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); - auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); - DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); - RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); + column_conjuncts, &used_dictionary_filter, + &used_fixed_width_filter)); + if (*selected_rows != 0 && conjunct_it != schedule.single_column_conjuncts.end()) { + if (used_dictionary_filter) { + const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); + DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); + RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( + residual_it->second)); + } else if (!used_fixed_width_filter) { + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); } - return Status::OK(); } - const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); - if (conjunct_it == schedule.single_column_conjuncts.end()) { - continue; - } - if (used_dictionary_filter) { - const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); - DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); - RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( - residual_it->second)); - } else { - RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + if (sample) { + const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / + std::max(rows_before, 1); + const double survival = + static_cast(*selected_rows) / std::max(rows_before, 1); + constexpr double ADAPTIVE_ALPHA = 0.25; + if (stats.samples == 0) { + stats.cost_per_input_row_ns = cost_per_row; + stats.survival_ratio = survival; + } else { + stats.cost_per_input_row_ns = + ADAPTIVE_ALPHA * cost_per_row + + (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; + stats.survival_ratio = + ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; + } + ++stats.samples; } if (*selected_rows != 0) { continue; } - for (size_t remaining_idx = idx + 1; remaining_idx < request.predicate_columns.size(); - ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + for (size_t remaining_order_idx = order_idx + 1; + remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { + const size_t remaining_idx = _predicate_indices_by_position_scratch.at( + ordered_positions[remaining_order_idx]); + const auto remaining_fid = request.predicate_columns[remaining_idx].column_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); @@ -1317,47 +1880,59 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; + auto compact_predicate_columns_with_profile = + [&](bool discard_predicate_only_payload) -> Status { + const int64_t start_ns = MonotonicNanos(); + auto status = compact_predicate_columns(discard_predicate_only_payload); + update_counter_if_not_null(_scan_profile.predicate_compaction_time, + MonotonicNanos() - start_ns); + return status; + }; + RETURN_IF_ERROR(read_round_by_round()); + // Single-column expressions only touch the just-read column, so earlier columns can retain + // their own row mappings. Compact only when a later expression needs a shared coordinate + // space; otherwise the final boundary can discard hidden predicate payloads without scanning + // them again. + if (!schedule.remaining_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); + } RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); - if (_scan_profile.predicate_filter_time == nullptr) { - return execute_scheduled_delete_conjuncts(); + if (!request.delete_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); } - SCOPED_TIMER(_scan_profile.predicate_filter_time); - return execute_scheduled_delete_conjuncts(); -} - -bool ParquetScanScheduler::prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx) { - if (file_context.metadata == nullptr) { - return false; + if (_scan_profile.predicate_filter_time == nullptr) { + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); + } else { + SCOPED_TIMER(_scan_profile.predicate_filter_time); + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } - const auto ranges = build_row_group_prefetch_ranges( - *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); - const size_t avg_io_size = detail::average_prefetch_range_size(ranges); - return file_context.set_random_access_ranges(ranges, avg_io_size, _profile, - _merge_read_slice_size); + return compact_predicate_columns_with_profile(true); } -void ParquetScanScheduler::prefetch_current_row_group_columns( +Status ParquetScanScheduler::prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched) { DORIS_CHECK(prefetched != nullptr); if (_current_merge_range_active || *prefetched || scan_columns.empty() || - _current_row_group_id < 0 || file_context.metadata == nullptr) { - return; + _current_row_group_id < 0 || file_context.native_metadata == nullptr) { + return Status::OK(); } *prefetched = true; // The scanner request separates predicate and non-predicate columns so Parquet can read // predicate columns first and lazily materialize the rest. Keep the same contract for // prefetch: callers decide which side to warm, and this helper only translates that selected // projection into physical column-chunk byte ranges for the current row group. - file_context.prefetch_ranges( - build_row_group_prefetch_ranges(*file_context.metadata, file_schema, scan_columns, - _current_row_group_id), - nullptr); + const auto& metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + std::vector ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + metadata, file_schema, scan_columns, _current_row_group_id, + file_context.native_file->size(), compat.parquet_816_padding, &ranges)); + file_context.prefetch_ranges(ranges, nullptr); + return Status::OK(); } Status ParquetScanScheduler::read_current_row_group_batch( @@ -1365,6 +1940,24 @@ Status ParquetScanScheduler::read_current_row_group_batch( const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows) { + // Reader statistics are cumulative plain integers. Publishing their delta recursively for + // every tiny batch is measurable on wide/nested scans, so flush periodically and force the + // tail at row-group reset/close. + Defer profile_flush {[this, batch_rows]() { + // A widened predicate batch can be emitted in several output slices. Its lazy readers + // have not consumed the whole physical batch until the last slice is drained. + if (_pending_predicate_selection.empty() && finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + const bool finishes_row_group = _current_range_idx + 1 == _current_selected_ranges.size() && + _current_range_rows_read + batch_rows == + _current_selected_ranges[_current_range_idx].length; + if (++_batches_since_profile_flush >= PROFILE_FLUSH_BATCH_INTERVAL || finishes_row_group) { + flush_current_reader_profiles(); + _batches_since_profile_flush = 0; + } + }}; if (_scan_profile.total_batches != nullptr) { COUNTER_UPDATE(_scan_profile.total_batches, 1); } @@ -1374,12 +1967,13 @@ Status ParquetScanScheduler::read_current_row_group_batch( _raw_rows_read += batch_rows; if (_current_predicate_columns.empty() && _current_non_predicate_columns.empty()) { *rows = static_cast(batch_rows); + materialize_count_star_placeholders(request, *rows, file_block); if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, batch_rows); } return Status::OK(); } - SelectionVector selection; + auto& selection = _selection; DORIS_CHECK(batch_rows <= std::numeric_limits::max()); uint16_t selected_rows = static_cast(batch_rows); int64_t conjunct_filtered_rows = 0; @@ -1390,6 +1984,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( mark_condition_cache_granules(selection, selected_rows, batch_first_file_row); const bool need_filter_output = selected_rows != batch_rows; + const double batch_survival = static_cast(selected_rows) / batch_rows; + _predicate_survival_ratio = _predicate_survival_ratio < 0 + ? batch_survival + : 0.25 * batch_survival + 0.75 * _predicate_survival_ratio; if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, selected_rows); } @@ -1402,6 +2000,11 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0 && _scan_profile.empty_selection_batches != nullptr) { COUNTER_UPDATE(_scan_profile.empty_selection_batches, 1); + } else if (static_cast(selected_rows) == batch_rows && + _scan_profile.dense_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.dense_batches, 1); + } else if (_scan_profile.selected_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_batches, 1); } if (need_filter_output && !predicate_columns_filtered) { IColumn::Filter output_filter = selection_to_filter(selection, selected_rows, batch_rows); @@ -1416,7 +2019,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0) { // Predicate readers have consumed this physical batch, but touching every lazy column here - // turns a long rejected prefix into `empty_batches * lazy_columns` Arrow calls. Record only + // turns a long rejected prefix into `empty_batches * lazy_columns` native calls. Record only // the positional lag. If [0, 32), [32, 64), and [64, 96) are empty, the first surviving // batch performs one skip(96) per lazy column. If the row group ends instead, reset drops the // lazy readers without flushing because no value from them can be observed. @@ -1431,8 +2034,30 @@ Status ParquetScanScheduler::read_current_row_group_batch( // Do not prefetch lazy output columns until at least one row survives filtering. This is // the same decision point where the v2 reader switches from predicate-only reads to // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. - prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); + } + + if (selected_rows > _batch_size) { + DORIS_CHECK(_pending_predicate_selection.empty()); + _pending_predicate_batch_rows = batch_rows; + _pending_predicate_batch_rows_consumed = 0; + _pending_predicate_selected_offset = 0; + _pending_predicate_selection.resize(selected_rows); + for (uint16_t idx = 0; idx < selected_rows; ++idx) { + _pending_predicate_selection[idx] = + static_cast(selection.get_index(idx)); + } + for (const auto& col : request.predicate_columns) { + const auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t block_position = position_it->second.value(); + const auto& column = file_block->get_by_position(block_position).column; + DORIS_CHECK_EQ(column->size(), selected_rows); + _pending_predicate_columns.emplace(block_position, column); + } + return materialize_pending_predicate_batch(request, file_block, rows); } { @@ -1441,7 +2066,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( // selection vector. This also merges pending range gaps with fully filtered batches. RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); for (const auto& [fid, column_reader] : _current_non_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); auto column = file_block->get_by_position(block_position).column->assert_mutable(); @@ -1472,10 +2097,81 @@ Status ParquetScanScheduler::read_current_row_group_batch( file_block->replace_by_position(block_position, std::move(column)); } } + materialize_count_star_placeholders(request, selected_rows, file_block); *rows = static_cast(selected_rows); return Status::OK(); } +Status ParquetScanScheduler::materialize_pending_predicate_batch( + const format::FileScanRequest& request, Block* file_block, size_t* rows) { + DORIS_CHECK(!_pending_predicate_selection.empty()); + DORIS_CHECK(_pending_predicate_selected_offset < _pending_predicate_selection.size()); + const size_t remaining_selected = + _pending_predicate_selection.size() - _pending_predicate_selected_offset; + const size_t output_rows = + std::min(static_cast(_batch_size), remaining_selected); + const size_t output_end = _pending_predicate_selected_offset + output_rows; + const int64_t physical_end = + output_end == _pending_predicate_selection.size() + ? _pending_predicate_batch_rows + : static_cast(_pending_predicate_selection[output_end - 1]) + 1; + DORIS_CHECK(physical_end > _pending_predicate_batch_rows_consumed); + const int64_t physical_rows = physical_end - _pending_predicate_batch_rows_consumed; + + _pending_output_selection.resize(output_rows); + for (size_t idx = 0; idx < output_rows; ++idx) { + const int64_t physical_row = + _pending_predicate_selection[_pending_predicate_selected_offset + idx]; + DORIS_CHECK(physical_row >= _pending_predicate_batch_rows_consumed); + _pending_output_selection.set_index( + idx, static_cast(physical_row - + _pending_predicate_batch_rows_consumed)); + } + + for (const auto& [block_position, column] : _pending_predicate_columns) { + file_block->replace_by_position( + block_position, column->cut(_pending_predicate_selected_offset, output_rows)); + } + { + SCOPED_TIMER(_scan_profile.column_read_time); + RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); + for (const auto& [fid, column_reader] : _current_non_predicate_columns) { + auto position_it = request.local_positions.find(fid); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = position_it->second.value(); + auto column = file_block->get_by_position(block_position).column->assert_mutable(); + [[maybe_unused]] const auto old_size = column->size(); + RETURN_IF_ERROR(column_reader->select(_pending_output_selection, + static_cast(output_rows), physical_rows, + column)); + if (column->size() != old_size + output_rows) { + return Status::Corruption( + "Parquet pending output column {} returned {} rows, expected {} rows", + column_reader->name(), column->size(), old_size + output_rows); + } + file_block->replace_by_position(block_position, std::move(column)); + } + } + materialize_count_star_placeholders(request, output_rows, file_block); + *rows = output_rows; + _pending_predicate_batch_rows_consumed = physical_end; + _pending_predicate_selected_offset = output_end; + if (_pending_predicate_selected_offset == _pending_predicate_selection.size()) { + DORIS_CHECK_EQ(_pending_predicate_batch_rows_consumed, _pending_predicate_batch_rows); + if (finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + _pending_predicate_batch_rows = 0; + _pending_predicate_batch_rows_consumed = 0; + _pending_predicate_selected_offset = 0; + _pending_predicate_selection.clear(); + _pending_predicate_columns.clear(); + _pending_output_selection.clear(); + } + return Status::OK(); +} + void ParquetScanScheduler::mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_first_file_row) { @@ -1499,8 +2195,28 @@ Status ParquetScanScheduler::read_next_batch( const std::vector>& file_schema, const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { *rows = 0; + if (!_pending_predicate_selection.empty()) { + RETURN_IF_ERROR(materialize_pending_predicate_batch(request, file_block, rows)); + *eof = false; + return Status::OK(); + } + int64_t predicate_batch_rows = _batch_size; + const int64_t max_predicate_batch_rows = std::min( + std::numeric_limits::max(), + std::max(DEFAULT_READ_BATCH_SIZE, _runtime_state == nullptr + ? DEFAULT_READ_BATCH_SIZE + : _runtime_state->batch_size())); + auto grow_empty_predicate_batch = [max_predicate_batch_rows](int64_t current) { + for (const int64_t target : + {int64_t {256}, int64_t {1024}, int64_t {4096}, max_predicate_batch_rows}) { + if (current < target) { + return std::min(target, max_predicate_batch_rows); + } + } + return max_predicate_batch_rows; + }; while (true) { - if (_current_row_group == nullptr) { + if (!_has_current_row_group) { bool has_row_group = false; RETURN_IF_ERROR( open_next_row_group(file_context, file_schema, request, &has_row_group)); @@ -1535,7 +2251,7 @@ Status ParquetScanScheduler::read_next_batch( continue; } - const int64_t batch_rows = std::min(_batch_size, remaining_rows); + const int64_t batch_rows = std::min(predicate_batch_rows, remaining_rows); const int64_t physical_rows_read = batch_rows; const int64_t batch_first_file_row = _current_row_group_first_row + _current_row_group_rows_read; @@ -1548,6 +2264,10 @@ Status ParquetScanScheduler::read_next_batch( _current_range_rows_read = 0; } if (*rows == 0) { + // Fully rejected probes carry no output-width sample. Widen predicate work to cross + // long empty prefixes cheaply; a later non-empty probe is sliced before lazy columns + // are materialized, so this internal width cannot escape the caller's row cap. + predicate_batch_rows = grow_empty_predicate_batch(predicate_batch_rows); continue; } *eof = false; diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index c656b146cefeaa..ac883cbbcbe099 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include @@ -33,13 +36,6 @@ #include "runtime/runtime_profile.h" #include "storage/segment/condition_cache.h" -namespace parquet { -class FileMetaData; -class ParquetFileReader; -class RowGroupMetaData; -class RowGroupReader; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -57,6 +53,42 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +struct ParquetPageCacheRange; +struct ParquetScanRange; +class NativeParquetMetadata; + +namespace detail { +struct PredicateConjunctSchedule { + std::map single_column_conjuncts; + VExprContextSPtrs remaining_conjuncts; +}; + +struct AdaptivePredicateStats { + double cost_per_input_row_ns = 0; + double survival_ratio = 1; + size_t samples = 0; +}; + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); +Status validate_ephemeral_expr_result_column(size_t original_columns, int result_column_id, + size_t current_columns); +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges); +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups); +} // namespace detail // ============================================================================ // ============================================================================ @@ -74,23 +106,39 @@ struct RowGroupReadPlan { std::vector selected_ranges; // row ranges to read after page-index pruning std::map page_skip_plans; // leaf_column_id -> data pages that can be skipped completely + // Deferred planning transfers parsed indexes to execution so narrowed scans never issue the + // same remote index reads a second time while opening the row group. + std::unordered_map offset_indexes; + // Footer statistics are cheap and eager. Remote dictionary/Bloom/page-index probes fill the + // remaining fields only when this row group reaches the scheduler. + bool expensive_pruning_pending = false; }; struct RowGroupScanPlan { std::vector row_groups; // row groups selected after pruning ParquetPruningStats pruning_stats; // pruning statistics + bool enable_bloom_filter = false; }; // ============================================================================ // ============================================================================ -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, + ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}); + +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile = nullptr); IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -116,6 +164,9 @@ class ParquetScanScheduler { _page_skip_profile = page_skip_profile; } void set_scan_profile(ParquetScanProfile scan_profile) { _scan_profile = scan_profile; } + void set_pruning_profile(const ParquetProfile* parquet_profile) { + _parquet_profile = parquet_profile; + } void set_merge_read_options(RuntimeProfile* profile, int64_t merge_read_slice_size) { _profile = profile; _merge_read_slice_size = merge_read_slice_size; @@ -128,6 +179,10 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } + void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } + // Release row-group readers before the owning RuntimeProfile is reported. Native readers + // publish their accumulated page/decode statistics from their destructor. + void close() { reset_current_row_group(); } // Upper scanner owns adaptive memory feedback; scheduler only applies the current row cap when // splitting selected row ranges into physical read batches. void set_batch_size(size_t batch_size) { @@ -145,7 +200,15 @@ class ParquetScanScheduler { bool* eof); private: + static constexpr size_t PROFILE_FLUSH_BATCH_INTERVAL = 16; + void reset_current_row_group(); + void flush_current_reader_profiles(); + bool finish_current_reader_batch_profiles(); + const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( + const format::FileScanRequest& request); + std::vector adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const; Status open_next_row_group(ParquetFileContext& file_context, const std::vector>& file_schema, @@ -163,38 +226,40 @@ class ParquetScanScheduler { ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata); + const tparquet::RowGroup& row_group_metadata); - void prefetch_current_row_group_columns( + Status prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched); - bool prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx); - Status read_current_row_group_batch( ParquetFileContext& file_context, const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows); + Status materialize_pending_predicate_batch(const format::FileScanRequest& request, + Block* file_block, size_t* rows); + void mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_first_file_row); std::vector _row_group_plans; // row group queue to scan size_t _next_row_group_plan_idx = 0; // index of the next row group to process - std::shared_ptr<::parquet::RowGroupReader> _current_row_group; // Arrow RowGroup reader - std::map> + bool _has_current_row_group = false; + // Readers retain pointers into this immutable row-group map, so it must outlive both maps below. + std::unordered_map _current_offset_indexes; + // File-local ids are signed because virtual columns use reserved negative values. Keeping the + // typed id as the map key prevents GLOBAL_ROWID_COLUMN_ID from wrapping to a storage ColumnId. + std::map> _current_predicate_columns; // predicate ColumnReaders - std::map> + std::map> _current_non_predicate_columns; // non-predicate ColumnReaders - std::map + std::map _current_dictionary_filters; // local id -> dict entry bitmap - std::map>> + std::map>> _current_dictionary_residual_conjuncts; // local id -> row-level residual conjuncts int64_t _current_row_group_rows = 0; // current row group row count int _current_row_group_id = -1; // current row group id in parquet metadata @@ -208,18 +273,46 @@ class ParquetScanScheduler { // readers can lag behind across fully filtered batches and range gaps; the lag is flushed once // before the next surviving batch is materialized, or discarded with the row group. int64_t _pending_non_predicate_skip_rows = 0; + // Empty predicate batches may widen their physical probe. If the first non-empty probe finds + // more rows than the caller's cap, keep its narrow predicate result here and materialize lazy + // columns in capped physical slices on subsequent calls. + int64_t _pending_predicate_batch_rows = 0; + int64_t _pending_predicate_batch_rows_consumed = 0; + size_t _pending_predicate_selected_offset = 0; + std::vector _pending_predicate_selection; + std::map _pending_predicate_columns; + SelectionVector _pending_output_selection; bool _current_predicate_prefetched = false; bool _current_non_predicate_prefetched = false; bool _current_merge_range_active = false; ParquetPageSkipProfile _page_skip_profile; ParquetScanProfile _scan_profile; + const ParquetProfile* _parquet_profile = nullptr; RuntimeProfile* _profile = nullptr; int64_t _merge_read_slice_size = -1; std::optional _global_rowid_context; const cctz::time_zone* _timezone = nullptr; bool _enable_strict_mode = false; + bool _enable_bloom_filter = false; + RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; + // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without + // reallocating selection indices, dense filter bytes, or compacted-column positions. + SelectionVector _selection; + std::vector _read_column_positions_scratch; + const format::FileScanRequest* _predicate_schedule_request = nullptr; + detail::PredicateConjunctSchedule _predicate_schedule; + std::vector _predicate_positions_scratch; + std::unordered_map _predicate_indices_by_position_scratch; + std::vector _ordered_predicate_positions_scratch; + std::unordered_map> + _predicate_column_selection_scratch; + IColumn::Filter _predicate_compaction_filter_scratch; + size_t _predicate_batch_sequence = 0; + size_t _batches_since_profile_flush = 0; + std::unordered_map _predicate_runtime_stats; + double _predicate_survival_ratio = -1; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_filtered_rows = 0; int64_t _predicate_filtered_rows = 0; diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index ff72c6982de301..7d1afb39240ac1 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -15,15 +15,6 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -48,15 +39,99 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" +#include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" +#include "util/thrift_util.h" +#include "util/unaligned.h" namespace doris::format::parquet { +namespace detail { + +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size) { + if (offset < 0 || header_size == 0 || payload_size < segment_v2::BloomFilter::MINIMUM_BYTES || + payload_size > segment_v2::BloomFilter::MAXIMUM_BYTES || payload_size % 32 != 0) { + return Status::Corruption( + "Invalid Parquet Bloom filter layout: offset {}, header {}, payload {}", offset, + header_size, payload_size); + } + const uint64_t unsigned_offset = static_cast(offset); + const uint64_t total_size = static_cast(header_size) + payload_size; + if (unsigned_offset > file_size || total_size > file_size - unsigned_offset) { + return Status::Corruption("Parquet Bloom filter range exceeds file size {}", file_size); + } + if (declared_length >= 0) { + const uint64_t unsigned_declared_length = static_cast(declared_length); + if (unsigned_declared_length < total_size || + unsigned_declared_length > file_size - unsigned_offset) { + return Status::Corruption( + "Parquet Bloom filter requires {} bytes, metadata declares {}, file has {}", + total_size, declared_length, file_size - unsigned_offset); + } + } + return Status::OK(); +} + +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id) { + return leaf_column_id >= 0 && metadata.__isset.column_orders && + leaf_column_id < static_cast(metadata.column_orders.size()) && + metadata.column_orders[leaf_column_id].__isset.TYPE_ORDER; +} + +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + auto sanitized = statistics; + if (!has_type_defined_order || !sanitized.__isset.min_value || !sanitized.__isset.max_value) { + sanitized.__isset.min_value = false; + sanitized.__isset.max_value = false; + sanitized.min_value.clear(); + sanitized.max_value.clear(); + } + const bool binary = type_descriptor.physical_type == tparquet::Type::BYTE_ARRAY || + type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (!sanitized.__isset.min || !sanitized.__isset.max || + (binary && sanitized.min != sanitized.max)) { + sanitized.__isset.min = false; + sanitized.__isset.max = false; + sanitized.min.clear(); + sanitized.max.clear(); + } + return sanitized; +} + +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + // Inexact bounds remain useful for pruning, but returning them as aggregate values changes the + // query result. Missing exactness fields are legacy-compatible; only an explicit false rejects. + if ((statistics.__isset.is_min_value_exact && !statistics.is_min_value_exact) || + (statistics.__isset.is_max_value_exact && !statistics.is_max_value_exact)) { + return false; + } + const auto sanitized = + sanitize_native_footer_statistics(type_descriptor, statistics, has_type_defined_order); + return (sanitized.__isset.min_value && sanitized.__isset.max_value) || + (sanitized.__isset.min && sanitized.__isset.max); +} + +} // namespace detail + namespace { +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + int64_t page_rows, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone); + enum class ParquetRowGroupPruneReason { NONE, // cannot prune; must read STATISTICS, // excluded by ZoneMap statistics @@ -64,6 +139,60 @@ enum class ParquetRowGroupPruneReason { BLOOM_FILTER, // excluded by bloom filter }; +Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, + const io::FileReaderSPtr& file, io::IOContext* io_ctx, + std::unique_ptr* result) { + if (result == nullptr || file == nullptr || !metadata.__isset.bloom_filter_offset) { + return Status::NotSupported("Parquet Bloom filter is unavailable"); + } + constexpr size_t MAX_BLOOM_HEADER_BYTES = 64; + if (metadata.bloom_filter_offset < 0 || + (metadata.__isset.bloom_filter_length && metadata.bloom_filter_length <= 0)) { + return Status::Corruption("Invalid Parquet Bloom filter offset or declared length"); + } + const uint64_t bloom_offset = static_cast(metadata.bloom_filter_offset); + if (bloom_offset >= file->size()) { + return Status::Corruption("Parquet Bloom filter offset exceeds file size {}", file->size()); + } + const size_t available = file->size() - bloom_offset; + const size_t declared_available = + metadata.__isset.bloom_filter_length + ? std::min(metadata.bloom_filter_length, available) + : available; + const size_t header_read_size = std::min(declared_available, MAX_BLOOM_HEADER_BYTES); + std::vector header_buffer(header_read_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset, + Slice(header_buffer.data(), header_buffer.size()), &bytes_read, + io_ctx)); + tparquet::BloomFilterHeader header; + uint32_t header_size = cast_set(bytes_read); + RETURN_IF_ERROR(deserialize_thrift_msg(header_buffer.data(), &header_size, true, &header)); + if (!header.algorithm.__isset.BLOCK || !header.compression.__isset.UNCOMPRESSED || + !header.hash.__isset.XXHASH || header.numBytes <= 0) { + return Status::NotSupported("Unsupported Parquet Bloom filter encoding"); + } + + // Validate the complete split-block layout before allocating or adding footer-controlled + // offsets; BloomFilter::init() otherwise receives a truncated or oversized backing buffer. + RETURN_IF_ERROR(detail::validate_native_bloom_filter_layout( + metadata.bloom_filter_offset, header_size, header.numBytes, + metadata.__isset.bloom_filter_length ? metadata.bloom_filter_length : -1, + file->size())); + + std::vector data(cast_set(header.numBytes)); + RETURN_IF_ERROR(file->read_at(static_cast(metadata.bloom_filter_offset) + header_size, + Slice(data.data(), data.size()), &bytes_read, io_ctx)); + if (bytes_read != data.size()) { + return Status::Corruption("Truncated Parquet Bloom filter payload"); + } + auto bloom_filter = std::make_unique(); + RETURN_IF_ERROR(bloom_filter->init(reinterpret_cast(data.data()), data.size(), + segment_v2::HashStrategyPB::XX_HASH_64)); + *result = std::move(bloom_filter); + return Status::OK(); +} + bool bloom_logical_type_supported(const ParquetColumnSchema& column_schema) { if (column_schema.type == nullptr) { return false; @@ -109,8 +238,15 @@ Status read_decoded_field(const ParquetColumnSchema& column_schema, DecodedColum view.fixed_length = column_schema.type_descriptor.fixed_length; view.timestamp_is_adjusted_to_utc = column_schema.type_descriptor.timestamp_is_adjusted_to_utc; view.timezone = timezone; - return column_schema.type->get_serde()->read_field_from_decoded_value(*column_schema.type, - field, view); + // Statistics are pruning proofs, not row materialization. A malformed non-NULL bound must + // disable pruning instead of being converted to NULL under permissive scan semantics. + view.enable_strict_mode = true; + RETURN_IF_ERROR(column_schema.type->get_serde()->read_field_from_decoded_value( + *column_schema.type, field, view)); + if (field->is_null()) { + return Status::DataQualityError("Non-NULL Parquet statistic decoded as NULL"); + } + return Status::OK(); } template @@ -172,30 +308,6 @@ bool decoded_min_max_is_ordered(const ParquetColumnStatistics& column_statistics return !(column_statistics.max_value < column_statistics.min_value); } -template -bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics>(statistics); - const auto& min_value = typed_statistics->min(); - const auto& max_value = typed_statistics->max(); - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; - } - } - if (!valid_min_max(min_value, max_value) || - !set_decoded_field(column_schema, value_kind, min_value, &column_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &column_statistics->max_value, - timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); -} - bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, const StringRef& value, Field* field, const cctz::time_zone* timezone) { @@ -206,54 +318,6 @@ bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedV return read_decoded_field(column_schema, view, field, timezone).ok(); } -bool set_string_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - switch (statistics->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::ByteArrayType>>( - statistics); - const auto min = ::parquet::ByteArrayToString(typed_statistics->min()); - const auto max = ::parquet::ByteArrayToString(typed_statistics->max()); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - if (column_schema.descriptor == nullptr || column_schema.descriptor->type_length() <= 0) { - return false; - } - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::FLBAType>>( - statistics); - const int type_length = column_schema.descriptor->type_length(); - const std::string min(reinterpret_cast(typed_statistics->min().ptr), - type_length); - const std::string max(reinterpret_cast(typed_statistics->max().ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - default: - return false; - } -} - template T load_predicate_value(const char* data) { T value; @@ -306,61 +370,19 @@ std::optional convert_logical_integer_to_physical_int32( return physical_value; } -class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { +class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { public: - ArrowParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, - const ::parquet::BloomFilter& bloom_filter) + NativeParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, + const segment_v2::BloomFilter& bloom_filter) : _column_schema(column_schema), _bloom_filter(bloom_filter) {} - void add_bytes(const char* buf, size_t size) override { DORIS_CHECK(false); } + void add_bytes(const char*, size_t) override { DORIS_CHECK(false); } bool test_bytes(const char* buf, size_t size) const override { - if (buf == nullptr) { - return true; - } - // Parquet bloom filters are populated from the physical column carrier, while VExpr - // literals are materialized as Doris logical values. Keep the logical type in - // BloomFilterEvalContext for expression compatibility, and normalize to the Parquet - // physical representation only at this adapter boundary. - switch (_column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - return test_boolean(buf, size); - case ::parquet::Type::INT32: - return test_physical_int32(buf, size); - case ::parquet::Type::INT64: - return test_int64(buf, size); - case ::parquet::Type::FLOAT: - return test_float(buf, size); - case ::parquet::Type::DOUBLE: - return test_double(buf, size); - case ::parquet::Type::BYTE_ARRAY: - return test_byte_array(buf, size); - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return test_fixed_len_byte_array(buf, size); - default: - return true; + if (buf == nullptr || + _column_schema.type_descriptor.physical_type != tparquet::Type::INT32) { + return _bloom_filter.test_bytes(buf, size); } - } - - void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } - bool has_null() const override { return false; } - void add_hash(uint64_t hash) override { DORIS_CHECK(false); } - bool test_hash(uint64_t hash) const override { return _bloom_filter.FindHash(hash); } - -private: - bool test_boolean(const char* buf, size_t size) const { - if (size == sizeof(bool)) { - const int32_t value = load_predicate_value(buf) ? 1 : 0; - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - if (size == sizeof(int32_t)) { - const int32_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value != 0 ? 1 : 0)); - } - return true; - } - - bool test_physical_int32(const char* buf, size_t size) const { const auto logical_value = load_predicate_integral_value(buf, size); if (!logical_value.has_value()) { return true; @@ -370,57 +392,20 @@ class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { if (!physical_value.has_value()) { return false; } - return find_int32(*physical_value); - } - - bool test_int64(const char* buf, size_t size) const { - if (size != sizeof(int64_t)) { - return true; - } - const int64_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_float(const char* buf, size_t size) const { - if (size != sizeof(float)) { - return true; - } - const float value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_double(const char* buf, size_t size) const { - if (size != sizeof(double)) { - return true; - } - const double value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_byte_array(const char* buf, size_t size) const { - ::parquet::ByteArray value(static_cast(size), - reinterpret_cast(buf)); - return _bloom_filter.FindHash(_bloom_filter.Hash(&value)); - } - - bool test_fixed_len_byte_array(const char* buf, size_t size) const { - if (_column_schema.type_descriptor.fixed_length <= 0) { - return true; - } - if (size != static_cast(_column_schema.type_descriptor.fixed_length)) { - return false; - } - ::parquet::FLBA value(reinterpret_cast(buf)); - return _bloom_filter.FindHash( - _bloom_filter.Hash(&value, _column_schema.type_descriptor.fixed_length)); + // Native file Bloom bytes are hashed from the Parquet physical carrier, not the wider + // Doris logical literal used by VExpr (for example UINT32 is exposed as BIGINT). + return _bloom_filter.test_bytes(reinterpret_cast(&*physical_value), + sizeof(*physical_value)); } - bool find_int32(int32_t value) const { - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } + void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } + bool has_null() const override { return false; } + void add_hash(uint64_t) override { DORIS_CHECK(false); } + bool test_hash(uint64_t hash) const override { return _bloom_filter.test_hash(hash); } +private: const ParquetColumnSchema& _column_schema; - const ::parquet::BloomFilter& _bloom_filter; + const segment_v2::BloomFilter& _bloom_filter; }; bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { @@ -428,14 +413,14 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { return false; } switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BOOLEAN: + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::BYTE_ARRAY: return true; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return column_schema.type_descriptor.is_string_like && column_schema.type_descriptor.fixed_length > 0; default: @@ -443,220 +428,6 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { } } -bool bloom_filter_excludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - if (!bloom_filter_supported(column_schema)) { - return false; - } - ArrowParquetBloomFilterAdapter adapter(column_schema, bloom_filter); - BloomFilterEvalContext ctx; - ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { - .data_type = column_schema.type, - .bloom_filter = &adapter, - }); - return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; -} - -struct RowGroupBloomFilterCache { - using CacheKey = std::pair; - - ::parquet::BloomFilterReader* bloom_filter_reader = nullptr; - std::map> column_bloom_filters; - std::set loaded_columns; - - ::parquet::BloomFilter* get(int row_group_idx, int leaf_column_id, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_reader == nullptr || leaf_column_id < 0) { - return nullptr; - } - const CacheKey cache_key {row_group_idx, leaf_column_id}; - if (loaded_columns.find(cache_key) == loaded_columns.end()) { - loaded_columns.insert(cache_key); - try { - std::shared_ptr<::parquet::RowGroupBloomFilterReader> row_group_reader; - if (pruning_stats != nullptr) { - SCOPED_RAW_TIMER(&pruning_stats->bloom_filter_read_time); - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } else { - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } - } catch (const ::parquet::ParquetException&) { - return nullptr; - } catch (const std::exception&) { - return nullptr; - } - } - auto it = column_bloom_filters.find(cache_key); - return it == column_bloom_filters.end() ? nullptr : it->second.get(); - } -}; - -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; -} - -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; -} - -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; -} - -bool is_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { - return false; - } - - const auto& encoding_stats = column_metadata.encoding_stats(); - if (!encoding_stats.empty()) { - bool has_dictionary_data_page = false; - for (const auto& encoding_stat : encoding_stats) { - if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { - continue; - } - if (!is_dictionary_data_encoding(encoding_stat.encoding)) { - return false; - } - has_dictionary_data_page = true; - } - return has_dictionary_data_page; - } - - bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { - if (is_dictionary_data_encoding(encoding)) { - has_dictionary_encoding = true; - continue; - } - if (!is_level_encoding(encoding)) { - return false; - } - } - return has_dictionary_encoding; -} - -bool supports_dictionary_pruning(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr) { - return false; - } - if (!column_schema.type_descriptor.is_string_like) { - return false; - } - if (column_metadata.type() != ::parquet::Type::BYTE_ARRAY && - column_metadata.type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - return false; - } - return true; -} - -} // namespace - -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words) { - DORIS_CHECK(dict_words != nullptr); - dict_words->clear(); - if (file_reader == nullptr || leaf_column_id < 0) { - return false; - } - - auto row_group_reader = file_reader->RowGroup(row_group_idx); - if (row_group_reader == nullptr) { - return false; - } - auto page_reader = row_group_reader->GetColumnPageReader(leaf_column_id); - if (page_reader == nullptr) { - return false; - } - - std::shared_ptr<::parquet::Page> page; - try { - page = page_reader->NextPage(); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; - } - if (page == nullptr || page->type() != ::parquet::PageType::DICTIONARY_PAGE) { - return false; - } - const auto* dictionary_page = static_cast(page.get()); - if (dictionary_page->encoding() != ::parquet::Encoding::PLAIN && - dictionary_page->encoding() != ::parquet::Encoding::PLAIN_DICTIONARY) { - return false; - } - const int32_t dictionary_length = dictionary_page->num_values(); - if (dictionary_length <= 0) { - return false; - } - const auto* dictionary_data = dictionary_page->data(); - const int dictionary_size = dictionary_page->size(); - - dict_words->values.reserve(static_cast(dictionary_length)); - if (column_schema.descriptor->physical_type() == ::parquet::Type::BYTE_ARRAY) { - auto decoder = ::parquet::MakeTypedDecoder<::parquet::ByteArrayType>( - ::parquet::Encoding::PLAIN, column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::ByteArray> byte_array_values(static_cast(dictionary_length)); - if (decoder->Decode(byte_array_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(byte_array_values[dict_idx].ptr), - byte_array_values[dict_idx].len); - } - dict_words->build_refs(); - return true; - } - if (column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; - } - auto decoder = ::parquet::MakeTypedDecoder<::parquet::FLBAType>(::parquet::Encoding::PLAIN, - column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::FixedLenByteArray> flba_values( - static_cast(dictionary_length)); - if (decoder->Decode(flba_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(flba_values[dict_idx].ptr), type_length); - } - dict_words->build_refs(); - return true; - } - return false; -} - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words) { - std::vector fields; - fields.reserve(dict_words.refs.size()); - for (const auto& ref : dict_words.refs) { - fields.push_back(Field::create_field(String(ref.data, ref.size))); - } - return fields; -} - -namespace { - const ParquetColumnSchema* resolve_local_leaf_schema( const std::vector>& schema, const format::LocalColumnId file_column_id) { @@ -762,164 +533,112 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats } // namespace +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state) { + return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state); +} + std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( const ParquetColumnStatistics& statistics) { return make_zonemap_from_statistics(statistics); } ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone) { + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone) { ParquetColumnStatistics result; - if (statistics == nullptr) { + if (statistics == nullptr || column_value_count < 0) { return result; } - result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; - result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); - result.has_null_count = statistics->HasNullCount(); - if (!result.has_not_null || !statistics->HasMinMax()) { + if (statistics->__isset.null_count && statistics->null_count > column_value_count) { + // An impossible null count makes all derived min/max and all-null flags untrustworthy; + // disable pruning instead of turning corrupt footer metadata into false negatives. return result; } - DORIS_CHECK(column_schema.type != nullptr); - switch (statistics->physical_type()) { - case ::parquet::Type::BOOLEAN: - result.has_min_max = set_decoded_min_max<::parquet::BooleanType>( - statistics, column_schema, DecodedValueKind::BOOL, &result, timezone); - return result; - case ::parquet::Type::INT32: - result.has_min_max = set_decoded_min_max<::parquet::Int32Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::INT64: - result.has_min_max = set_decoded_min_max<::parquet::Int64Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::FLOAT: - result.has_min_max = set_decoded_min_max<::parquet::FloatType>( - statistics, column_schema, DecodedValueKind::FLOAT, &result, timezone); - return result; - case ::parquet::Type::DOUBLE: - result.has_min_max = set_decoded_min_max<::parquet::DoubleType>( - statistics, column_schema, DecodedValueKind::DOUBLE, &result, timezone); - return result; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result.has_min_max = set_string_min_max(statistics, column_schema, &result, timezone); - return result; - default: - return result; + const bool has_null_count = statistics->__isset.null_count && statistics->null_count >= 0; + const int64_t null_count = has_null_count ? statistics->null_count : 0; + const bool has_not_null = has_null_count ? column_value_count > null_count : true; + const std::string* min_value = statistics->__isset.min_value + ? &statistics->min_value + : (statistics->__isset.min ? &statistics->min : nullptr); + const std::string* max_value = statistics->__isset.max_value + ? &statistics->max_value + : (statistics->__isset.max ? &statistics->max : nullptr); + + tparquet::ColumnIndex index; + index.__set_null_pages({!has_not_null}); + index.__set_null_counts({null_count}); + if (min_value != nullptr && max_value != nullptr) { + index.__set_min_values({*min_value}); + index.__set_max_values({*max_value}); + } + // Footer statistics and page indexes share the same little-endian physical encoding. Reusing + // one decoder keeps native row-group and page pruning identical for logical types and NaNs. + if (!build_native_page_statistics(index, column_schema, 0, column_value_count, &result, + timezone)) { + return {}; + } + if (!has_null_count) { + result.has_null_count = false; + result.has_null = true; } + return result; } -bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, - int slot_index, const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); +bool ParquetStatisticsUtils::NativeBloomFilterExcludes( + const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter) { + if (!bloom_filter_supported(column_schema)) { + return false; + } + NativeParquetBloomFilterAdapter adapter(column_schema, bloom_filter); + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema.type, + .bloom_filter = &adapter, + }); + return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; } namespace { -ParquetRowGroupPruneReason dictionary_prune_reason( - const ::parquet::RowGroupMetaData& row_group, ::parquet::ParquetFileReader* file_reader, - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request) { - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_dictionary_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { - continue; - } - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_dictionary_pruning(*column_schema, *column_chunk) || - !is_dictionary_encoded_chunk(*column_chunk)) { - continue; - } - - ParquetDictionaryWords dict_words; - if (!read_dictionary_words(file_reader, row_group_idx, column_schema->leaf_column_id, - *column_schema, &dict_words)) { - continue; - } - DictionaryEvalContext ctx; - ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { - .data_type = column_schema->type, - .values = dictionary_fields_from_words(dict_words), - }); - if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch) { - return ParquetRowGroupPruneReason::DICTIONARY; +void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::set* leaf_column_ids) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); } + return; } - return ParquetRowGroupPruneReason::NONE; -} - -ParquetRowGroupPruneReason bloom_filter_prune_reason( - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, RowGroupBloomFilterCache* bloom_filter_cache, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_cache == nullptr) { - return ParquetRowGroupPruneReason::NONE; - } - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr || - !bloom_filter_supported(*column_schema)) { - continue; - } - auto* bloom_filter = bloom_filter_cache->get(row_group_idx, column_schema->leaf_column_id, - pruning_stats); - if (bloom_filter == nullptr) { + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { continue; } - if (ParquetStatisticsUtils::BloomFilterExcludes(*column_schema, slot_index, conjuncts, - *bloom_filter)) { - return ParquetRowGroupPruneReason::BLOOM_FILTER; - } + collect_filtered_leaf_ids(*child_schema, + format::find_child_projection(projection, child_schema->local_id), + leaf_column_ids); } - return ParquetRowGroupPruneReason::NONE; } -void init_bloom_filter_cache(::parquet::ParquetFileReader* file_reader, bool enable_bloom_filter, - RowGroupBloomFilterCache* bloom_filter_cache) { - DORIS_CHECK(bloom_filter_cache != nullptr); - if (!enable_bloom_filter || file_reader == nullptr) { - return; - } - try { - bloom_filter_cache->bloom_filter_reader = &file_reader->GetBloomFilterReader(); - } catch (const ::parquet::ParquetException&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } catch (const std::exception&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } +bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.type != nullptr); + // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in + // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. + return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; } -bool check_statistics(const ::parquet::RowGroupMetaData& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { +bool check_native_statistics(const tparquet::FileMetaData& metadata, + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); if (slot_indexes.empty()) { return false; } - ZoneMapEvalContext ctx; for (const int slot_index : slot_indexes) { const auto file_column_id = file_column_id_by_block_position(request, slot_index); @@ -927,270 +646,320 @@ bool check_statistics(const ::parquet::RowGroupMetaData& row_group, continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } - + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; std::shared_ptr zone_map; - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk != nullptr) { + if (chunk.__isset.meta_data) { + const auto& column_metadata = chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + column_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order(metadata, + column_schema->leaf_column_id)); + } zone_map = ParquetStatisticsUtils::MakeZoneMap( ParquetStatisticsUtils::TransformColumnStatistics( - *column_schema, column_chunk->statistics(), timezone)); + *column_schema, + safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, timezone)); } add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); } - const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); accumulate_zonemap_stats(ctx, pruning_stats); return result == ZoneMapFilterResult::kNoMatch; } -Status select_row_groups_by_metadata_impl( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - int64_t row_group_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &row_group_filter_time_sink - : &pruning_stats->row_group_filter_time); - if (selected_row_groups == nullptr) { - return Status::InvalidArgument("selected_row_groups is null"); - } - selected_row_groups->clear(); +bool is_native_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; +} - const int num_row_groups = metadata.num_row_groups(); - const auto candidate_size = candidate_row_groups == nullptr - ? static_cast(num_row_groups) - : candidate_row_groups->size(); - if (pruning_stats != nullptr) { - // Scan-range ownership is decided before metadata pruning. Count only row groups owned by - // this split so a file divided into multiple splits does not report the full-file total and - // out-of-split groups once per split. - pruning_stats->total_row_groups = cast_set(candidate_size); +bool is_native_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; +} + +bool is_native_dictionary_encoded_chunk(const tparquet::ColumnMetaData& metadata) { + if (!metadata.__isset.dictionary_page_offset || metadata.dictionary_page_offset < 0) { + return false; } - selected_row_groups->reserve(candidate_size); - RowGroupBloomFilterCache bloom_filter_cache; - init_bloom_filter_cache(file_reader, enable_bloom_filter, &bloom_filter_cache); - for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { - const int row_group_idx = candidate_row_groups == nullptr - ? static_cast(candidate_idx) - : (*candidate_row_groups)[candidate_idx]; - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(row_group_idx < num_row_groups); - auto row_group = metadata.RowGroup(row_group_idx); - if (row_group == nullptr) { - selected_row_groups->push_back(row_group_idx); - continue; + if (metadata.__isset.encoding_stats && !metadata.encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : metadata.encoding_stats) { + if ((encoding_stat.page_type != tparquet::PageType::DATA_PAGE && + encoding_stat.page_type != tparquet::PageType::DATA_PAGE_V2) || + encoding_stat.count <= 0) { + continue; + } + if (!is_native_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; } - ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; - if (has_expr_zonemap_filter(request, runtime_state) && - check_statistics(*row_group, file_schema, request, pruning_stats, timezone)) { - prune_reason = ParquetRowGroupPruneReason::STATISTICS; + return has_dictionary_data_page; + } + bool has_dictionary_encoding = false; + for (const auto encoding : metadata.encodings) { + if (is_native_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + } else if (!is_native_level_encoding(encoding)) { + return false; } + } + return has_dictionary_encoding; +} - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = dictionary_prune_reason(*row_group, file_reader, row_group_idx, - file_schema, request); - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = bloom_filter_prune_reason(row_group_idx, file_schema, request, - &bloom_filter_cache, pruning_stats); - } +const format::LocalColumnIndex* find_request_projection(const format::FileScanRequest& request, + format::LocalColumnId file_column_id) { + for (const auto& projection : request.predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - - if (prune_reason != ParquetRowGroupPruneReason::NONE) { - if (pruning_stats != nullptr) { - pruning_stats->filtered_group_rows += row_group->num_rows(); - if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { - ++pruning_stats->filtered_row_groups_by_statistics; - } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { - ++pruning_stats->filtered_row_groups_by_dictionary; - } else if (prune_reason == ParquetRowGroupPruneReason::BLOOM_FILTER) { - ++pruning_stats->filtered_row_groups_by_bloom_filter; - } - } - continue; + } + for (const auto& projection : request.non_predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - selected_row_groups->push_back(row_group_idx); } - return Status::OK(); + return nullptr; } -} // namespace - -Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +ParquetRowGroupPruneReason native_dictionary_prune_reason( + const tparquet::RowGroup& row_group, int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - return select_row_groups_by_metadata_impl( - metadata, file_reader, file_schema, request, candidate_row_groups, selected_row_groups, - enable_bloom_filter, pruning_stats, timezone, runtime_state); -} - -namespace { - -template -bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - DecodedValueKind value_kind, ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - const auto typed_index = - std::static_pointer_cast<::parquet::TypedColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + const format::FileScanRequest& request, const cctz::time_zone* timezone, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile) { + if (file_context == nullptr || file_context->native_metadata == nullptr) { + return ParquetRowGroupPruneReason::NONE; } - const auto& min_value = typed_index->min_values()[page_idx]; - const auto& max_value = typed_index->max_values()[page_idx]; - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_dictionary_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + const auto* projection = find_request_projection(request, *file_column_id); + if (column_schema == nullptr || projection == nullptr || column_schema->type == nullptr || + !column_schema->type_descriptor.is_string_like || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + if (!native_metadata_predicate_is_type_safe(*column_schema)) { + // The file-local VARBINARY may feed a table-side STRING cast. Pruning before that cast + // can compare different Field kinds and incorrectly discard a matching row group. + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data || + (chunk.meta_data.type != tparquet::Type::BYTE_ARRAY && + chunk.meta_data.type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) || + !is_native_dictionary_encoded_chunk(chunk.meta_data)) { + continue; + } + std::unique_ptr reader; + const std::vector ranges {{0, row_group.num_rows}}; + const std::unordered_map offset_indexes; + // Metadata pruning uses the real native reader, so its page work must be attributed to the + // scan profile even when the row group is eliminated before execution readers are built. + const auto status = NativeColumnReader::create( + *column_schema, projection, file_context->native_file, + file_context->native_metadata, row_group_idx, ranges, offset_indexes, timezone, + file_context->native_io_ctx, nullptr, file_context->native_page_cache_enabled, + file_context->native_page_cache_file_key, true, column_reader_profile, &reader); + if (!status.ok() || reader == nullptr) { + continue; + } + auto dictionary_result = reader->dictionary_values(); + if (!dictionary_result.has_value()) { + continue; + } + auto dictionary = std::move(dictionary_result).value(); + std::vector values(dictionary->size()); + for (size_t value_idx = 0; value_idx < dictionary->size(); ++value_idx) { + dictionary->get(value_idx, values[value_idx]); + } + DictionaryEvalContext ctx; + ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { + .data_type = column_schema->type, + .values = std::move(values), + }); + if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::DICTIONARY; } } - if (!valid_min_max(min_value, max_value)) { - // A NaN invalidates only this page's bounds, not the ColumnIndex itself. Keep the page - // conservatively by returning usable null-count statistics with has_min_max=false, while - // allowing later pages with finite bounds to remain eligible for pruning. - return true; - } - if (!set_decoded_field(column_schema, value_kind, min_value, &page_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &page_statistics->max_value, - timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; + return ParquetRowGroupPruneReason::NONE; } -bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - const auto typed_index = - std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const auto min = ::parquet::ByteArrayToString(typed_index->min_values()[page_idx]); - const auto max = ::parquet::ByteArrayToString(typed_index->max_values()[page_idx]); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; +ParquetRowGroupPruneReason native_bloom_filter_prune_reason( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, ParquetFileContext* file_context, + ParquetPruningStats* pruning_stats) { + if (file_context == nullptr || file_context->native_file == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !bloom_filter_supported(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; } - page_statistics->has_min_max = true; - return true; - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data) { + continue; } - const auto typed_index = std::static_pointer_cast<::parquet::FLBAColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + std::unique_ptr bloom_filter; + Status status; + { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->bloom_filter_read_time); + status = read_native_bloom_filter(chunk.meta_data, file_context->native_file, + file_context->native_io_ctx, &bloom_filter); } - const std::string min( - reinterpret_cast(typed_index->min_values()[page_idx].ptr), - type_length); - const std::string max( - reinterpret_cast(typed_index->max_values()[page_idx].ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; + if (!status.ok() || bloom_filter == nullptr) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + if (ParquetStatisticsUtils::NativeBloomFilterExcludes(*column_schema, slot_index, conjuncts, + *bloom_filter)) { + return ParquetRowGroupPruneReason::BLOOM_FILTER; } - page_statistics->has_min_max = true; - return true; - } - default: - return false; } + return ParquetRowGroupPruneReason::NONE; } -bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - DORIS_CHECK(column_schema.type != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BOOLEAN: - return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, - page_idx, DecodedValueKind::BOOL, - page_statistics, timezone); - case ::parquet::Type::INT32: - return set_page_decoded_min_max<::parquet::Int32Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::INT64: - return set_page_decoded_min_max<::parquet::Int64Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::FLOAT: - return set_page_decoded_min_max<::parquet::FloatType>(column_index, column_schema, page_idx, - DecodedValueKind::FLOAT, - page_statistics, timezone); - case ::parquet::Type::DOUBLE: - return set_page_decoded_min_max<::parquet::DoubleType>(column_index, column_schema, - page_idx, DecodedValueKind::DOUBLE, - page_statistics, timezone); - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return set_page_string_min_max(column_index, column_schema, page_idx, page_statistics, - timezone); - default: - return false; +int64_t native_requested_compressed_bytes( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request) { + std::set leaf_column_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return; + } + collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); + } + int64_t bytes = 0; + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (chunk.__isset.meta_data && chunk.meta_data.total_compressed_size > 0) { + bytes += chunk.meta_data.total_compressed_size; + } } + return bytes; } -bool build_page_statistics(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - DORIS_CHECK(page_statistics != nullptr); - *page_statistics = ParquetColumnStatistics {}; +} // namespace - const auto& null_pages = column_index->null_pages(); - if (!column_index->has_null_counts() || page_idx >= null_pages.size() || - page_idx >= column_index->null_counts().size()) { - return false; +Status select_row_groups_by_metadata( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile, + ParquetMetadataProbeMode probe_mode) { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->row_group_filter_time); + if (selected_row_groups == nullptr) { + return Status::InvalidArgument("selected_row_groups is null"); } - - page_statistics->has_null_count = true; - page_statistics->has_null = column_index->null_counts()[page_idx] > 0; - page_statistics->has_not_null = !null_pages[page_idx]; - if (!page_statistics->has_not_null) { - return true; + selected_row_groups->clear(); + const size_t candidate_size = candidate_row_groups == nullptr ? metadata.row_groups.size() + : candidate_row_groups->size(); + if (pruning_stats != nullptr) { + pruning_stats->total_row_groups = cast_set(candidate_size); } - return set_page_min_max(column_index, column_schema, page_idx, page_statistics, timezone); + selected_row_groups->reserve(candidate_size); + for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { + const int row_group_idx = candidate_row_groups == nullptr + ? static_cast(candidate_idx) + : (*candidate_row_groups)[candidate_idx]; + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + // Candidate ids originate in external split metadata; a corrupt id must not terminate + // the BE while planning an otherwise recoverable file scan. + return Status::Corruption("Invalid Parquet row group candidate {} for {} row groups", + row_group_idx, metadata.row_groups.size()); + } + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.num_rows == 0) { + // Native metadata probes construct positive row ranges; empty groups contribute no + // rows and must be discarded before dictionary, statistics, or Bloom reader setup. + continue; + } + ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; + if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY && + has_expr_zonemap_filter(request, runtime_state) && + check_native_statistics(metadata, row_group, file_schema, request, pruning_stats, + timezone)) { + prune_reason = ParquetRowGroupPruneReason::STATISTICS; + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = + native_dictionary_prune_reason(row_group, row_group_idx, file_schema, request, + timezone, file_context, column_reader_profile); + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { + prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, + file_context, pruning_stats); + } + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + selected_row_groups->push_back(row_group_idx); + continue; + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_group_rows += row_group.num_rows; + pruning_stats->filtered_bytes += + native_requested_compressed_bytes(row_group, file_schema, request); + if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { + ++pruning_stats->filtered_row_groups_by_statistics; + } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { + ++pruning_stats->filtered_row_groups_by_dictionary; + } else { + ++pruning_stats->filtered_row_groups_by_bloom_filter; + } + } + } + return Status::OK(); } +namespace { + std::vector intersect_ranges(const std::vector& left, const std::vector& right) { std::vector result; @@ -1223,19 +992,6 @@ int64_t count_range_rows(const std::vector& ranges) { return rows; } -RowRange page_row_range(const ::parquet::OffsetIndex& offset_index, size_t page_idx, - int64_t row_group_rows) { - const auto& page_locations = offset_index.page_locations(); - const int64_t start = page_locations[page_idx].first_row_index; - const int64_t end = page_idx + 1 == page_locations.size() - ? row_group_rows - : page_locations[page_idx + 1].first_row_index; - DORIS_CHECK(start >= 0); - DORIS_CHECK(end >= start); - DORIS_CHECK(end <= row_group_rows); - return RowRange {start, end - start}; -} - void append_row_range(const RowRange& range, std::vector* ranges) { if (range.length == 0) { return; @@ -1250,85 +1006,6 @@ void append_row_range(const RowRange& range, std::vector* ranges) { ranges->push_back(range); } -std::optional< - std::pair, std::shared_ptr<::parquet::OffsetIndex>>> -load_page_indexes_for_slot(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, - const ParquetColumnSchema** column_schema) { - DORIS_CHECK(column_schema != nullptr); - *column_schema = nullptr; - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - return std::nullopt; - } - *column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (*column_schema == nullptr || (*column_schema)->descriptor == nullptr) { - return std::nullopt; - } - - try { - auto column_index = row_group->GetColumnIndex((*column_schema)->leaf_column_id); - auto offset_index = row_group->GetOffsetIndex((*column_schema)->leaf_column_id); - if (column_index == nullptr || offset_index == nullptr || - column_index->null_pages().size() != offset_index->page_locations().size()) { - return std::nullopt; - } - return std::make_pair(std::move(column_index), std::move(offset_index)); - } catch (const ::parquet::ParquetException&) { - return std::nullopt; - } catch (const std::exception&) { - return std::nullopt; - } -} - -bool select_ranges_for_expr_zonemap( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, const VExprContextSPtrs& conjuncts, - int64_t row_group_rows, std::vector* ranges, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { - DORIS_CHECK(ranges != nullptr); - if (conjuncts.empty()) { - return false; - } - const ParquetColumnSchema* column_schema = nullptr; - const auto page_indexes = - load_page_indexes_for_slot(row_group, file_schema, request, slot_index, &column_schema); - if (!page_indexes.has_value()) { - return false; - } - const auto& [column_index, offset_index] = *page_indexes; - - ranges->clear(); - ZoneMapEvalStats page_stats; - const auto page_count = offset_index->page_locations().size(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - ParquetColumnStatistics page_statistics; - if (!ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *column_schema, page_idx, &page_statistics, timezone)) { - ranges->clear(); - return false; - } - - ZoneMapEvalContext ctx; - add_slot_zonemap(&ctx, slot_index, column_schema->type, - ParquetStatisticsUtils::MakeZoneMap(page_statistics)); - const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx); - page_stats.merge_page_eval_stats(ctx.stats); - if (result == ZoneMapFilterResult::kNoMatch) { - continue; - } - append_row_range(page_row_range(*offset_index, page_idx, row_group_rows), ranges); - } - if (pruning_stats != nullptr) { - pruning_stats->expr_zonemap_unusable_evals += page_stats.unusable_zonemap_eval_count; - pruning_stats->in_zonemap_point_check_count += page_stats.in_zonemap_point_check_count; - pruning_stats->in_zonemap_range_only_count += page_stats.in_zonemap_range_only_count; - } - return true; -} - bool ranges_intersect(const std::vector& ranges, const RowRange& range) { const int64_t range_end = range.start + range.length; for (const auto& selected_range : ranges) { @@ -1388,127 +1065,201 @@ void collect_request_leaf_schemas( } } -bool build_page_skip_plan_for_leaf( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const ParquetColumnSchema& column_schema, const std::vector& selected_ranges, - int64_t row_group_rows, ParquetPageSkipPlan* page_skip_plan) { - DORIS_CHECK(page_skip_plan != nullptr); - *page_skip_plan = ParquetPageSkipPlan {}; - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.leaf_column_id < 0 || - column_schema.descriptor->max_repetition_level() != 0) { +template +bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + DecodedValueKind kind, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != sizeof(ValueType) || + column_index.max_values[page_idx].size() != sizeof(ValueType)) { return false; } - - std::shared_ptr<::parquet::OffsetIndex> offset_index; - try { - offset_index = row_group->GetOffsetIndex(column_schema.leaf_column_id); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; + const auto min_value = unaligned_load(column_index.min_values[page_idx].data()); + const auto max_value = unaligned_load(column_index.max_values[page_idx].data()); + if constexpr (std::is_integral_v) { + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_TIMEV2) { + int64_t units_per_day = 0; + switch (column_schema.type_descriptor.time_unit) { + case ParquetTimeUnit::MILLIS: + units_per_day = 86400000; + break; + case ParquetTimeUnit::MICROS: + units_per_day = 86400000000; + break; + case ParquetTimeUnit::NANOS: + units_per_day = 86400000000000; + break; + default: + return false; + } + // TIME statistics are pruning proofs. Validate the raw carrier before rescaling so an + // invalid bound at or beyond 24:00 cannot publish a misleading ZoneMap. + if (min_value < 0 || max_value < 0 || min_value >= units_per_day || + max_value >= units_per_day) { + return false; + } + } } - if (offset_index == nullptr) { + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, kind, min_value, &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, kind, max_value, &page_statistics->max_value, timezone)) { return false; } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; +} - const auto page_count = offset_index->page_locations().size(); - page_skip_plan->leaf_column_id = column_schema.leaf_column_id; - page_skip_plan->skipped_pages.resize(page_count); - page_skip_plan->skipped_page_compressed_sizes.resize(page_count); - const auto& page_locations = offset_index->page_locations(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - const RowRange row_range = page_row_range(*offset_index, page_idx, row_group_rows); - if (row_range.length == 0 || ranges_intersect(selected_ranges, row_range)) { - continue; - } - page_skip_plan->skipped_pages[page_idx] = 1; - page_skip_plan->skipped_page_compressed_sizes[page_idx] = - page_locations[page_idx].compressed_page_size; - append_row_range(row_range, &page_skip_plan->skipped_ranges); +bool set_native_page_boolean_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != 1 || + column_index.max_values[page_idx].size() != 1) { + return false; } - if (page_skip_plan->empty()) { - *page_skip_plan = ParquetPageSkipPlan {}; + // Parquet BOOLEAN statistics use the same one-bit value representation as PLAIN pages; bits + // outside the value bit are padding and must not change false into true. + const uint8_t min_value = static_cast(column_index.min_values[page_idx][0]) & 1; + const uint8_t max_value = static_cast(column_index.max_values[page_idx][0]) & 1; + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, DecodedValueKind::BOOL, min_value, + &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, DecodedValueKind::BOOL, max_value, + &page_statistics->max_value, timezone)) { return false; } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } return true; } -void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const std::vector& selected_ranges, int64_t row_group_rows, - std::map* page_skip_plans) { - DORIS_CHECK(page_skip_plans != nullptr); - page_skip_plans->clear(); - std::vector leaf_schemas; - collect_request_leaf_schemas(file_schema, request, &leaf_schemas); - for (const auto* leaf_schema : leaf_schemas) { - DORIS_CHECK(leaf_schema != nullptr); - ParquetPageSkipPlan page_skip_plan; - if (build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, row_group_rows, - &page_skip_plan)) { - page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + int64_t page_rows, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + DORIS_CHECK(page_statistics != nullptr); + *page_statistics = {}; + if (!column_index.__isset.null_counts || page_idx >= column_index.null_pages.size() || + page_idx >= column_index.null_counts.size()) { + return false; + } + const int64_t null_count = column_index.null_counts[page_idx]; + const bool all_null = column_index.null_pages[page_idx]; + if (page_rows < 0 || null_count < 0 || null_count > page_rows || + all_null != (null_count == page_rows)) { + // The caller supplies the exact flat page or row-group span. Contradictory optional null + // metadata must disable pruning instead of turning a partial span into an all-null proof. + return false; + } + page_statistics->has_null_count = true; + page_statistics->has_null = null_count > 0; + page_statistics->has_not_null = !all_null; + if (!page_statistics->has_not_null) { + return true; + } + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BOOLEAN: + return set_native_page_boolean_min_max(column_index, column_schema, page_idx, + page_statistics, timezone); + case tparquet::Type::INT32: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::INT64: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::FLOAT: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::FLOAT, page_statistics, + timezone); + case tparquet::Type::DOUBLE: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::DOUBLE, page_statistics, + timezone); + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + if (page_idx >= column_index.min_values.size() || + page_idx >= column_index.max_values.size()) { + return false; + } + const auto& min_value = column_index.min_values[page_idx]; + const auto& max_value = column_index.max_values[page_idx]; + const bool fixed = + column_schema.type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (fixed && + (column_schema.type_descriptor.fixed_length <= 0 || + min_value.size() != static_cast(column_schema.type_descriptor.fixed_length) || + max_value.size() != static_cast(column_schema.type_descriptor.fixed_length))) { + return false; } + const auto kind = fixed ? DecodedValueKind::FIXED_BINARY : DecodedValueKind::BINARY; + if (!set_decoded_binary_field(column_schema, kind, + StringRef(min_value.data(), min_value.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, kind, + StringRef(max_value.data(), max_value.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; + } + default: + return false; } } -} // namespace - -bool ParquetStatisticsUtils::TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - return build_page_statistics(column_index, column_schema, page_idx, page_statistics, timezone); +RowRange native_page_row_range(const tparquet::OffsetIndex& offset_index, size_t page_idx, + int64_t row_group_rows) { + const auto& locations = offset_index.page_locations; + const int64_t start = locations[page_idx].first_row_index; + const int64_t end = page_idx + 1 == locations.size() ? row_group_rows + : locations[page_idx + 1].first_row_index; + return {.start = start, .length = end - start}; } -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +} // namespace + +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - int64_t page_index_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &page_index_filter_time_sink + int64_t filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink : &pruning_stats->page_index_filter_time); DORIS_CHECK(selected_ranges != nullptr); selected_ranges->clear(); + selected_ranges->push_back({.start = 0, .length = row_group_rows}); if (page_skip_plans != nullptr) { page_skip_plans->clear(); } - if (row_group_rows <= 0) { - return Status::OK(); - } - selected_ranges->push_back(RowRange {0, row_group_rows}); - if (!config::enable_parquet_page_index || !has_expr_zonemap_filter(request, runtime_state) || - file_reader == nullptr) { - return Status::OK(); - } - - std::shared_ptr<::parquet::PageIndexReader> page_index_reader; - std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_index_reader; - try { - if (pruning_stats != nullptr) { - ++pruning_stats->page_index_read_calls; - } - { - int64_t read_page_index_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &read_page_index_time_sink - : &pruning_stats->read_page_index_time); - page_index_reader = file_reader->GetPageIndexReader(); - if (page_index_reader == nullptr) { - return Status::OK(); - } - row_group_index_reader = page_index_reader->RowGroup(row_group_idx); - } - } catch (const ::parquet::ParquetException&) { - return Status::OK(); - } catch (const std::exception&) { + if (row_group_rows <= 0 || !config::enable_parquet_page_index || + !has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) { return Status::OK(); } - if (row_group_index_reader == nullptr) { - return Status::OK(); + if (pruning_stats != nullptr) { + ++pruning_stats->page_index_read_calls; } std::map conjuncts_by_slot; @@ -1518,19 +1269,53 @@ Status select_row_group_ranges_by_page_index( conjuncts_by_slot[slot_index].push_back(conjunct); } } - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !detail::has_supported_type_defined_order(metadata, column_schema->leaf_column_id)) { + continue; + } + const auto index_it = page_indexes.find(column_schema->leaf_column_id); + if (index_it == page_indexes.end()) { + continue; + } + const auto& indexes = index_it->second; std::vector filter_ranges; - if (!select_ranges_for_expr_zonemap(row_group_index_reader, file_schema, request, - slot_index, conjuncts, row_group_rows, &filter_ranges, - pruning_stats, timezone)) { + bool usable = true; + for (size_t page_idx = 0; page_idx < indexes.offset_index.page_locations.size(); + ++page_idx) { + const auto page_range = + native_page_row_range(indexes.offset_index, page_idx, row_group_rows); + ParquetColumnStatistics statistics; + if (!build_native_page_statistics(indexes.column_index, *column_schema, page_idx, + page_range.length, &statistics, timezone)) { + usable = false; + break; + } + ZoneMapEvalContext ctx; + add_slot_zonemap(&ctx, slot_index, column_schema->type, + ParquetStatisticsUtils::MakeZoneMap(statistics)); + if (VExprContext::evaluate_zonemap_filter(conjuncts, ctx) != + ZoneMapFilterResult::kNoMatch) { + append_row_range(page_range, &filter_ranges); + } + if (pruning_stats != nullptr) { + pruning_stats->expr_zonemap_unusable_evals += ctx.stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += + ctx.stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += ctx.stats.in_zonemap_range_only_count; + } + } + if (!usable) { continue; } *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); if (selected_ranges->empty()) { - if (page_skip_plans != nullptr) { - page_skip_plans->clear(); - } if (pruning_stats != nullptr) { pruning_stats->filtered_page_rows += row_group_rows; ++pruning_stats->filtered_row_groups_by_page_index; @@ -1538,14 +1323,37 @@ Status select_row_group_ranges_by_page_index( return Status::OK(); } } + if (page_skip_plans != nullptr) { - build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, - row_group_rows, page_skip_plans); + std::vector leaves; + collect_request_leaf_schemas(file_schema, request, &leaves); + for (const auto* leaf : leaves) { + const auto index_it = page_indexes.find(leaf->leaf_column_id); + if (index_it == page_indexes.end() || leaf->max_repetition_level != 0) { + continue; + } + const auto& offset_index = index_it->second.offset_index; + ParquetPageSkipPlan skip_plan; + skip_plan.leaf_column_id = leaf->leaf_column_id; + skip_plan.skipped_pages.resize(offset_index.page_locations.size()); + skip_plan.skipped_page_compressed_sizes.resize(offset_index.page_locations.size()); + for (size_t page_idx = 0; page_idx < offset_index.page_locations.size(); ++page_idx) { + const auto range = native_page_row_range(offset_index, page_idx, row_group_rows); + if (range.length == 0 || ranges_intersect(*selected_ranges, range)) { + continue; + } + skip_plan.skipped_pages[page_idx] = 1; + skip_plan.skipped_page_compressed_sizes[page_idx] = + offset_index.page_locations[page_idx].compressed_page_size; + append_row_range(range, &skip_plan.skipped_ranges); + } + if (!skip_plan.empty()) { + page_skip_plans->emplace(skip_plan.leaf_column_id, std::move(skip_plan)); + } + } } if (pruning_stats != nullptr) { - const int64_t selected_rows = count_range_rows(*selected_ranges); - DORIS_CHECK(selected_rows <= row_group_rows); - pruning_stats->filtered_page_rows += row_group_rows - selected_rows; + pruning_stats->filtered_page_rows += row_group_rows - count_range_rows(*selected_ranges); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 9e94562bf2d37b..72381548656f9d 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include "common/status.h" @@ -27,16 +30,9 @@ #include "core/string_ref.h" #include "exprs/vexpr_fwd.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/selection_vector.h" -namespace parquet { -class BloomFilter; -class ColumnIndex; -class FileMetaData; -class ParquetFileReader; -class Statistics; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -44,6 +40,7 @@ class time_zone; namespace doris { class RuntimeState; namespace segment_v2 { +class BloomFilter; struct ZoneMap; } // namespace segment_v2 } // namespace doris @@ -51,36 +48,21 @@ struct ZoneMap; namespace doris::format::parquet { struct ParquetColumnSchema; - -// ============================================================================ -// ============================================================================ - -struct ParquetDictionaryWords { - std::vector values; - std::vector refs; - - void clear() { - values.clear(); - refs.clear(); - } - - void build_refs() { - refs.clear(); - refs.reserve(values.size()); - for (const auto& value : values) { - refs.emplace_back(value.data(), value.size()); - } - } -}; - -// Reads the PLAIN dictionary page for BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY columns and owns copied -// dictionary bytes in `values`. Both row-group pruning and row-level dictionary predicates use this -// helper so they agree on dictionary id -> Doris string value mapping. -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words); - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words); +struct ParquetFileContext; +struct ParquetTypeDescriptor; + +namespace detail { +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size); +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id); +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +} // namespace detail // ============================================================================ // ============================================================================ @@ -93,6 +75,7 @@ struct ParquetPruningStats { int64_t filtered_row_groups_by_bloom_filter = 0; // row groups pruned by bloom filter int64_t filtered_row_groups_by_page_index = 0; // row groups fully pruned by page index int64_t filtered_group_rows = 0; // rows in pruned row groups + int64_t filtered_bytes = 0; // requested bytes in pruned row groups int64_t filtered_page_rows = 0; // rows pruned by page index int64_t selected_row_ranges = 0; // selected row range count int64_t page_index_read_calls = 0; // Page Index read count @@ -100,6 +83,7 @@ struct ParquetPruningStats { int64_t row_group_filter_time = 0; // row-group pruning time (ns) int64_t page_index_filter_time = 0; // page-index pruning time (ns) int64_t read_page_index_time = 0; // page-index read time (ns) + int64_t parse_page_index_time = 0; // lazy page-index materialization time (ns) int64_t expr_zonemap_unusable_evals = 0; // VExpr ZoneMap unusable evaluations int64_t in_zonemap_point_check_count = 0; // VExpr IN ZoneMap point checks int64_t in_zonemap_range_only_count = 0; // VExpr IN ZoneMap range-only checks @@ -116,6 +100,20 @@ struct ParquetColumnStatistics { bool has_any_statistics() const { return has_null_count || has_min_max; } }; +struct NativeParquetPageIndex { + tparquet::ColumnIndex column_index; + tparquet::OffsetIndex offset_index; +}; + +enum class ParquetMetadataProbeMode { + ALL, + FOOTER_ONLY, + EXPENSIVE_ONLY, +}; + +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state); + // ============================================================================ // ============================================================================ // VExpr ZoneMap(TransformColumnStatistics + evaluate_zonemap_filter) @@ -128,32 +126,29 @@ struct ParquetStatisticsUtils { const ParquetColumnStatistics& statistics); static ParquetColumnStatistics TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, - const cctz::time_zone* timezone = nullptr); - - static bool TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone = nullptr); - - static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter); + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone = nullptr); + + static bool NativeBloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const segment_v2::BloomFilter& bloom_filter); }; Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector* candidate_row_groups, std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}, + ParquetMetadataProbeMode probe_mode = ParquetMetadataProbeMode::ALL); -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr); diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index d35181d0397178..be5ecd1c8b0772 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -17,339 +17,31 @@ #include "format_v2/parquet/parquet_type.h" -#include - -#include -#include - -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/primitive_type.h" - namespace doris::format::parquet { -namespace { - -DataTypePtr create_type(PrimitiveType type, bool nullable, int precision = 0, int scale = 0) { - return DataTypeFactory::instance().create_data_type(type, nullable, precision, scale); -} - -PrimitiveType decimal_primitive_type(int precision) { - return precision > 38 ? TYPE_DECIMAL256 : TYPE_DECIMAL128I; -} - -void mark_decimal(const ::parquet::ColumnDescriptor* column, int precision, int scale, - ParquetTypeDescriptor* result) { - result->is_decimal = true; - result->decimal_precision = precision; - result->decimal_scale = scale; - switch (column->physical_type()) { - case ::parquet::Type::INT32: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; - break; - case ::parquet::Type::INT64: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; - break; - default: - result->extra_type_info = ParquetExtraTypeInfo::NONE; - break; - } -} - -void mark_integer(int bit_width, bool is_signed, ParquetTypeDescriptor* result) { - result->integer_bit_width = bit_width; - result->is_unsigned_integer = !is_signed; -} - -DataTypePtr converted_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const bool nullable = column->max_definition_level() > 0; - switch (column->converted_type()) { - case ::parquet::ConvertedType::UTF8: - case ::parquet::ConvertedType::ENUM: - case ::parquet::ConvertedType::JSON: - case ::parquet::ConvertedType::BSON: - return create_type(TYPE_STRING, nullable); - case ::parquet::ConvertedType::DECIMAL: - mark_decimal(column, column->type_precision(), column->type_scale(), result); - return create_type(decimal_primitive_type(column->type_precision()), nullable, - column->type_precision(), column->type_scale()); - case ::parquet::ConvertedType::DATE: - return create_type(TYPE_DATEV2, nullable); - case ::parquet::ConvertedType::TIME_MILLIS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIME_MICROS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIMESTAMP_MILLIS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 3); - case ::parquet::ConvertedType::TIMESTAMP_MICROS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 6); - // Parquet stores signed and unsigned integer logical annotations on signed physical carriers: - // INT_8/UINT_8/INT_16/UINT_16/INT_32/UINT_32 use physical INT32, and - // INT_64/UINT_64 use physical INT64. Doris maps unsigned integers to the next wider - // signed type so all values in the unsigned range can be represented. - case ::parquet::ConvertedType::INT_8: - mark_integer(8, true, result); - return create_type(TYPE_TINYINT, nullable); - case ::parquet::ConvertedType::UINT_8: - mark_integer(8, false, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::INT_16: - mark_integer(16, true, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::UINT_16: - mark_integer(16, false, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::INT_32: - mark_integer(32, true, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::UINT_32: - mark_integer(32, false, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::INT_64: - mark_integer(64, true, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::UINT_64: - mark_integer(64, false, result); - return create_type(TYPE_LARGEINT, nullable); - case ::parquet::ConvertedType::NONE: - default: - return nullptr; - } -} - -DataTypePtr logical_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const auto& logical_type = column->logical_type(); - if (logical_type == nullptr || !logical_type->is_valid() || logical_type->is_none()) { - return nullptr; - } - const bool nullable = column->max_definition_level() > 0; - if (logical_type->is_string() || logical_type->is_enum() || logical_type->is_JSON() || - logical_type->is_BSON() || logical_type->is_UUID()) { - return create_type(TYPE_STRING, nullable); - } - if (logical_type->is_decimal()) { - const auto& decimal_type = static_cast(*logical_type); - mark_decimal(column, decimal_type.precision(), decimal_type.scale(), result); - return create_type(decimal_primitive_type(decimal_type.precision()), nullable, - decimal_type.precision(), decimal_type.scale()); - } - if (logical_type->is_date()) { - return create_type(TYPE_DATEV2, nullable); - } - if (logical_type->is_time()) { - const auto& time_type = static_cast(*logical_type); - if (time_type.is_adjusted_to_utc()) { - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - } - int scale = 0; - if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else { - return nullptr; - } - return create_type(TYPE_TIMEV2, nullable, 0, scale); - } - if (logical_type->is_timestamp()) { - const auto& timestamp_type = - static_cast(*logical_type); - int scale = 0; - if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::NANOS) { - scale = 6; - result->time_unit = ParquetTimeUnit::NANOS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_NS; - } else { - return nullptr; - } - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = timestamp_type.is_adjusted_to_utc(); - return create_type(TYPE_DATETIMEV2, nullable, 0, scale); - } - if (logical_type->is_int()) { - const auto& int_type = static_cast(*logical_type); - mark_integer(int_type.bit_width(), int_type.is_signed(), result); - switch (int_type.bit_width()) { - case 8: - return create_type(int_type.is_signed() ? TYPE_TINYINT : TYPE_SMALLINT, nullable); - case 16: - return create_type(int_type.is_signed() ? TYPE_SMALLINT : TYPE_INT, nullable); - case 32: - return create_type(int_type.is_signed() ? TYPE_INT : TYPE_BIGINT, nullable); - case 64: - return create_type(int_type.is_signed() ? TYPE_BIGINT : TYPE_LARGEINT, nullable); - default: - return nullptr; - } - } - if (logical_type->is_float16()) { - if (column->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY || - column->type_length() != 2) { - return nullptr; - } - result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; - return create_type(TYPE_FLOAT, nullable); - } - return nullptr; -} - -DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* column) { - const bool nullable = column->max_definition_level() > 0; - DataTypePtr type; - switch (column->physical_type()) { - case ::parquet::Type::BOOLEAN: - type = std::make_shared(); - break; - case ::parquet::Type::INT32: - type = std::make_shared(); - break; - case ::parquet::Type::INT64: - type = std::make_shared(); - break; - case ::parquet::Type::FLOAT: - type = std::make_shared(); - break; - case ::parquet::Type::DOUBLE: - type = std::make_shared(); - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - type = std::make_shared(); - break; - case ::parquet::Type::INT96: - type = create_type(TYPE_DATETIMEV2, nullable, 0, 6); - break; - default: - return nullptr; - } - return nullable ? make_nullable(type) : type; -} - -bool record_reader_physical_type_supported(::parquet::Type::type physical_type) { - switch (physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::INT96: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return true; - default: - return false; - } -} - -} // namespace - -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { - if (column == nullptr) { - return {}; - } - auto path = column->path(); - if (path) { - return path->ToDotString(); - } - return column->name(); -} - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column) { - ParquetTypeDescriptor result; - if (column == nullptr) { - return result; - } - - result.physical_type = column->physical_type(); - result.converted_type = column->converted_type(); - result.fixed_length = column->type_length(); - - if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { - result.doris_type = logical_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else if (auto converted_type = converted_type_to_doris_type(column, &result); - converted_type != nullptr) { - result.doris_type = converted_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else { - result.doris_type = physical_type_to_doris_type(column); - if (result.physical_type == ::parquet::Type::INT96) { - result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; - } - } - - result.is_string_like = !result.is_decimal && - result.extra_type_info != ParquetExtraTypeInfo::FLOAT16 && - (result.physical_type == ::parquet::Type::BYTE_ARRAY || - result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - - if (!record_reader_physical_type_supported(result.physical_type)) { - result.supports_record_reader = false; - } - return result; -} - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor) { - return type_descriptor.supports_record_reader; -} DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: + case tparquet::Type::BOOLEAN: return DecodedValueKind::BOOL; - case ::parquet::Type::INT32: + case tparquet::Type::INT32: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 32) { return DecodedValueKind::UINT32; } return DecodedValueKind::INT32; - case ::parquet::Type::INT64: + case tparquet::Type::INT64: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 64) { return DecodedValueKind::UINT64; } return DecodedValueKind::INT64; - case ::parquet::Type::INT96: + case tparquet::Type::INT96: return DecodedValueKind::INT96; - case ::parquet::Type::FLOAT: + case tparquet::Type::FLOAT: return DecodedValueKind::FLOAT; - case ::parquet::Type::DOUBLE: + case tparquet::Type::DOUBLE: return DecodedValueKind::DOUBLE; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return DecodedValueKind::FIXED_BINARY; - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: default: return DecodedValueKind::BINARY; } diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index 5d21aae6bae092..7ab3d2b3b39d8d 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -15,17 +15,13 @@ #pragma once -#include +#include #include #include "core/data_type/data_type.h" #include "core/data_type_serde/decoded_column_view.h" -namespace parquet { -class ColumnDescriptor; -} // namespace parquet - namespace doris::format::parquet { // ============================================================================ @@ -53,11 +49,15 @@ enum class ParquetTimeUnit { // ============================================================================ // ============================================================================ struct ParquetTypeDescriptor { + // Keep the V2 semantic tree on generated Thrift enums; using parquet-cpp descriptors here + // would reintroduce a second metadata model and make native planning bypassable. DataTypePtr doris_type; + // Physical fallback used only to keep file schema construction alive when the logical type is + // unsupported. Column reader creation still rejects unsupported_reason before decoding. + DataTypePtr physical_doris_type; ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; - ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; - ::parquet::ConvertedType::type converted_type = ::parquet::ConvertedType::UNDEFINED; + tparquet::Type::type physical_type = tparquet::Type::INT32; int integer_bit_width = -1; // bit width for INT_8/16/32/64 int decimal_precision = -1; // precision for DECIMAL(p,s) int decimal_scale = -1; // scale for DECIMAL(p,s) @@ -67,16 +67,9 @@ struct ParquetTypeDescriptor { bool is_timestamp = false; // whether this is a timestamp type bool timestamp_is_adjusted_to_utc = false; // whether the timestamp is UTC-normalized bool is_string_like = false; // binary type that is neither decimal nor FLOAT16 - bool supports_record_reader = true; // whether Arrow RecordReader can read this type std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported }; -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor); - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 352fbbd7c3d215..db5def75b8c8a7 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -15,145 +15,25 @@ #include "format_v2/parquet/reader/column_reader.h" -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include -#include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "runtime/runtime_profile.h" namespace doris::format::parquet { -namespace { - -class DataPageSkipFilter { -public: - DataPageSkipFilter(const ParquetPageSkipPlan* page_skip_plan, - ParquetPageSkipProfile page_skip_profile) - : _page_skip_plan(page_skip_plan), _page_skip_profile(page_skip_profile) { - DORIS_CHECK(_page_skip_plan != nullptr); - } - - bool operator()(const ::parquet::DataPageStats&) { - // Arrow invokes this callback once for each DATA_PAGE/DATA_PAGE_V2 and never for - // dictionary pages, so this ordinal matches Parquet OffsetIndex page locations. - const size_t page_idx = _next_data_page_idx++; - const bool skip = _page_skip_plan->should_skip_page(page_idx); - if (!skip) { - return false; - } - update_skip_profile(page_idx); - return true; - } - -private: - void update_skip_profile(size_t page_idx) const { - if (_page_skip_profile.skipped_pages != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); - } - if (_page_skip_profile.skipped_bytes != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_bytes, - _page_skip_plan->skipped_page_compressed_size(page_idx)); - } - } - - const ParquetPageSkipPlan* _page_skip_plan = nullptr; - ParquetPageSkipProfile _page_skip_profile; - size_t _next_data_page_idx = 0; -}; - -const ParquetPageSkipPlan* find_page_skip_plan( - const std::map* page_skip_plans, int leaf_column_id) { - if (page_skip_plans == nullptr) { - return nullptr; - } - const auto plan_it = page_skip_plans->find(leaf_column_id); - return plan_it == page_skip_plans->end() ? nullptr : &plan_it->second; -} - -void install_data_page_filter(std::unique_ptr<::parquet::PageReader>& page_reader, - const std::map* page_skip_plans, - int leaf_column_id, ParquetPageSkipProfile page_skip_profile) { - DORIS_CHECK(page_reader != nullptr); - const ParquetPageSkipPlan* page_skip_plan = - find_page_skip_plan(page_skip_plans, leaf_column_id); - if (page_skip_plan == nullptr) { - return; - } - page_reader->set_data_page_filter(DataPageSkipFilter(page_skip_plan, page_skip_profile)); -} - -bool supports_nested_scalar_record_reader(const ParquetColumnSchema& column_schema) { - if (column_schema.type_descriptor.supports_record_reader) { - return true; - } - const auto& type_descriptor = column_schema.type_descriptor; - if ((type_descriptor.extra_type_info != ParquetExtraTypeInfo::NONE && - type_descriptor.extra_type_info != ParquetExtraTypeInfo::FLOAT16) || - type_descriptor.is_decimal || type_descriptor.is_timestamp || - type_descriptor.is_string_like) { - return false; - } - if (type_descriptor.converted_type != ::parquet::ConvertedType::NONE && - type_descriptor.converted_type != ::parquet::ConvertedType::UNDEFINED) { - return false; - } - switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - return true; - default: - return false; - } - return true; -} -} // namespace +ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + ParquetColumnReaderProfile profile) + : _profile(profile), + _field_id(schema.local_id), + _leaf_column_id(schema.leaf_column_id), + _type(std::move(type)), + _name(schema.name) {} Status ParquetColumnReader::skip(int64_t rows) { return Status::NotSupported("Parquet column skip is not implemented, rows={}", rows); } -void ParquetColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - int64_t child_cursor = nested_build_level_cursor(); - const auto& child_rep_levels = nested_repetition_levels(); - const int64_t child_levels_written = nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!is_or_has_repeated_child() || child_rep_level <= parent_repetition_level) { - break; - } - } - set_nested_build_level_cursor(child_cursor); -} - void ParquetColumnReader::update_reader_read_rows(int64_t rows) const { if (_profile.reader_read_rows != nullptr) { COUNTER_UPDATE(_profile.reader_read_rows, rows); @@ -166,21 +46,16 @@ void ParquetColumnReader::update_reader_skip_rows(int64_t rows) const { } } -Status ParquetColumnReader::select(const SelectionVector& sel, uint16_t selected_rows, +Status ParquetColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, MutableColumnPtr& column) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Parquet selected read result is null for column {}", - name()); - } - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + DORIS_CHECK(column); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); - const auto ranges = selection_to_ranges(sel, selected_rows); + const auto ranges = selection_to_ranges(selection, selected_rows); int64_t cursor = 0; for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument("Invalid parquet selection range [{}, {}) for column {}", - range.start, range.start + range.length, name()); - } + DORIS_CHECK(range.start >= cursor); + DORIS_CHECK(range.start + range.length <= batch_rows); RETURN_IF_ERROR(skip(range.start - cursor)); int64_t range_rows_read = 0; @@ -206,453 +81,15 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& name()); } -ParquetColumnReaderFactory::ParquetColumnReaderFactory( - std::shared_ptr<::parquet::RowGroupReader> row_group, int num_leaf_columns, - const std::map* page_skip_plans, - ParquetPageSkipProfile page_skip_profile, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile column_reader_profile) - : _row_group(std::move(row_group)), - _record_readers(static_cast(num_leaf_columns)), - _dictionary_record_readers(static_cast(num_leaf_columns)), - _page_skip_plans(page_skip_plans), - _page_skip_profile(page_skip_profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _column_reader_profile(column_reader_profile) {} - -std::unique_ptr ParquetColumnReaderFactory::create_row_position_column_reader( - int64_t row_group_first_row) const { - return std::make_unique(row_group_first_row, _column_reader_profile); -} - -std::unique_ptr ParquetColumnReaderFactory::create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const { - return std::make_unique(context, row_group_first_row, - _column_reader_profile); -} - -Status ParquetColumnReaderFactory::make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, bool use_page_skip_plan, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - const auto* page_skip_plan = - use_page_skip_plan ? find_page_skip_plan(_page_skip_plans, column_schema.leaf_column_id) - : nullptr; - *reader = std::make_unique(column_schema, std::move(record_reader), - page_skip_plan, _timezone, _enable_strict_mode, - _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_scalar_column_reader( - const ParquetColumnSchema& column_schema, bool is_nested, bool read_dictionary, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (!column_schema.type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, - column_schema.type_descriptor.unsupported_reason); - } - if (is_nested && column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { - return Status::InvalidArgument("Parquet nested scalar reader requires primitive column {}", - column_schema.name); - } - if (column_schema.leaf_column_id < 0 || - column_schema.leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - column_schema.leaf_column_id, column_schema.name); - } - if (column_schema.descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", - column_schema.name); - } - if (!is_nested && (column_schema.descriptor->max_repetition_level() != 0 || - column_schema.descriptor->max_definition_level() > 1)) { - return Status::NotSupported( - "Current parquet scalar reader only supports flat primitive columns; column {} is " - "not supported", - column_schema.name); - } - if (is_nested && !supports_nested_scalar_record_reader(column_schema)) { - return Status::NotSupported( - "Current parquet nested scalar reader does not support column {}", - column_schema.name); - } - if (!is_nested && !column_schema.type_descriptor.supports_record_reader) { - return Status::NotSupported("Current parquet scalar reader does not support column {}", - column_schema.name); - } - std::shared_ptr<::parquet::internal::RecordReader> record_reader; - // Nested readers implement skip() by materializing rows into a scratch column. If Arrow - // page filtering is also installed, those scratch reads can consume the next selected row - // after a page-index range gap. Keep page filtering on flat scalar readers only. - RETURN_IF_ERROR(get_record_reader(column_schema.leaf_column_id, column_schema.descriptor, - column_schema.name, !is_nested, read_dictionary, - &record_reader)); - return make_scalar_column_reader(column_schema, std::move(record_reader), !is_nested, reader); -} - -// 1. RowGroupReader::GetColumnPageReader(leaf_column_id) -> Arrow PageReader -Status ParquetColumnReaderFactory::get_record_reader( - int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, const std::string& name, - bool install_page_filter, bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (_row_group == nullptr) { - return Status::InternalError("Parquet row group reader is not initialized for column {}", - name); - } - if (leaf_column_id < 0 || leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - leaf_column_id, name); - } - if (descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", name); - } - auto& record_readers = read_dictionary ? _dictionary_record_readers : _record_readers; - if (record_readers[leaf_column_id] == nullptr) { - try { - auto page_reader = _row_group->GetColumnPageReader(leaf_column_id); - if (install_page_filter) { - install_data_page_filter(page_reader, _page_skip_plans, leaf_column_id, - _page_skip_profile); - } - const auto level_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(descriptor); - record_readers[leaf_column_id] = ::parquet::internal::RecordReader::Make( - descriptor, level_info, ::arrow::default_memory_pool(), - /*read_dictionary=*/read_dictionary, - /*read_dense_for_nullable=*/false); - record_readers[leaf_column_id]->SetPageReader(std::move(page_reader)); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } - } - if (record_readers[leaf_column_id] == nullptr) { - return Status::Corruption("Failed to create parquet record reader for column {}", name); - } - *reader = record_readers[leaf_column_id]; - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_struct_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - std::vector> child_readers; - child_readers.reserve(column_schema.children.size()); - std::vector child_output_indices; - child_output_indices.reserve(column_schema.children.size()); - DataTypes projected_child_types; - Strings projected_child_names; - for (size_t child_idx = 0; child_idx < column_schema.children.size(); ++child_idx) { - const auto& child_schema = column_schema.children[child_idx]; - const auto* child_projection = - format::find_child_projection(projection, child_schema->local_id); - if (!format::is_child_projected(projection, child_schema->local_id)) { - continue; - } - std::unique_ptr child_reader; - RETURN_IF_ERROR( - create_column_reader(*child_schema, child_projection, true, false, &child_reader)); - child_output_indices.push_back(static_cast(projected_child_types.size())); - projected_child_types.push_back(make_nullable(child_reader->type())); - projected_child_names.push_back(child_reader->name()); - child_readers.push_back(std::move(child_reader)); - } - if (format::is_partial_projection(projection) && - projected_child_types.size() != projection->children.size()) { - return Status::InvalidArgument( - "Parquet STRUCT projection for column {} contains invalid child", - column_schema.name); - } - if (projected_child_types.empty() && !column_schema.children.empty()) { - return Status::NotSupported("Parquet STRUCT projection for column {} contains no children", - column_schema.name); - } - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(projection)) { - type = std::make_shared(projected_child_types, projected_child_names); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique( - column_schema, std::move(type), std::move(child_readers), - std::move(child_output_indices), _column_reader_profile); +Status ParquetColumnReader::select_with_fixed_width_filter(const SelectionVector&, uint16_t, + int64_t, const VExprSPtrs&, int, + IColumn*, IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *used_filter = false; return Status::OK(); } -Status ParquetColumnReaderFactory::create_list_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for column {}", - column_schema.name); - } - std::unique_ptr element_reader; - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - if (format::is_partial_projection(projection) && element_projection == nullptr) { - return Status::NotSupported("Parquet LIST projection for column {} contains no element", - column_schema.name); - } - RETURN_IF_ERROR( - create_column_reader(element_schema, element_projection, true, false, &element_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(element_projection)) { - type = std::make_shared(element_reader->type()); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique(column_schema, std::move(type), - std::move(element_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_map_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP layout for column {}", - column_schema.name); - } - const auto& key_schema = *column_schema.children[0]; - const auto& value_schema = *column_schema.children[1]; - const auto* value_projection = format::find_child_projection(projection, value_schema.local_id); - if (format::is_partial_projection(projection)) { - if (value_projection == nullptr) { - return Status::NotSupported("Parquet MAP projection for column {} contains no value", - column_schema.name); - } - for (const auto& child_projection : projection->children) { - if (child_projection.local_id() == key_schema.local_id) { - continue; - } - if (child_projection.local_id() != value_schema.local_id) { - return Status::InvalidArgument( - "Parquet MAP projection for column {} contains invalid child", - column_schema.name); - } - } - } - std::unique_ptr key_reader; - // MAP materialization always needs the full key stream. It owns entry existence, offsets and - // key equality semantics, so MAP projection is defined only as value-subtree pruning. - RETURN_IF_ERROR(create_column_reader(key_schema, nullptr, true, false, &key_reader)); - std::unique_ptr value_reader; - RETURN_IF_ERROR( - create_column_reader(value_schema, value_projection, true, false, &value_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(value_projection)) { - type = std::make_shared(make_nullable(key_reader->type()), - make_nullable(value_reader->type())); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = - std::make_unique(column_schema, std::move(type), std::move(key_reader), - std::move(value_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, - bool read_dictionary) const { - return create_column_reader(column_schema, projection, false, read_dictionary, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - return create_count_shape_reader_impl(column_schema, projection, false, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader_impl( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, is_nested, false, reader); - case ParquetColumnSchemaKind::STRUCT: { - if (column_schema.children.empty()) { - return Status::NotSupported("Parquet COUNT shape reader found empty STRUCT column {}", - column_schema.name); - } - const ParquetColumnSchema* child_schema = nullptr; - const format::LocalColumnIndex* child_projection = nullptr; - if (format::is_partial_projection(projection)) { - const auto child_id = projection->children[0].local_id(); - const auto child_it = std::ranges::find_if( - column_schema.children, - [&](const auto& child) { return child->local_id == child_id; }); - if (child_it == column_schema.children.end()) { - return Status::InvalidArgument( - "Parquet COUNT projection for column {} contains invalid child", - column_schema.name); - } - child_schema = child_it->get(); - child_projection = &projection->children[0]; - } else { - child_schema = column_schema.children[0].get(); - } - DORIS_CHECK(child_schema != nullptr); - return create_count_shape_reader_impl(*child_schema, child_projection, true, reader); - } - case ParquetColumnSchemaKind::LIST: { - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for COUNT column {}", - column_schema.name); - } - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - return create_count_shape_reader_impl(element_schema, element_projection, true, reader); - } - case ParquetColumnSchemaKind::MAP: { - if (column_schema.children.empty()) { - return Status::NotSupported("Unsupported parquet MAP layout for COUNT column {}", - column_schema.name); - } - // The key stream defines MAP entry existence and offsets. Counting top-level MAP NULL-ness - // from it avoids creating a value reader, which is the expensive path for files with huge - // MAP value strings. - return create_count_shape_reader_impl(*column_schema.children[0], nullptr, true, reader); - } - } - return Status::NotSupported("Unsupported parquet column schema kind for COUNT column {}", - column_schema.name); -} - -Status ParquetColumnReaderFactory::create_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, bool read_dictionary, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (is_nested) { - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet scalar projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, true, false, reader); - } - return create_scalar_column_reader(column_schema, false, read_dictionary, reader); - case ParquetColumnSchemaKind::STRUCT: - return create_struct_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::LIST: - return create_list_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::MAP: - return create_map_column_reader(column_schema, projection, reader); - } - return Status::NotSupported("Unsupported parquet column schema kind for column {}", - column_schema.name); -} - -ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, - ParquetColumnReaderProfile profile) - : _profile(profile), - _field_id(schema.local_id), - _leaf_column_id(schema.leaf_column_id), - _nullable_definition_level(schema.nullable_definition_level), - _repeated_repetition_level(schema.repeated_repetition_level), - _definition_level(schema.definition_level), - _repetition_level(schema.repetition_level), - _repeated_ancestor_definition_level(schema.repeated_ancestor_definition_level), - _type(std::move(type)), - _name(schema.name) {} - -Status ParquetColumnReader::load_nested_batch(int64_t) { - return Status::NotSupported("Parquet nested batch load is not supported for column {}", _name); -} - -Status ParquetColumnReader::load_nested_levels_batch(int64_t) { - return Status::NotSupported("Parquet nested levels batch load is not supported for column {}", - _name); -} - -Status ParquetColumnReader::build_nested_column(int64_t, MutableColumnPtr&, int64_t*) { - return Status::NotSupported("Parquet nested column build is not supported for column {}", - _name); -} - -Status ParquetColumnReader::consume_nested_column(int64_t, int64_t*) { - return Status::NotSupported("Parquet nested column consume is not supported for column {}", - _name); -} - -Status ParquetColumnReader::skip_nested_rows(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - // A nested parent row may expand to many child values. Capping the number of parent rows per - // loaded batch bounds that amplification for large holes. The consume interface advances the - // loaded definition/repetition levels recursively without constructing a discarded Column. - constexpr int64_t MAX_NESTED_SKIP_BATCH_SIZE = 4096; - int64_t remaining_rows = rows; - while (remaining_rows > 0) { - const int64_t batch_rows = std::min(remaining_rows, MAX_NESTED_SKIP_BATCH_SIZE); - RETURN_IF_ERROR(load_nested_levels_batch(batch_rows)); - int64_t rows_consumed = 0; - RETURN_IF_ERROR(consume_nested_column(batch_rows, &rows_consumed)); - if (rows_consumed != batch_rows) { - return Status::Corruption( - "Failed to skip nested parquet column {}: skipped {} of {} rows in batch", - _name, rows_consumed, batch_rows); - } - remaining_rows -= batch_rows; - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -const std::vector& ParquetColumnReader::nested_definition_levels() const { - static const std::vector empty; - return empty; -} - -const std::vector& ParquetColumnReader::nested_repetition_levels() const { - static const std::vector empty; - return empty; -} - -int64_t ParquetColumnReader::nested_levels_written() const { - return 0; -} - -bool ParquetColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 51dbd44c11c226..6914a5be7d165d 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -16,216 +16,81 @@ #pragma once #include -#include -#include #include -#include #include "common/status.h" -#include "core/column/column_nullable.h" +#include "core/column/column.h" #include "core/data_type/data_type.h" -#include "format_v2/column_data.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" #include "format_v2/parquet/selection_vector.h" -#include "runtime/runtime_profile.h" - -namespace parquet { -class ColumnDescriptor; -class RowGroupReader; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris { -class IColumn; -} // namespace doris namespace doris::format::parquet { struct ParquetColumnSchema; +// Scan-time column contract for FileScannerV2. +// +// Physical Parquet columns are implemented by NativeColumnReader, which owns Doris' native page +// decoder and writes directly into the destination Doris column. Synthetic row-position/global-id +// readers implement the same cursor contract. Arrow RecordReader, ParquetLeafBatch, decoded value +// views, and the former nested build/consume protocol intentionally do not belong to this API. class ParquetColumnReader { public: virtual ~ParquetColumnReader() = default; virtual int file_column_id() const { return _field_id; } - virtual int parquet_leaf_column_id() const { return _leaf_column_id; } - - int16_t nullable_definition_level() const { return _nullable_definition_level; } - int16_t repeated_repetition_level() const { return _repeated_repetition_level; } - virtual const DataTypePtr& type() const { return _type; } virtual const std::string& name() const { return _name; } const ParquetColumnReaderProfile& profile() const { return _profile; } + // Consume rows from the row-group cursor and append them to column. virtual Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) = 0; + // Consume rows without materializing values. virtual Status skip(int64_t rows); - virtual Status select(const SelectionVector& sel, uint16_t selected_rows, int64_t batch_rows, - MutableColumnPtr& column); + // Consume batch_rows and append only selection[0, selected_rows). The default implementation + // coalesces adjacent indices into ranges; native readers override it with one decoder call. + virtual Status select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column); - virtual Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, + virtual Status select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter); - virtual Status load_nested_batch(int64_t rows); - - // Shape-only load interface for COUNT(col) and skip. Implementations guarantee only that - // nested_definition_levels(), nested_repetition_levels(), and nested_levels_written() are - // available; value indices and payload columns may be absent. Callers may inspect the levels or - // call consume_nested_column(), but must not call build_nested_column() afterwards. For example, - // skipping ARRAY uses this method to find ARRAY boundaries without constructing a - // ColumnString. The underlying Arrow reader may still decode a page because it has no public - // levels-only API. Normal scans that need output values use load_nested_batch() instead. - virtual Status load_nested_levels_batch(int64_t rows); - - virtual Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read); - - // Consume logical values from a batch previously loaded by load_nested_batch() or - // load_nested_levels_batch() without appending them to an output Column. Implementations must - // advance exactly the same nested level cursors and perform the same shape/null/alignment - // validation as build_nested_column(). The levels-only form is preferred for skip paths because - // it avoids transferring leaf payloads into Doris Columns when they will be discarded. - // - // `length_upper_bound` is expressed at this reader's logical level, not in physical leaf - // values. For example, consuming two rows from ARRAY [[1, 2], []] consumes two parent ARRAY - // rows but only two element values. A MAP implementation must also consume key/value streams - // in lockstep, while a nullable STRUCT consumes no child value for a null parent. - // - // Callers must not use the ordinary skip() after either load call: the leaf stream has already - // advanced into an in-memory nested batch, and doing so would advance it twice. - // `values_consumed` may be smaller than the requested bound only when the loaded batch ends. - virtual Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed); - - virtual const std::vector& nested_definition_levels() const; - virtual const std::vector& nested_repetition_levels() const; - virtual int64_t nested_levels_written() const; - virtual bool is_or_has_repeated_child() const; - virtual void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level); - - int64_t nested_build_level_cursor() const { return _nested_build_level_cursor; } - void set_nested_build_level_cursor(int64_t cursor) { - DORIS_CHECK(cursor >= 0); - _nested_build_level_cursor = cursor; + // Consume batch_rows and evaluate eligible fixed-width values without first constructing a + // complete predicate column. Append survivors when projected_column is non-null. Implementations + // must leave the cursor untouched when used_filter=false. + virtual Status select_with_fixed_width_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter); + + // Native statistics are cumulative and can be recursively aggregated for complex columns. + // Flush once at the scheduler batch boundary instead of snapshotting after each operation. + virtual void flush_profile() {} + virtual bool crossed_page_since_last_batch() { return false; } + virtual Result dictionary_values() { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } - void reset_nested_build_level_cursor() { _nested_build_level_cursor = 0; } protected: - ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, ParquetColumnReaderProfile profile = {}); ParquetColumnReader() = default; - // Load shape levels and consume skipped parent rows in bounded batches. The bound limits level - // memory when a parent expands to many children; the levels-only load plus - // consume_nested_column() avoids payload materialization and output Columns. - Status skip_nested_rows(int64_t rows); + void update_reader_read_rows(int64_t rows) const; void update_reader_skip_rows(int64_t rows) const; ParquetColumnReaderProfile _profile; - const int _field_id = -1; // child ordinal in the parent node - const int _leaf_column_id = -1; // Parquet physical leaf column id (-1 = non-leaf) - const int16_t _nullable_definition_level = - 0; // definition-level threshold where this node becomes nullable - const int16_t _repeated_repetition_level = - 0; // repetition level of the nearest repeated ancestor - const int16_t _definition_level = 0; // definition level accumulated to this node - const int16_t _repetition_level = 0; // repetition level accumulated to this node - const int16_t _repeated_ancestor_definition_level = - 0; // definition level of the nearest repeated ancestor - const DataTypePtr _type; // Doris target type - const std::string _name; // column name for error messages - int64_t _nested_build_level_cursor = 0; // nested build cursor (current level position) + const int _field_id = -1; + const int _leaf_column_id = -1; + const DataTypePtr _type; + const std::string _name; }; -class ParquetColumnReaderFactory { -public: - ParquetColumnReaderFactory(std::shared_ptr<::parquet::RowGroupReader> row_group, - int num_leaf_columns, - const std::map* page_skip_plans = nullptr, - ParquetPageSkipProfile page_skip_profile = {}, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false, - ParquetColumnReaderProfile column_reader_profile = {}); - - Status create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, bool read_dictionary = false) const; - - // Create a scalar reader for one representative leaf that carries the top-level column shape. - // This is used by COUNT(col): the caller needs definition/repetition levels to decide whether - // the top-level value is NULL, but must not materialize heavy payload leaves. MAP deliberately - // uses the key leaf because the key stream owns entry existence and avoids reading value pages. - Status create_count_shape_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create(const ParquetColumnSchema& column_schema, - std::unique_ptr* reader) const { - return create(column_schema, nullptr, reader); - } - - std::unique_ptr create_row_position_column_reader( - int64_t row_group_first_row) const; - std::unique_ptr create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const; - -private: - Status create_scalar_column_reader(const ParquetColumnSchema& column_schema, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - - Status create_struct_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_list_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_map_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - Status create_count_shape_reader_impl(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - bool is_nested, - std::unique_ptr* reader) const; - - Status get_record_reader(int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, - const std::string& name, bool install_page_filter, - bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const; - - Status make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - bool use_page_skip_plan, std::unique_ptr* reader) const; - - std::shared_ptr<::parquet::RowGroupReader> _row_group; // Arrow RowGroup reader - mutable std::vector> - _record_readers; // RecordReader cache by leaf_column_id - mutable std::vector> - _dictionary_record_readers; // dictionary-exposing RecordReader cache by leaf_column_id - const std::map* _page_skip_plans = - nullptr; // page-index pruning result - ParquetPageSkipProfile _page_skip_profile; // page skip profile - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - ParquetColumnReaderProfile _column_reader_profile; // column reader profile -}; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp new file mode 100644 index 00000000000000..171107107f1a38 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/count_column_reader.h" + +#include +#include +#include + +#include "common/config.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { +namespace { + +Status find_count_leaf(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const ParquetColumnSchema** leaf) { + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", + schema.name); + } + *leaf = &schema; + return Status::OK(); + case ParquetColumnSchemaKind::STRUCT: { + DORIS_CHECK(!schema.children.empty()); + if (!format::is_partial_projection(projection)) { + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + DORIS_CHECK(!projection->children.empty()); + const auto child_id = projection->children.front().local_id(); + const auto child = std::ranges::find_if(schema.children, [child_id](const auto& candidate) { + return candidate->local_id == child_id; + }); + if (child == schema.children.end()) { + return Status::InvalidArgument( + "Parquet COUNT projection for column {} contains invalid child", schema.name); + } + return find_count_leaf(**child, &projection->children.front(), leaf); + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto& element = *schema.children.front(); + return find_count_leaf(element, format::find_child_projection(projection, element.local_id), + leaf); + } + case ParquetColumnSchemaKind::MAP: + // The key leaf owns entry existence and top-level MAP shape. Choosing it also avoids + // reading a potentially huge value BYTE_ARRAY for COUNT(map_col). + DORIS_CHECK(!schema.children.empty()); + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); +} + +NativeFieldSchema* find_physical_leaf(NativeFieldSchema* field, int physical_column_index) { + DORIS_CHECK(field != nullptr); + if (field->children.empty()) { + return field->physical_column_index == physical_column_index ? field : nullptr; + } + for (auto& child : field->children) { + if (auto* result = find_physical_leaf(&child, physical_column_index); result != nullptr) { + return result; + } + } + return nullptr; +} + +} // namespace + +CountColumnReader::CountColumnReader(std::string name, + std::unique_ptr level_reader, + ParquetColumnReaderProfile profile) + : _level_reader(std::move(level_reader)), _profile(profile), _name(std::move(name)) {} + +CountColumnReader::~CountColumnReader() { + sync_profile(); +} + +Status CountColumnReader::create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(reader != nullptr); + const ParquetColumnSchema* leaf_schema = nullptr; + RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf_schema)); + DORIS_CHECK(leaf_schema != nullptr); + DORIS_CHECK(leaf_schema->leaf_column_id >= 0); + + const auto& thrift_metadata = metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::InvalidArgument("Invalid Parquet COUNT row group {}", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + if (leaf_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet COUNT leaf {} for row group {}", + leaf_schema->leaf_column_id, row_group_id); + } + + DORIS_CHECK(root_schema.local_id >= 0); + auto* root_field = + const_cast(metadata->schema().get_column(root_schema.local_id)); + DORIS_CHECK(root_field != nullptr); + auto* leaf_field = find_physical_leaf(root_field, leaf_schema->leaf_column_id); + if (leaf_field == nullptr) { + return Status::Corruption("Cannot resolve Parquet COUNT physical leaf {} for column {}", + leaf_schema->leaf_column_id, root_schema.name); + } + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + std::unique_ptr level_reader; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::LevelReader::create( + std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, + row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, + enable_page_cache, page_cache_file_key, compat, &level_reader)); + reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); + return Status::OK(); +} + +Status CountColumnReader::skip(int64_t rows) { + DORIS_CHECK(rows >= 0); + if (rows == 0) { + return Status::OK(); + } + { + SCOPED_TIMER(_profile.level_only_skip_time); + RETURN_IF_ERROR(_level_reader->skip_rows(static_cast(rows))); + } + if (_profile.reader_skip_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_skip_rows, rows); + } + sync_profile(); + return Status::OK(); +} + +Status CountColumnReader::read_levels(int64_t rows, int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(rows_read != nullptr); + _definition_levels.clear(); + _repetition_levels.clear(); + _levels_written = 0; + size_t native_rows_read = 0; + { + SCOPED_TIMER(_profile.level_only_read_time); + RETURN_IF_ERROR(_level_reader->read_rows(static_cast(rows), &_repetition_levels, + &_definition_levels, &native_rows_read)); + } + *rows_read = static_cast(native_rows_read); + if (*rows_read != rows || _definition_levels.size() != _repetition_levels.size()) { + return Status::Corruption( + "Parquet COUNT level reader returned {} rows and {}/{} levels for {}", *rows_read, + _definition_levels.size(), _repetition_levels.size(), _name); + } + _levels_written = static_cast(_definition_levels.size()); + if (_levels_written < *rows_read) { + return Status::Corruption("Parquet COUNT returned {} levels for {} rows in column {}", + _levels_written, *rows_read, _name); + } + if (_profile.reader_read_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_read_rows, *rows_read); + } + sync_profile(); + return Status::OK(); +} + +void CountColumnReader::sync_profile() { + if (_level_reader == nullptr) { + return; + } + const auto stats = _level_reader->statistics(); + const auto& reported = _reported_native_stats; +#define UPDATE_NATIVE_PROFILE(counter, field) \ + if (_profile.counter != nullptr) { \ + COUNTER_UPDATE(_profile.counter, stats.field - reported.field); \ + } + UPDATE_NATIVE_PROFILE(decompress_time, decompress_time); + UPDATE_NATIVE_PROFILE(decompress_count, decompress_cnt); + UPDATE_NATIVE_PROFILE(decode_header_time, decode_header_time); + UPDATE_NATIVE_PROFILE(decode_value_time, decode_value_time); + UPDATE_NATIVE_PROFILE(decode_dictionary_time, decode_dict_time); + UPDATE_NATIVE_PROFILE(decode_level_time, decode_level_time); + UPDATE_NATIVE_PROFILE(skip_page_header_count, skip_page_header_num); + UPDATE_NATIVE_PROFILE(parse_page_header_count, parse_page_header_num); + UPDATE_NATIVE_PROFILE(read_page_header_time, read_page_header_time); + UPDATE_NATIVE_PROFILE(page_read_count, page_read_counter); + UPDATE_NATIVE_PROFILE(page_cache_write_count, page_cache_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_write_count, page_cache_compressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_write_count, + page_cache_decompressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_hit_count, page_cache_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_miss_count, page_cache_missing_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_hit_count, page_cache_compressed_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_hit_count, page_cache_decompressed_hit_counter); +#undef UPDATE_NATIVE_PROFILE + _reported_native_stats = stats; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h new file mode 100644 index 00000000000000..789da8534d2ce3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris { +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { +class NativeParquetMetadata; +struct ParquetColumnSchema; +// Shape-only COUNT(nullable_col) reader. It uses v2's native LevelReader, so BYTE_ARRAY payloads +// are skipped in the encoding stream and never copied into Arrow builders or Doris strings. +class CountColumnReader { +public: + ~CountColumnReader(); + + static Status create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + Status skip(int64_t rows); + Status read_levels(int64_t rows, int64_t* rows_read); + + const std::vector& definition_levels() const { return _definition_levels; } + const std::vector& repetition_levels() const { return _repetition_levels; } + int64_t levels_written() const { return _levels_written; } + +private: + CountColumnReader(std::string name, std::unique_ptr level_reader, + ParquetColumnReaderProfile profile); + void sync_profile(); + + std::unique_ptr _level_reader; + ParquetColumnReaderProfile _profile; + std::string _name; + std::vector _definition_levels; + std::vector _repetition_levels; + native::ColumnChunkReaderStatistics _reported_native_stats; + int64_t _levels_written = 0; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp deleted file mode 100644 index c042fc99b512aa..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/list_column_reader.h" - -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status ListColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status ListColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status ListColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_batch(rows); -} - -Status ListColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_levels_batch(rows); -} - -Status ListColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet list build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status ListColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet list process result pointer for column {}", - _name); - } - DORIS_CHECK(_element_reader != nullptr); - ColumnArray* array_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr nested_column; - if (column != nullptr) { - array_column = array_column_from_output(*column); - DORIS_CHECK(array_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - nested_column = array_column->get_data_ptr()->assert_mutable(); - const auto& element_output_type = - assert_cast(*remove_nullable(_type)).get_nested_type(); - remove_nullable_wrapper_if_not_expected(element_output_type, &nested_column); - } - - const auto& def_levels = _element_reader->nested_definition_levels(); - const auto& rep_levels = _element_reader->nested_repetition_levels(); - const int64_t levels_written = _element_reader->nested_levels_written(); - std::vector entry_counts; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet LIST column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - int64_t child_value_count = 0; - if (!_element_reader->is_or_has_repeated_child()) { - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(total_entries), nested_column, &child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(total_entries), &child_value_count)); - } - } else { - uint64_t pending_entries = 0; - auto flush_pending_entries = [&]() -> Status { - if (pending_entries == 0) { - return Status::OK(); - } - int64_t span_child_value_count = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(pending_entries), nested_column, - &span_child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(pending_entries), &span_child_value_count)); - } - if (span_child_value_count != static_cast(pending_entries)) { - return Status::Corruption( - "Parquet LIST column {} built {} child values, expected {}", _name, - span_child_value_count, pending_entries); - } - child_value_count += span_child_value_count; - pending_entries = 0; - return Status::OK(); - }; - - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - if (entry_count > 0) { - pending_entries += entry_count; - continue; - } - RETURN_IF_ERROR(flush_pending_entries()); - _element_reader->advance_nested_build_level_cursor_past_parent(_repetition_level); - } - RETURN_IF_ERROR(flush_pending_entries()); - } - if (child_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet LIST column {} built {} child values, expected {}", - _name, child_value_count, total_entries); - } - if (column != nullptr) { - array_column->get_data_ptr() = std::move(nested_column); - append_offsets(array_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& ListColumnReader::nested_definition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_definition_levels(); -} - -const std::vector& ListColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_repetition_levels(); -} - -int64_t ListColumnReader::nested_levels_written() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_levels_written(); -} - -bool ListColumnReader::is_or_has_repeated_child() const { - return true; -} - -void ListColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_element_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _element_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h deleted file mode 100644 index d64be546e394d7..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class ListColumnReader final : public ParquetColumnReader { -public: - ListColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr element_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _element_reader(std::move(element_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr - _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp deleted file mode 100644 index 8217d0c013abc0..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/map_column_reader.h" - -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status MapColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status MapColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status MapColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_batch(rows)); - return _value_reader->load_nested_batch(rows); -} - -Status MapColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_levels_batch(rows)); - return _value_reader->load_nested_levels_batch(rows); -} - -Status MapColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet map build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status MapColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet map process result pointer for column {}", - _name); - } - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ColumnMap* map_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr key_column; - MutableColumnPtr value_column; - if (column != nullptr) { - map_column = map_column_from_output(*column); - DORIS_CHECK(map_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - key_column = map_column->get_keys_ptr()->assert_mutable(); - value_column = map_column->get_values_ptr()->assert_mutable(); - const auto& map_output_type = assert_cast(*remove_nullable(_type)); - remove_nullable_wrapper_if_not_expected(map_output_type.get_key_type(), &key_column); - remove_nullable_wrapper_if_not_expected(map_output_type.get_value_type(), &value_column); - } - - const auto& def_levels = _key_reader->nested_definition_levels(); - const auto& rep_levels = _key_reader->nested_repetition_levels(); - const int64_t levels_written = _key_reader->nested_levels_written(); - - std::vector entry_counts; - std::vector map_level_indices; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - const int64_t current_level_idx = level_idx; - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - map_level_indices.push_back(current_level_idx); - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet MAP column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - int64_t key_value_count = 0; - size_t key_start = 0; - if (column != nullptr) { - key_start = key_column->size(); - RETURN_IF_ERROR(_key_reader->build_nested_column(static_cast(total_entries), - key_column, &key_value_count)); - } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { - // MAP keys are required even if a projected Doris key type is nullable. Validate each - // actual entry directly from the key level stream while advancing past empty/null maps. - for (const int64_t key_level_idx : map_level_indices) { - if (def_levels[key_level_idx] >= _definition_level) { - RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); - ++key_value_count; - } - } - scalar_key_reader->set_nested_build_level_cursor(level_idx); - } else { - RETURN_IF_ERROR(_key_reader->consume_nested_column(static_cast(total_entries), - &key_value_count)); - } - if (key_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} keys, expected {}", _name, - key_value_count, total_entries); - } - if (column != nullptr) { - if (const auto* nullable_key_column = check_and_get_column(*key_column); - nullable_key_column != nullptr && - nullable_key_column->has_null(key_start, nullable_key_column->size())) { - return Status::Corruption("Parquet MAP column {} contains null key", _name); - } - } - int64_t value_count = 0; - if (auto* scalar_value_reader = dynamic_cast(_value_reader.get())) { - const auto& value_def_levels = scalar_value_reader->nested_definition_levels(); - const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); - const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); - int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); - for (const int64_t key_level_idx : map_level_indices) { - while (value_level_idx < value_levels_written && - (value_rep_levels[value_level_idx] > _repetition_level || - value_def_levels[value_level_idx] < min_parent_definition_level || - (value_rep_levels[value_level_idx] >= _repetition_level && - value_def_levels[value_level_idx] < _repeated_ancestor_definition_level))) { - ++value_level_idx; - } - if (value_level_idx >= value_levels_written) { - return Status::Corruption( - "Parquet MAP column {} value stream ended before key stream", _name); - } - // MAP is encoded as a repeated key/value struct. The key stream owns entry existence, - // but the value stream still has one shape slot for every consumed MAP slot. Consume - // value slots in lockstep with key slots so shape-only slots from empty/null maps do - // not become scalar values. - if (value_rep_levels[value_level_idx] != rep_levels[key_level_idx]) { - return Status::Corruption( - "Parquet MAP column {} value repetition level is not aligned with key " - "stream", - _name); - } - if (def_levels[key_level_idx] >= _definition_level) { - if (column != nullptr) { - RETURN_IF_ERROR(scalar_value_reader->append_nested_value(value_level_idx, - value_column)); - } else { - RETURN_IF_ERROR( - scalar_value_reader->validate_nested_value(value_level_idx, false)); - } - ++value_count; - } - ++value_level_idx; - } - scalar_value_reader->set_nested_build_level_cursor(value_level_idx); - } else { - // Complex MAP values own their nested shape below the entry slot, so they recursively - // process exactly one child value for each MAP entry. - if (column != nullptr) { - RETURN_IF_ERROR(_value_reader->build_nested_column(static_cast(total_entries), - value_column, &value_count)); - } else { - RETURN_IF_ERROR(_value_reader->consume_nested_column( - static_cast(total_entries), &value_count)); - } - } - if (value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} values, expected {}", _name, - value_count, total_entries); - } - - if (column != nullptr) { - map_column->get_keys_ptr() = std::move(key_column); - map_column->get_values_ptr() = std::move(value_column); - append_offsets(map_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& MapColumnReader::nested_definition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_definition_levels(); -} - -const std::vector& MapColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_repetition_levels(); -} - -int64_t MapColumnReader::nested_levels_written() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_levels_written(); -} - -bool MapColumnReader::is_or_has_repeated_child() const { - return true; -} - -void MapColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _key_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _value_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h deleted file mode 100644 index 1a8ca9c70d8c5b..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -// 2. build_nested_column() -> -class MapColumnReader final : public ParquetColumnReader { -public: - MapColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr key_reader, - std::unique_ptr value_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _key_reader(std::move(key_reader)), - _value_reader(std::move(value_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr _key_reader; // key column reader (always read fully) - std::unique_ptr - _value_reader; // value column reader (can be pruned by projection) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp new file mode 100644 index 00000000000000..3421ecdd296892 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" + +#include + +#include "util/hash_util.hpp" + +namespace doris::format::parquet::native { + +namespace { +Status set_hash_strategy(segment_v2::HashStrategyPB strategy, + std::function* hash_func) { + if (strategy != segment_v2::HashStrategyPB::XX_HASH_64) { + return Status::InvalidArgument("Invalid Parquet Bloom filter hash strategy {}", strategy); + } + *hash_func = [](const void* buf, int64_t len, uint64_t, void* out) { + *reinterpret_cast(out) = + HashUtil::xxhash64_compat_with_seed(reinterpret_cast(buf), len, 0); + }; + return Status::OK(); +} +} // namespace + +Status BlockSplitBloomFilter::init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) { + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _num_bytes = filter_size; + _size = _num_bytes; + _data = new char[_size]; + memset(_data, 0, _size); + _has_null = nullptr; + _is_write = true; + segment_v2::g_write_bloom_filter_num << 1; + segment_v2::g_write_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +Status BlockSplitBloomFilter::init(const char* buf, size_t size, + segment_v2::HashStrategyPB strategy) { + if (buf == nullptr || size <= 1) { + return Status::InvalidArgument("Invalid Parquet Bloom filter buffer of size {}", size); + } + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _data = new char[size]; + memcpy(_data, buf, size); + _size = size; + _num_bytes = size; + _has_null = nullptr; + segment_v2::g_read_bloom_filter_num << 1; + segment_v2::g_read_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +void BlockSplitBloomFilter::add_bytes(const char* buf, size_t size) { + DCHECK(buf != nullptr); + add_hash(hash(buf, size)); +} + +bool BlockSplitBloomFilter::test_bytes(const char* buf, size_t size) const { + return test_hash(hash(buf, size)); +} + +void BlockSplitBloomFilter::set_has_null(bool has_null) { + DCHECK(!has_null) << "Parquet Bloom filters do not track nulls"; +} + +void BlockSplitBloomFilter::set_masks(uint32_t key, BlockMask* block_mask) { + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + block_mask->item[i] = uint32_t {1} << ((key * SALT[i]) >> 27); + } +} + +void BlockSplitBloomFilter::add_hash(uint64_t hash) { + DCHECK_GE(_num_bytes, BYTES_PER_BLOCK); + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + bitset[bucket_index * BITS_SET_PER_BLOCK + i] |= block_mask.item[i]; + } +} + +bool BlockSplitBloomFilter::test_hash(uint64_t hash) const { + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + const auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + if ((bitset[bucket_index * BITS_SET_PER_BLOCK + i] & block_mask.item[i]) == 0) { + return false; + } + } + return true; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h new file mode 100644 index 00000000000000..38dc97712ac46c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/bloom_filter/bloom_filter.h" + +namespace doris::format::parquet::native { + +// Keep the split-block implementation inside v2; the native scan path must not link through the +// legacy Parquet reader merely to evaluate footer Bloom filters. +class BlockSplitBloomFilter final : public segment_v2::BloomFilter { +public: + Status init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) override; + Status init(const char* buf, size_t size, segment_v2::HashStrategyPB strategy) override; + void add_bytes(const char* buf, size_t size) override; + bool test_bytes(const char* buf, size_t size) const override; + void set_has_null(bool has_null) override; + bool has_null() const override { return false; } + void add_hash(uint64_t hash) override; + bool test_hash(uint64_t hash) const override; + +private: + static constexpr int BYTES_PER_BLOCK = 32; + static constexpr int BITS_SET_PER_BLOCK = 8; + static constexpr uint32_t SALT[BITS_SET_PER_BLOCK] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + + struct BlockMask { + uint32_t item[BITS_SET_PER_BLOCK]; + }; + + static void set_masks(uint32_t key, BlockMask* block_mask); +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp new file mode 100644 index 00000000000000..06e4b239f3abb4 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" + +#include + +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/bit_util.h" + +namespace doris::format::parquet::native { +Status BoolPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + std::array values; + size_t decoded = 0; + while (decoded < num_values) { + const size_t batch_size = std::min(values.size(), num_values - decoded); + for (size_t row = 0; row < batch_size; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain decoder"); + } + values[row] = static_cast(value); + } + RETURN_IF_ERROR(consumer.consume(values.data(), batch_size, sizeof(uint8_t))); + decoded += batch_size; + } + return Status::OK(); +} + +Status BoolPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + selected_values_.resize(selection.selected_values); + size_t range_index = 0; + size_t output = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + selected_values_[output++] = static_cast(value); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(selected_values_.data(), output, sizeof(uint8_t)); +} + +Status BoolPlainDecoder::skip_values(size_t num_values) { + int skip_cached = + std::min(num_unpacked_values_ - unpacked_value_idx_, cast_set(num_values)); + unpacked_value_idx_ += skip_cached; + if (skip_cached == num_values) { + return Status::OK(); + } + int num_remaining = cast_set(num_values - skip_cached); + int num_to_skip = BitUtil::RoundDownToPowerOf2(num_remaining, 32); + if (num_to_skip > 0) { + // A failed bulk skip must not be reported as success for a truncated boolean page. + if (!bool_values_.SkipBatch(1, num_to_skip)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + } + num_remaining -= num_to_skip; + if (num_remaining > 0) { + DCHECK_LE(num_remaining, UNPACKED_BUFFER_LEN); + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ < num_remaining)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + unpacked_value_idx_ = num_remaining; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h new file mode 100644 index 00000000000000..ee49a236b847ce --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris { +class ColumnSelectVector; +} // namespace doris + +namespace doris::format::parquet::native { +/// Decoder bit-packed boolean-encoded values. +/// Implementation from https://github.com/apache/impala/blob/master/be/src/exec/parquet/parquet-bool-decoder.h +//bit-packed-run-len and rle-run-len must be in the range [1, 2^31 - 1]. +// This means that a Parquet implementation can always store the run length in a signed 32-bit integer +class BoolPlainDecoder final : public Decoder { +public: + BoolPlainDecoder() = default; + ~BoolPlainDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + bool_values_.Reset((const uint8_t*)data->data, data->size); + num_unpacked_values_ = 0; + unpacked_value_idx_ = 0; + _offset = 0; + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&selected_values_, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return selected_values_.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return selected_values_.size() * sizeof(uint8_t); + } + +protected: + inline bool _decode_value(bool* value) { + if (LIKELY(unpacked_value_idx_ < num_unpacked_values_)) { + *value = unpacked_values_[unpacked_value_idx_++]; + } else { + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ == 0)) { + return false; + } + *value = unpacked_values_[0]; + unpacked_value_idx_ = 1; + } + return true; + } + + /// A buffer to store unpacked values. Must be a multiple of 32 size to use the + /// batch-oriented interface of BatchedBitReader. We use uint8_t instead of bool because + /// bit unpacking is only supported for unsigned integers. The values are converted to + /// bool when returned to the user. + static const int UNPACKED_BUFFER_LEN = 128; + uint8_t unpacked_values_[UNPACKED_BUFFER_LEN]; + + /// The number of valid values in 'unpacked_values_'. + int num_unpacked_values_ = 0; + + /// The next value to return from 'unpacked_values_'. + int unpacked_value_idx_ = 0; + + /// Bit packed decoder, used if 'encoding_' is PLAIN. + BatchedBitReader bool_values_; + + std::vector selected_values_; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp new file mode 100644 index 00000000000000..d541550cef180f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" + +#include + +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +Status BoolRLEDecoder::set_data(Slice* slice) { + _data = slice; + _num_bytes = slice->size; + _offset = 0; + if (_num_bytes < 4) { + return Status::IOError("Received invalid length : " + std::to_string(_num_bytes) + + " (corrupt data page?)"); + } + // Load the first 4 bytes in little-endian, which indicates the length + const auto* data = reinterpret_cast(_data->data); + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > static_cast(_num_bytes - 4)) { + return Status::IOError("Received invalid number of bytes : " + std::to_string(num_bytes) + + " (corrupt data page?)"); + } + _num_bytes = num_bytes; + auto decoder_data = data + 4; + _decoder = RleBatchDecoder(const_cast(decoder_data), num_bytes, 1); + return Status::OK(); +} + +Status BoolRLEDecoder::skip_values(size_t num_values) { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. + if (_decoder.GetBatch(_values.data(), static_cast(batch_size)) != batch_size) { + return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + } + skipped += batch_size; + } + return Status::OK(); +} + +Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { + _values.resize(num_values); + if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE decoder"); + } + return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); +} + +Status BoolRLEDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + _values.resize(selection.total_values); + if (_decoder.GetBatch(_values.data(), cast_set(selection.total_values)) != + selection.total_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE selection decoder"); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, + range.count * sizeof(uint8_t)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_values.data(), output, sizeof(uint8_t)); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h new file mode 100644 index 00000000000000..55d7a331bb27ca --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/rle_encoding.h" + +namespace doris { +class ColumnSelectVector; +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +class BoolRLEDecoder final : public Decoder { +public: + BoolRLEDecoder() = default; + ~BoolRLEDecoder() override = default; + + Status set_data(Slice* slice) override; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { return _values.size() * sizeof(uint8_t); } + +private: + RleBatchDecoder _decoder; + std::vector _values; + size_t _num_bytes; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp new file mode 100644 index 00000000000000..cc3dc66b3cacdc --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" + +#include "core/custom_allocator.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { +Status ByteArrayDictDecoder::set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + _dict_items.clear(); + _dict = std::move(dict); + if (_dict == nullptr) { + return Status::Corruption("Wrong dictionary data for byte array type, dict is null."); + } + if (UNLIKELY(length < 0)) { + return Status::Corruption("Wrong data length in dictionary"); + } + const size_t dict_length = cast_set(length); + // Every BYTE_ARRAY entry needs a four-byte length prefix; bound metadata-driven reserve first. + if (UNLIKELY(num_values > dict_length / sizeof(uint32_t))) { + return Status::Corruption("Dictionary value count exceeds byte array payload"); + } + _dict_items.reserve(num_values); + size_t offset_cursor = 0; + char* dict_item_address = reinterpret_cast(_dict.get()); + for (size_t i = 0; i < num_values; ++i) { + if (UNLIKELY(offset_cursor > dict_length || + dict_length - offset_cursor < sizeof(uint32_t))) { + return Status::Corruption("Wrong data length in dictionary"); + } + uint32_t l = decode_fixed32_le(_dict.get() + offset_cursor); + offset_cursor += sizeof(uint32_t); + if (UNLIKELY(l > dict_length - offset_cursor)) { + return Status::Corruption("Wrong data length in dictionary"); + } + _dict_items.emplace_back(dict_item_address + offset_cursor, l); + offset_cursor += l; + } + if (offset_cursor != dict_length) { + return Status::Corruption("Wrong dictionary data for byte array type"); + } + ++_dictionary_generation; + return Status::OK(); +} + +Status ByteArrayDictDecoder::decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return binary_consumer.consume(_dict_items.data(), _dict_items.size()); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h new file mode 100644 index 00000000000000..790d9f2e4b65ba --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/string_ref.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteArrayDictDecoder final : public BaseDictDecoder { +public: + ByteArrayDictDecoder() = default; + ~ByteArrayDictDecoder() override = default; + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override; + + size_t dictionary_size() const override { return _dict_items.size(); } + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override; + +protected: + // StringRef entries point into BaseDictDecoder::_dict. The dictionary buffer lives for the + // entire column chunk, so retaining a second copy of every byte is unnecessary. + DorisVector _dict_items; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp new file mode 100644 index 00000000000000..064bcffec8dd1f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" + +#include +#include +#include + +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" + +namespace doris::format::parquet::native { +namespace { +Status read_length(const Slice* data, size_t* offset, uint32_t* length) { + if (UNLIKELY(*offset > data->size || data->size - *offset < sizeof(uint32_t))) { + return Status::IOError("Can't read byte array length from plain decoder"); + } + *length = decode_fixed32_le(reinterpret_cast(data->data) + *offset); + *offset += sizeof(uint32_t); + return Status::OK(); +} +} // namespace + +Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(num_values); + _value_offsets.clear(); + _value_offsets.reserve(num_values + 1); + _value_offsets.push_back(0); + for (size_t row = 0; row < num_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain decoder"); + } + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY batch exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + _offset += length; + } + _value_spans.clear(); + if (num_values != 0) { + _value_spans.push_back({.first = 0, .count = num_values}); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), num_values, _value_spans); +} + +Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(selection.selected_values); + _value_offsets.clear(); + _value_offsets.reserve(selection.selected_values + 1); + _value_offsets.push_back(0); + _value_spans.clear(); + _value_spans.reserve(selection.ranges.size()); + size_t range_index = 0; + size_t selected_in_range = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + if (selected_in_range != 0) { + _value_spans.push_back({.first = _payload_offsets.size() - selected_in_range, + .count = selected_in_range}); + selected_in_range = 0; + } + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY selection exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + ++selected_in_range; + } + _offset += length; + } + if (selected_in_range != 0) { + _value_spans.push_back( + {.first = _payload_offsets.size() - selected_in_range, .count = selected_in_range}); + } + DORIS_CHECK_EQ(_payload_offsets.size(), selection.selected_values); + if (_payload_offsets.empty()) { + return Status::OK(); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), _payload_offsets.size(), + _value_spans); +} + +Status ByteArrayPlainDecoder::skip_values(size_t num_values) { + for (int i = 0; i < num_values; ++i) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't skip enough bytes in plain decoder"); + } + _offset += length; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h new file mode 100644 index 00000000000000..45bc82fe823f1a --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_util.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +template +class ColumnDecimal; +} // namespace doris + +namespace doris::format::parquet::native { +class ByteArrayPlainDecoder final : public Decoder { +public: + ByteArrayPlainDecoder() = default; + ~ByteArrayPlainDecoder() override = default; + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override; + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_payload_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_spans, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return (_payload_offsets.capacity() + _value_offsets.capacity()) * sizeof(uint32_t) + + _value_spans.capacity() * sizeof(ParquetSelectionRange); + } + size_t active_scratch_bytes() const override { + return (_payload_offsets.size() + _value_offsets.size()) * sizeof(uint32_t) + + _value_spans.size() * sizeof(ParquetSelectionRange); + } + +private: + std::vector _payload_offsets; + std::vector _value_offsets; + std::vector _value_spans; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp new file mode 100644 index 00000000000000..caf3b675e2dfd7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" + +#include +#include + +#include "core/column/column_fixed_length_object.h" +#include "util/byte_stream_split.h" + +namespace doris::format::parquet::native { +Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet byte-stream-split decoder"); + } + const int64_t stride = static_cast(_data->size / _type_length); + _decoded_values.resize(byte_size); + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + _offset / _type_length, num_values, stride, _decoded_values.data()); + _offset += byte_size; + return consumer.consume(_decoded_values.data(), num_values, static_cast(_type_length)); +} + +Status ByteStreamSplitDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet byte-stream-split selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError( + "Out-of-bounds access in Parquet byte-stream-split selection decoder"); + } + DORIS_CHECK_EQ(_data->size % value_width, 0); + const int64_t stride = static_cast(_data->size / value_width); + _decoded_values.resize(selection.selected_values * value_width); + size_t output = 0; + const size_t first_row = _offset / value_width; + for (const auto& range : selection.ranges) { + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + first_row + range.first, range.count, stride, + _decoded_values.data() + output * value_width); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + _offset += input_bytes; + return consumer.consume(_decoded_values.data(), output, value_width); +} + +Status ByteStreamSplitDecoder::skip_values(size_t num_values) { + // Check in row units before multiplication so a corrupt skip count cannot wrap back in-bounds. + if (UNLIKELY(_type_length <= 0 || _offset > _data->size || + num_values > (_data->size - _offset) / _type_length)) { + return Status::IOError( + "Out-of-bounds access in parquet data decoder: offset = {}, size = {}", _offset, + _data->size); + } + _offset += _type_length * num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h new file mode 100644 index 00000000000000..dfe1d954b61641 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteStreamSplitDecoder final : public Decoder { +public: + ByteStreamSplitDecoder() = default; + ~ByteStreamSplitDecoder() override = default; + + Status set_data(Slice* data) override { + if (UNLIKELY(_type_length <= 0 || data == nullptr || + data->size % static_cast(_type_length) != 0)) { + // Every byte lane must contain the same number of complete physical values. + return Status::Corruption("Invalid Parquet byte-stream-split page length"); + } + return Decoder::set_data(data); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_decoded_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _decoded_values.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _decoded_values.size() * sizeof(uint8_t); + } + +private: + std::vector _decoded_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp new file mode 100644 index 00000000000000..26afaf15bf4c4c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -0,0 +1,1951 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_chunk_reader.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "core/column/column.h" +#include "core/column/column_decimal.h" +#include "core/column/column_dictionary.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/bit_util.h" +#include "util/block_compression.h" +#include "util/cpu_info.h" +#include "util/unaligned.h" + +namespace cctz { +class time_zone; +} // namespace cctz +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { + +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available) { + return session_cache_enabled && !storage_cache_disabled && cache_available && header_available; +} + +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed) { + const bool is_v2 = header.__isset.data_page_header_v2; + const bool page_is_compressed = + codec != tparquet::CompressionCodec::UNCOMPRESSED && + (!is_v2 || header.data_page_header_v2.is_compressed || data_page_v2_always_compressed); + if (!page_is_compressed && + UNLIKELY(header.compressed_page_size != header.uncompressed_page_size)) { + // An uncompressed payload has one physical representation, so accepting two lengths makes + // cold reads and decompressed cache hits consume different byte boundaries. + return Status::Corruption( + "Uncompressed Parquet page sizes differ: compressed={}, uncompressed={}", + header.compressed_page_size, header.uncompressed_page_size); + } + return Status::OK(); +} + +Status validate_fixed_width_page_size(const tparquet::PageHeader& header, int32_t type_length, + level_t max_rep_level, level_t max_def_level, + bool schema_is_required) { + if (type_length <= 0) { + return Status::OK(); + } + const bool is_v2 = header.__isset.data_page_header_v2; + if (!is_v2 && !header.__isset.data_page_header) { + return Status::OK(); + } + const auto encoding = + is_v2 ? header.data_page_header_v2.encoding : header.data_page_header.encoding; + if (encoding != tparquet::Encoding::PLAIN && + encoding != tparquet::Encoding::BYTE_STREAM_SPLIT) { + return Status::OK(); + } + int32_t num_physical_values = 0; + int64_t level_bytes = 0; + if (is_v2) { + const auto& page = header.data_page_header_v2; + if (UNLIKELY(page.num_values < 0 || page.num_nulls < 0 || + page.num_nulls > page.num_values || page.repetition_levels_byte_length < 0 || + page.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has invalid value or level counts"); + } + num_physical_values = page.num_values - page.num_nulls; + level_bytes = static_cast(page.repetition_levels_byte_length) + + page.definition_levels_byte_length; + } else { + if (max_rep_level != 0 || max_def_level != 0 || !schema_is_required) { + return Status::OK(); + } + num_physical_values = header.data_page_header.num_values; + } + if (level_bytes > std::numeric_limits::max() || num_physical_values < 0 || + static_cast(num_physical_values) > + (static_cast(std::numeric_limits::max()) - level_bytes) / + static_cast(type_length)) { + return Status::Corruption("Parquet fixed-width PLAIN page byte size overflows"); + } + const int64_t expected = level_bytes + static_cast(num_physical_values) * type_length; + if (UNLIKELY(header.uncompressed_page_size != expected)) { + // V2 exposes null and level extents separately, so fixed-width payload size is known before + // decompression even for optional columns and must gate attacker-controlled allocation. + return Status::Corruption("Parquet fixed-width page has {} uncompressed bytes, expected {}", + header.uncompressed_page_size, expected); + } + return Status::OK(); +} + +Status validate_dictionary_page_size(const tparquet::PageHeader& header, int32_t type_length) { + DORIS_CHECK(header.__isset.dictionary_page_header); + const int32_t num_values = header.dictionary_page_header.num_values; + if (UNLIKELY(num_values < 0 || (num_values == 0 && header.uncompressed_page_size != 0))) { + // An empty dictionary owns no payload; validate before allocating from its untrusted size. + return Status::Corruption("Parquet dictionary has {} values and {} uncompressed bytes", + num_values, header.uncompressed_page_size); + } + if (type_length > 0) { + if (UNLIKELY(static_cast(num_values) > + static_cast(std::numeric_limits::max()) / + static_cast(type_length))) { + return Status::Corruption("Parquet fixed-width dictionary byte size overflows"); + } + const int64_t expected = static_cast(num_values) * type_length; + if (UNLIKELY(header.uncompressed_page_size != expected)) { + // Fixed-width dictionaries have no level section, so reject forged extents before the + // decoder allocates storage based on the untrusted page header. + return Status::Corruption( + "Parquet fixed-width dictionary has {} uncompressed bytes, expected {}", + header.uncompressed_page_size, expected); + } + } + return Status::OK(); +} + +Status validate_compressed_page_size(tparquet::CompressionCodec::type codec, + const Slice& compressed_data, + size_t expected_uncompressed_size) { + if (codec != tparquet::CompressionCodec::SNAPPY) { + return Status::OK(); + } + size_t actual_uncompressed_size = 0; + if (UNLIKELY(!snappy::GetUncompressedLength(compressed_data.data, compressed_data.size, + &actual_uncompressed_size))) { + return Status::Corruption("Invalid Snappy-compressed Parquet page"); + } + if (UNLIKELY(actual_uncompressed_size != expected_uncompressed_size)) { + // Snappy exposes its decoded extent without an output buffer. Check it before trusting the + // page header so malformed variable-width pages cannot force a header-sized allocation. + return Status::Corruption("Snappy Parquet page expands to {} bytes, expected {}", + actual_uncompressed_size, expected_uncompressed_size); + } + return Status::OK(); +} + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { + if (created_by.empty()) { + return {}; + } + const ::parquet::ApplicationVersion version(created_by); + return {.parquet_816_padding = + version.VersionLt(::parquet::ApplicationVersion::PARQUET_816_FIXED_VERSION()), + .data_page_v2_always_compressed = version.VersionLt( + ::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION())}; +} + +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range) { + DORIS_CHECK(range != nullptr); + int64_t start = metadata.data_page_offset; + if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset > 0 && + metadata.dictionary_page_offset < start) { + // Some writers use dictionary_page_offset=0 as an absence sentinel. Range validation must + // follow has_dict_page() or the native reader starts at the Parquet magic bytes. + start = metadata.dictionary_page_offset; + } + const int64_t length = metadata.total_compressed_size; + if (UNLIKELY(start < 0 || length < 0)) { + return Status::Corruption("Parquet column chunk has a negative offset or length"); + } + const uint64_t unsigned_start = static_cast(start); + const uint64_t unsigned_length = static_cast(length); + if (UNLIKELY(unsigned_start > file_size || unsigned_length > file_size - unsigned_start)) { + // Thrift range fields are signed and untrusted; validate before converting them to the + // unsigned stream-reader coordinates so overflow cannot wrap back into the file. + return Status::Corruption("Parquet column chunk [{}, {}) exceeds file size {}", start, + unsigned_start + unsigned_length, file_size); + } + size_t bounded_length = static_cast(unsigned_length); + if (parquet_816_padding) { + // parquet-mr before PARQUET-816 under-reported the chunk by up to 100 bytes. Padding stays + // file-bounded and is only enabled for the affected writer versions. + bounded_length += std::min(100, file_size - unsigned_start - unsigned_length); + } + range->offset = static_cast(unsigned_start); + range->length = bounded_length; + return Status::OK(); +} + +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count) { + if (index.page_locations.empty() || data_page_offset < 0 || row_count < 0 || + index.page_locations.front().first_row_index != 0 || + index.page_locations.front().offset != data_page_offset || + chunk_range.length > std::numeric_limits::max() - chunk_range.offset) { + return false; + } + // Row indexes alone cannot detect a uniformly shifted OffsetIndex. Anchor its first location + // to the owning metadata so page-to-row mapping cannot silently move by one physical page. + const uint64_t chunk_begin = chunk_range.offset; + const uint64_t chunk_end = chunk_begin + chunk_range.length; + uint64_t previous_end = chunk_begin; + int64_t previous_row = -1; + for (const auto& location : index.page_locations) { + if (location.first_row_index <= previous_row || location.first_row_index >= row_count || + location.offset < 0 || location.compressed_page_size <= 0) { + return false; + } + const uint64_t begin = static_cast(location.offset); + const uint64_t size = static_cast(location.compressed_page_size); + if (begin < chunk_begin || begin < previous_end || begin > chunk_end || + size > chunk_end - begin) { + return false; + } + previous_row = location.first_row_index; + previous_end = begin + size; + } + return true; +} + +namespace { + +class EmptyValueSectionDecoder final : public Decoder { +public: + Status skip_values(size_t num_values) override { + if (UNLIKELY(num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + return Status::OK(); + } +}; + +Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, + const ParquetInt96Timestamp& value, + const cctz::time_zone& timezone) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_PER_DAY = 86400000000LL; + static constexpr int64_t MICROS_PER_SECOND = 1000000LL; + + // Arrow normalized out-of-day INT96 nanos before the native V2 path replaced it. Preserve that + // legacy-writer compatibility here; rejecting the carrier loses valid pre-epoch/year-0 values + // already accepted by Doris external-table scans. + const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + // Truncate the signed nanos field before day normalization. Flooring a negative value first + // changes the historical result by one microsecond. + const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; + if (timestamp_micros < std::numeric_limits::min() || + timestamp_micros > std::numeric_limits::max()) { + return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); + } + + const int64_t micros = static_cast(timestamp_micros); + int64_t epoch_seconds = micros / MICROS_PER_SECOND; + int64_t micros_of_second = micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + DateV2Value datetime; + datetime.from_unixtime(epoch_seconds, timezone); + datetime.set_microsecond(static_cast(micros_of_second)); + if (!datetime.is_valid_date()) { + return Status::DataQualityError("Parquet INT96 timestamp is outside the Doris range"); + } + data.push_back(datetime); + return Status::OK(); +} + +class V2Int96DateTimeConsumer final : public ParquetFixedValueConsumer { +public: + V2Int96DateTimeConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state) + : _data(assert_cast(column).get_data()), _state(state) { + static const auto utc = cctz::utc_time_zone(); + _timezone = context.timezone == nullptr ? &utc : context.timezone; + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load( + values + row * sizeof(ParquetInt96Timestamp)); + const auto status = append_v2_int96_datetime(_data, value, *_timezone); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + ParquetMaterializationState* _state; + const cctz::time_zone* _timezone = nullptr; +}; + +class RejectV2Int96BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + return Status::NotSupported("INT96 cannot be decoded from binary Parquet values"); + } +}; + +Status read_v2_int96_datetime(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) { + V2Int96DateTimeConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + V2Int96DateTimeConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectV2Int96BinaryConsumer binary_consumer; + const auto dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + return state.materialize_dictionary(column); +} + +Status read_native_or_serde(IColumn& column, const DataTypeSerDe& serde, + ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + auto* output = check_and_get_column(&column); + if (output == nullptr) { + return Status::InternalError("Dictionary indices require an INT32 output column"); + } + // Dictionary IDs have the same RLE/bit-packed representation for every physical type. + // Decode them before dispatching to a typed SerDe; otherwise a fixed-width SerDe treats + // the IDs as values and the row filter indexes its bitmap with materialized data. + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = output->size(); + auto& indices = output->get_data(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + if (context.physical_type == ParquetPhysicalType::INT96 && + check_and_get_column(&column) != nullptr) { + return read_v2_int96_datetime(column, source, context, num_values, state); + } + return serde.read_column_from_parquet(column, source, context, num_values, state); +} + +Status translate_value_encoding(tparquet::Encoding::type encoding, + ParquetValueEncoding* translated) { + DORIS_CHECK(translated != nullptr); + switch (encoding) { + case tparquet::Encoding::PLAIN: + *translated = ParquetValueEncoding::PLAIN; + return Status::OK(); + case tparquet::Encoding::RLE_DICTIONARY: + case tparquet::Encoding::PLAIN_DICTIONARY: + *translated = ParquetValueEncoding::DICTIONARY; + return Status::OK(); + case tparquet::Encoding::RLE: + *translated = ParquetValueEncoding::RLE; + return Status::OK(); + case tparquet::Encoding::BIT_PACKED: + *translated = ParquetValueEncoding::BIT_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + *translated = ParquetValueEncoding::DELTA_BINARY_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_LENGTH_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + *translated = ParquetValueEncoding::BYTE_STREAM_SPLIT; + return Status::OK(); + default: + return Status::NotSupported("Unsupported Parquet encoding {}", + tparquet::to_string(encoding)); + } +} + +template +Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decoder& decoder, + const ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector, + int64_t* materialization_time) { + SCOPED_RAW_TIMER(materialization_time); + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + RETURN_IF_ERROR( + read_native_or_serde(column, serde, decoder, context, run_length, state)); + break; + case ColumnSelectVector::NULL_DATA: + column.insert_many_defaults(run_length); + break; + case ColumnSelectVector::FILTERED_CONTENT: + RETURN_IF_ERROR(decoder.skip_values(run_length)); + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + return Status::OK(); +} + +// Presents one sparse page request as an ordinary sequential source to DataTypeSerDe. SerDe is +// entered once per page fragment; the concrete decoder decides whether to gather selected spans, +// batch-decode and compact, or use the cursor-preserving range fallback. +class SelectedDecodeSource final : public ParquetDecodeSource { +public: + SelectedDecodeSource(Decoder& decoder, const ParquetSelection& selection) + : _decoder(decoder), _selection(selection) {} + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_fixed_values(_selection, consumer); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_binary_values(_selection, consumer); + } + + Status skip_values(size_t num_values) override { + return Status::NotSupported("Selected Parquet source cannot be skipped, values={}", + num_values); + } + + bool has_dictionary() const override { return _decoder.has_dictionary(); } + uint64_t dictionary_generation() const override { return _decoder.dictionary_generation(); } + size_t dictionary_size() const override { return _decoder.dictionary_size(); } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return _decoder.decode_dictionary(fixed_consumer, binary_consumer); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_indices(_selection, indices); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_values(_selection, consumer); + } + + bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const override { + // Avoid touching a dictionary larger than L2 for rows already removed by sparse selection. + // Cache-resident or dense batches instead fuse RLE decode and gather, eliminating the + // page-sized intermediate ID vector. + return _selection.selected_values < _selection.total_values && + dictionary_bytes > static_cast(CpuInfo::get_l2_cache_size()); + } + +private: + Decoder& _decoder; + const ParquetSelection& _selection; +}; + +Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = select_vector.num_values(); + selection.selected_values = 0; + + size_t cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + DORIS_CHECK(read_type == ColumnSelectVector::CONTENT || + read_type == ColumnSelectVector::FILTERED_CONTENT); + if (read_type == ColumnSelectVector::CONTENT) { + selection.ranges.push_back({.first = cursor, .count = run_length}); + selection.selected_values += run_length; + } + cursor += run_length; + } + DORIS_CHECK_EQ(cursor, selection.total_values); + if (selection.selected_values == 0) { + return decoder.skip_values(selection.total_values); + } + + SCOPED_RAW_TIMER(materialization_time); + SelectedDecodeSource selected_source(decoder, selection); + return read_native_or_serde(column, serde, selected_source, context, selection.selected_values, + state); +} + +template +bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { +#define VISIT_COLUMN(TYPE) \ + if (auto* typed = check_and_get_column(&column)) { \ + visitor(*typed); \ + return true; \ + } + VISIT_COLUMN(ColumnUInt8) + VISIT_COLUMN(ColumnInt8) + VISIT_COLUMN(ColumnInt16) + VISIT_COLUMN(ColumnInt32) + VISIT_COLUMN(ColumnInt64) + VISIT_COLUMN(ColumnInt128) + VISIT_COLUMN(ColumnDate) + VISIT_COLUMN(ColumnDateTime) + VISIT_COLUMN(ColumnDateV2) + VISIT_COLUMN(ColumnDateTimeV2) + VISIT_COLUMN(ColumnFloat32) + VISIT_COLUMN(ColumnFloat64) + VISIT_COLUMN(ColumnIPv4) + VISIT_COLUMN(ColumnIPv6) + VISIT_COLUMN(ColumnTimeV2) + VISIT_COLUMN(ColumnTimeStampTz) + VISIT_COLUMN(ColumnOffset32) + VISIT_COLUMN(ColumnOffset64) + VISIT_COLUMN(ColumnDecimal32) + VISIT_COLUMN(ColumnDecimal64) + VISIT_COLUMN(ColumnDecimal128V2) + VISIT_COLUMN(ColumnDecimal128V3) + VISIT_COLUMN(ColumnDecimal256) + VISIT_COLUMN(ColumnString) + VISIT_COLUMN(ColumnString64) + VISIT_COLUMN(ColumnVarbinary) + VISIT_COLUMN(ColumnDictI32) +#undef VISIT_COLUMN + return false; +} + +template +void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + auto& data = column.get_data(); + DORIS_CHECK_EQ(data.size(), old_size + compact_values); + data.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + data[old_size + output] = typename ColumnType::value_type {}; + } else { + DORIS_CHECK(source > 0); + --source; + data[old_size + output] = std::move(data[old_size + source]); + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_string_values(ColumnStr& column, size_t old_size, + size_t compact_values, const NullMap& selected_nulls) { + auto& offsets = column.get_offsets(); + DORIS_CHECK_EQ(offsets.size(), old_size + compact_values); + const Offset prefix_end = old_size == 0 ? 0 : offsets[old_size - 1]; + offsets.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] == 0) { + DORIS_CHECK(source > 0); + --source; + offsets[old_size + output] = offsets[old_size + source]; + } else { + offsets[old_size + output] = source == 0 ? prefix_end : offsets[old_size + source - 1]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_pod_values(column, old_size, compact_values, selected_nulls); +} + +template +void expand_nullable_values(ColumnStr& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_string_values(column, old_size, compact_values, selected_nulls); +} + +void remap_nullable_conversion_failures(IColumn::Filter* conversion_failure_null_map, + size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + if (conversion_failure_null_map == nullptr) { + return; + } + DORIS_CHECK(conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + size_t source = compact_values; + // Walk backwards so writing an expanded row cannot overwrite an unread compact failure bit. + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + (*conversion_failure_null_map)[old_size + output] = 1; + } else { + DORIS_CHECK(source > 0); + --source; + (*conversion_failure_null_map)[old_size + output] = + (*conversion_failure_null_map)[old_size + source]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, NullMap& selected_nulls, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = 0; + selection.selected_values = 0; + selected_nulls.clear(); + selected_nulls.reserve(select_vector.num_values() - select_vector.num_filtered()); + + size_t physical_cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls.resize_fill(selected_nulls.size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls.resize_fill(selected_nulls.size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls.size(), + select_vector.num_values() - select_vector.num_filtered()); + + const size_t old_size = column.size(); + SCOPED_RAW_TIMER(materialization_time); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(decoder.skip_values(selection.total_values)); + column.insert_many_defaults(selected_nulls.size()); + return Status::OK(); + } + + if (state.conversion_failure_null_map != nullptr) { + DORIS_CHECK(state.conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + memset(state.conversion_failure_null_map->data() + old_size, 0, selection.selected_values); + } + SelectedDecodeSource selected_source(decoder, selection); + const auto status = read_native_or_serde(column, serde, selected_source, context, + selection.selected_values, state); + if (!status.ok()) { + if (state.conversion_failure_null_map != nullptr) { + memcpy(state.conversion_failure_null_map->data() + old_size, selected_nulls.data(), + selected_nulls.size()); + } + return status; + } + DORIS_CHECK_EQ(column.size(), old_size + selection.selected_values); + + remap_nullable_conversion_failures(state.conversion_failure_null_map, old_size, + selection.selected_values, selected_nulls); + const bool expanded = visit_nullable_expandable_column(column, [&](auto& typed_column) { + // Decode into the final nested column compactly, then expand in place. This preserves the + // V2 no-intermediate-column invariant while restoring the nullable sparse row layout. + expand_nullable_values(typed_column, old_size, selection.selected_values, selected_nulls); + }); + DORIS_CHECK(expanded); + return Status::OK(); +} + +class FixedWidthPredicateConsumer final : public ParquetFixedValueConsumer { +public: + FixedWidthPredicateConsumer(const VExprSPtrs& conjuncts, DataTypePtr data_type, int column_id, + IColumn::Filter* matches, IColumn* projected_column) + : _conjuncts(conjuncts), + _data_type(std::move(data_type)), + _column_id(column_id), + _matches(matches), + _projected_column(projected_column) { + DORIS_CHECK(_matches != nullptr); + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _matches->size(); + _matches->resize_fill(old_size + num_values, 1); + for (const auto& conjunct : _conjuncts) { + RETURN_IF_ERROR(conjunct->execute_on_raw_fixed_values(values, num_values, value_width, + _data_type, _column_id, + _matches->data() + old_size)); + } + if (_projected_column != nullptr) { + size_t row = 0; + while (row < num_values) { + while (row < num_values && (*_matches)[old_size + row] == 0) { + ++row; + } + const size_t run_begin = row; + while (row < num_values && (*_matches)[old_size + row] != 0) { + ++row; + } + if (row != run_begin) { + _projected_column->insert_many_raw_data( + reinterpret_cast(values + run_begin * value_width), + row - run_begin); + } + } + } + return Status::OK(); + } + +private: + const VExprSPtrs& _conjuncts; + DataTypePtr _data_type; + int _column_id; + IColumn::Filter* _matches; + IColumn* _projected_column; +}; + +} // namespace + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range) + : _field_schema(field_schema), + _max_rep_level(field_schema->repetition_level), + _max_def_level(field_schema->definition_level), + _stream_reader(reader), + _metadata(column_chunk->meta_data), + _offset_index(offset_index), + _total_rows(total_rows), + _io_ctx(io_ctx), + _page_read_ctx(page_read_ctx) { + if (chunk_range != nullptr) { + _chunk_range = *chunk_range; + _has_validated_chunk_range = true; + } +} + +template +Status ColumnChunkReader::init() { + size_t start_offset = _has_validated_chunk_range + ? _chunk_range.offset + : (has_dict_page(_metadata) ? _metadata.dictionary_page_offset + : _metadata.data_page_offset); + size_t chunk_size = + _has_validated_chunk_range ? _chunk_range.length : _metadata.total_compressed_size; + // create page reader + _page_reader = create_page_reader( + _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata, + _page_read_ctx, _offset_index); + // get the block compression codec + RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); + _state = INITIALIZED; + RETURN_IF_ERROR(_parse_first_page_header()); + return Status::OK(); +} + +template +Status ColumnChunkReader::skip_nested_values( + const std::vector& def_levels, size_t start_index) { + size_t no_value_cnt = 0; + size_t value_cnt = 0; + + DORIS_CHECK(start_index <= def_levels.size()); + for (size_t idx = start_index; idx < def_levels.size(); idx++) { + level_t def_level = def_levels[idx]; + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + no_value_cnt++; + } else if (def_level < _field_schema->definition_level) { + no_value_cnt++; + } else { + value_cnt++; + } + } + + RETURN_IF_ERROR(skip_values(value_cnt, true)); + RETURN_IF_ERROR(skip_values(no_value_cnt, false)); + return Status::OK(); +} + +template +Status ColumnChunkReader::read_levels( + size_t num_values, std::vector* rep_levels, std::vector* def_levels) { + DORIS_CHECK(rep_levels != nullptr); + DORIS_CHECK(def_levels != nullptr); + if (_remaining_num_values < num_values || _remaining_rep_nums < num_values || + _remaining_def_nums < num_values) { + return Status::Corruption( + "Parquet level reader requested {} slots with only {}/{}/{} remaining", num_values, + _remaining_num_values, _remaining_rep_nums, _remaining_def_nums); + } + + const size_t start_index = def_levels->size(); + rep_levels->resize(rep_levels->size() + num_values, 0); + def_levels->resize(def_levels->size() + num_values, 0); + if (_max_rep_level > 0) { + const size_t decoded = _rep_level_decoder.get_levels( + rep_levels->data() + rep_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet repetition level stream ended after {} of {} slots", + decoded, num_values); + } + } + if (_max_def_level > 0) { + const size_t decoded = _def_level_decoder.get_levels( + def_levels->data() + def_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet definition level stream ended after {} of {} slots", + decoded, num_values); + } + } + _remaining_rep_nums -= num_values; + _remaining_def_nums -= num_values; + return skip_nested_values(*def_levels, start_index); +} + +template +Status ColumnChunkReader::_parse_first_page_header() { + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + _state = INITIALIZED; + return parse_page_header(); + } + if (header->type != tparquet::PageType::DICTIONARY_PAGE) { + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + _state = INITIALIZED; + continue; + } + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + RETURN_IF_ERROR(_page_reader->dict_next_page()); + _state = INITIALIZED; + // A dictionary is the only non-data page with decoder state. Any following index or + // extension pages are skipped by the same pre-data loop. + } +} + +template +Status ColumnChunkReader::parse_page_header() { + if (_state == HEADER_PARSED || _state == DATA_LOADED) { + return Status::OK(); + } + const tparquet::PageHeader* header = nullptr; + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + break; + } + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + return Status::Corruption("Parquet dictionary page appears after data pages"); + } + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + } + int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values + : header->data_page_header.num_values; + if constexpr (IN_COLLECTION && OFFSET_INDEX) { + if (!_page_reader->is_header_v2() && _page_reader->has_active_offset_index()) { + // V1 nested pages do not declare their logical row count. An OffsetIndex span cannot + // be trusted until repetition levels are decoded, so keep the sequential cursor path. + _page_reader->discard_offset_index(); + _offset_index = nullptr; + } + } + const bool active_offset_index = _page_reader->has_active_offset_index(); + if (page_num_values < 0 || page_num_values > _metadata.num_values || + (!active_offset_index && + static_cast(page_num_values) > + static_cast(_metadata.num_values) - _chunk_parsed_values)) { + // Page counts are untrusted and feed both level decoders and scratch sizing. Bound each + // page by the column metadata before converting to unsigned counters. + return Status::Corruption("Parquet data page value count {} exceeds column total {}", + page_num_values, _metadata.num_values); + } + if constexpr (!IN_COLLECTION) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(page_num_values) != page_end_row - page_start_row)) { + // Flat columns have exactly one physical value slot per logical row. Rejecting a + // divergent header/OffsetIndex span prevents every later page from shifting rows. + return Status::Corruption( + "Parquet flat data page has {} values for logical row range [{}, {})", + page_num_values, page_start_row, page_end_row); + } + } else if (_page_reader->is_header_v2() && active_offset_index) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(header->data_page_header_v2.num_rows) != + page_end_row - page_start_row)) { + // V2 is the only repeated-page format that states its logical row count in the page + // header, so it must agree with the optional index before indexed seeking is allowed. + return Status::Corruption( + "Parquet nested data page has {} rows for indexed row range [{}, {})", + header->data_page_header_v2.num_rows, page_start_row, page_end_row); + } + } + _remaining_rep_nums = page_num_values; + _remaining_def_nums = page_num_values; + _remaining_num_values = page_num_values; + + // no offset will parse all header. + if (!active_offset_index) { + _chunk_parsed_values += _remaining_num_values; + } + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status ColumnChunkReader::next_page() { + // Level parsing advances _page_data past the allocation base, so retain explicit ownership + // state instead of inferring whether current decoders still reference decompressed storage. + _page_uses_decompress_buf = false; + _active_decompress_bytes = 0; + if (_decompress_release_pending) { + if (_decompress_buf_size > _decompress_release_threshold) { + _decompress_buf.reset(); + _decompress_buf_size = 0; + } + _decompress_release_pending = false; + _decompress_release_threshold = std::numeric_limits::max(); + } + _state = INITIALIZED; + RETURN_IF_ERROR(_page_reader->next_page()); + return Status::OK(); +} + +template +Status ColumnChunkReader::_get_uncompressed_levels( + const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { + const size_t rl = page_v2.repetition_levels_byte_length; + const size_t dl = page_v2.definition_levels_byte_length; + if (UNLIKELY(rl > page_data.size || dl > page_data.size - rl)) { + // Validate the physical slice again because a cached entry may itself be truncated. + return Status::Corruption("Parquet data page v2 level bytes exceed available payload"); + } + _v2_rep_levels = Slice(page_data.data, rl); + _v2_def_levels = Slice(page_data.data + rl, dl); + page_data.data += dl + rl; + page_data.size -= dl + rl; + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_data() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::Corruption("Should parse page header"); + } + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + // Zero levels alone are insufficient: test/protocol adapters can leave repetition unset, so + // only an explicitly REQUIRED schema proves that every logical value has fixed-width bytes. + const bool schema_is_required = _field_schema->parquet_schema.__isset.repetition_type && + _field_schema->parquet_schema.repetition_type == + tparquet::FieldRepetitionType::REQUIRED; + RETURN_IF_ERROR(validate_fixed_width_page_size(*header, _get_type_length(), _max_rep_level, + _max_def_level, schema_is_required)); + int32_t uncompressed_size = header->uncompressed_page_size; + bool page_loaded = false; + _page_uses_decompress_buf = false; + _active_decompress_bytes = 0; + + // First, try to reuse a cache handle previously discovered by PageReader + // (header-only lookup) to avoid a second lookup here. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + size_t levels_size = 0; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + levels_size = rl + dl; + if (UNLIKELY(header_size > cached.size || + levels_size > cached.size - header_size)) { + return Status::Corruption("Cached Parquet page is shorter than its v2 levels"); + } + _v2_rep_levels = + Slice(reinterpret_cast(cached.data) + header_size, rl); + _v2_def_levels = + Slice(reinterpret_cast(cached.data) + header_size + rl, dl); + } + // payload_slice points to the bytes after header and levels + if (UNLIKELY(header_size + levels_size > cached.size)) { + return Status::Corruption("Cached Parquet page is shorter than its header"); + } + Slice payload_slice(cached.data + header_size + levels_size, + cached.size - header_size - levels_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + const size_t expected_payload_size = + cache_payload_is_decompressed + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->compressed_page_size) - levels_size; + if (UNLIKELY(payload_slice.size != expected_payload_size)) { + return Status::Corruption("Cached Parquet page payload has size {}, expected {}", + payload_slice.size, expected_payload_size); + } + + if (cache_payload_is_decompressed) { + // Cached payload is already uncompressed + _page_data = payload_slice; + } else { + CHECK(_block_compress_codec); + // Decompress cached payload into _decompress_buf for decoding + size_t uncompressed_payload_size = + header->__isset.data_page_header_v2 + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->uncompressed_page_size); + RETURN_IF_ERROR(validate_compressed_page_size(_metadata.codec, payload_slice, + uncompressed_payload_size)); + _reserve_decompress_buf(uncompressed_payload_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size); + _page_uses_decompress_buf = true; + _active_decompress_bytes = uncompressed_payload_size; + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); + if (UNLIKELY(_page_data.size != uncompressed_payload_size)) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_payload_size); + } + } + // page cache counters were incremented when PageReader did the header-only + // cache lookup. Do not increment again to avoid double-counting. + page_loaded = true; + } + } + + if (!page_loaded) { + const bool prepare_cache_payload = can_prepare_page_cache_payload( + _page_read_ctx.enable_parquet_file_page_cache, config::disable_storage_page_cache, + StoragePageCache::instance() != nullptr, !_page_reader->header_bytes().empty()); + if (_block_compress_codec != nullptr) { + Slice compressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + std::vector level_bytes; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + // uncompressed_size = rl + dl + uncompressed_data_size + // compressed_size = rl + dl + compressed_data_size + uncompressed_size -= header_v2.repetition_levels_byte_length + + header_v2.definition_levels_byte_length; + // copy level bytes (rl + dl) so that we can cache header + levels + uncompressed payload + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), compressed_data.data, level_sz); + } + // now remove levels from compressed_data for decompression + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, compressed_data)); + } + bool is_v2_compressed = header->__isset.data_page_header_v2 && + (header->data_page_header_v2.is_compressed || + _page_read_ctx.data_page_v2_always_compressed); + bool page_has_compression = header->__isset.data_page_header || is_v2_compressed; + + if (page_has_compression) { + // Decompress payload for immediate decoding + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, compressed_data, static_cast(uncompressed_size))); + _reserve_decompress_buf(uncompressed_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_size); + _page_uses_decompress_buf = true; + _active_decompress_bytes = static_cast(uncompressed_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); + if (UNLIKELY(_page_data.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_size); + } + + // Decide whether to cache decompressed payload or compressed payload based on threshold + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (prepare_cache_payload) { + if (cache_payload_decompressed) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + // cache the compressed payload as-is (header | levels | compressed_payload) + _insert_page_into_cache( + level_bytes, Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + } else { + // no compression on this page, use the data directly + _page_data = Slice(compressed_data.data, compressed_data.size); + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } else { + // For uncompressed page, we may still need to extract v2 levels + std::vector level_bytes; + Slice uncompressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(uncompressed_data)); + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), uncompressed_data.data, level_sz); + } + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, uncompressed_data)); + } + // copy page data out + _page_data = Slice(uncompressed_data.data, uncompressed_data.size); + // Optionally cache uncompressed data for uncompressed pages + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Initialize repetition level and definition level. Skip when level = 0, which means required field. + if (_max_rep_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, + _remaining_rep_nums)); + } else { + RETURN_IF_ERROR(_rep_level_decoder.init( + &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, + _remaining_rep_nums)); + } + } + if (_max_def_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, + _remaining_def_nums)); + } else { + RETURN_IF_ERROR(_def_level_decoder.init( + &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, + _remaining_def_nums)); + } + } + auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding + : header->data_page_header.encoding; + // change the deprecated encoding to RLE_DICTIONARY + if (encoding == tparquet::Encoding::PLAIN_DICTIONARY) { + encoding = tparquet::Encoding::RLE_DICTIONARY; + } + _current_encoding = encoding; + + // Reuse page decoder + Decoder* encoding_decoder = nullptr; + if (_decoders.find(static_cast(encoding)) != _decoders.end()) { + encoding_decoder = _decoders[static_cast(encoding)].get(); + } else { + std::unique_ptr page_decoder; + RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + _decoders[static_cast(encoding)] = std::move(page_decoder); + encoding_decoder = _decoders[static_cast(encoding)].get(); + } + _empty_value_section = _page_data.empty() && _max_def_level > 0; + if (_empty_value_section) { + // Nullable all-NULL pages legally contain only definition levels. Keep them decodable for + // every advertised encoding, but make a non-NULL definition level fail before stale decoder + // state from the preceding page can be consumed. + if (_empty_value_decoder == nullptr) { + _empty_value_decoder = std::make_unique(); + } + _page_decoder = _empty_value_decoder.get(); + } else { + _page_decoder = encoding_decoder; + // Encoding headers cannot legitimately advertise more physical values than the data page's + // logical value count; establish the bound before decoders inspect external counts. + _page_decoder->set_expected_values(_remaining_num_values); + RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); + } + + _state = DATA_LOADED; + return Status::OK(); +} + +template +Status ColumnChunkReader::_decode_dict_page() { + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); + SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time); + + // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. + // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files. + // refer: https://github.com/apache/parquet-format/blob/master/Encodings.md + tparquet::Encoding::type dict_encoding = header->dictionary_page_header.encoding; + if (dict_encoding != tparquet::Encoding::PLAIN_DICTIONARY && + dict_encoding != tparquet::Encoding::PLAIN) { + return Status::InternalError("Unsupported dictionary encoding {}", + tparquet::to_string(dict_encoding)); + } + + // Prepare dictionary data + int32_t uncompressed_size = header->uncompressed_page_size; + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + RETURN_IF_ERROR(validate_dictionary_page_size(*header, _get_type_length())); + DorisUniqueBufferPtr dict_data; + bool dict_buffer_allocated = false; + const auto allocate_dict_buffer = [&]() { + if (!dict_buffer_allocated) { + dict_data = make_unique_buffer(uncompressed_size); + dict_buffer_allocated = true; + } + }; + bool dict_loaded = false; + + // Try to load dictionary page from cache + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + if (UNLIKELY(header_size > cached.size)) { + return Status::Corruption( + "Cached Parquet dictionary is shorter than its page header"); + } + // Dictionary page layout in cache: header | payload (compressed or uncompressed) + Slice payload_slice(cached.data + header_size, cached.size - header_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + + if (cache_payload_is_decompressed) { + // Use cached decompressed dictionary data + if (UNLIKELY(payload_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Cached Parquet dictionary payload has size {}, expected {}", + payload_slice.size, uncompressed_size); + } + allocate_dict_buffer(); + memcpy(dict_data.get(), payload_slice.data, payload_slice.size); + dict_loaded = true; + } else { + CHECK(_block_compress_codec); + // Decompress cached compressed dictionary data + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, payload_slice, static_cast(uncompressed_size))); + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + dict_loaded = true; + } + + // When dictionary page is loaded from cache, we need to skip the page data + // to update the offset correctly (similar to calling get_page_data()) + if (dict_loaded) { + _page_reader->skip_page_data(); + } + } + } + + if (!dict_loaded) { + // Load and decompress dictionary page from file + if (_block_compress_codec != nullptr) { + auto dict_num = header->dictionary_page_header.num_values; + Slice compressed_data; + if (dict_num != 0) { + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, compressed_data, static_cast(uncompressed_size))); + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); + // Dictionary probes stop before data pages, so count their decompression here or + // metadata pruning profiles will report no codec work for the scan. + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR( + _block_compress_codec->decompress(compressed_data, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + } + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); + + // Decide whether to cache decompressed or compressed dictionary based on threshold + // If uncompressed_page_size == 0, should_cache_decompressed will return true + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; // Dictionary pages don't have levels + if (cache_payload_decompressed) { + // Cache the decompressed dictionary page + // If dict_num == 0, `dict_slice` will be empty + _insert_page_into_cache(empty_levels, dict_slice); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + DCHECK(!compressed_data.empty()); + // Cache the compressed dictionary page + _insert_page_into_cache(empty_levels, + Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + // `get_page_data` not called, we should skip the page data + // Because `_insert_page_into_cache` will use _page_reader, we should exec `skip_page_data` after `_insert_page_into_cache` + if (dict_num == 0) { + _page_reader->skip_page_data(); + } + } else { + Slice dict_slice; + RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice)); + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet dictionary payload has size {}, expected {}", + dict_slice.size, uncompressed_size); + } + allocate_dict_buffer(); + // The data is stored by BufferedStreamReader, we should copy it out + memcpy(dict_data.get(), dict_slice.data, dict_slice.size); + + // Cache the uncompressed dictionary page + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; + Slice payload(dict_data.get(), uncompressed_size); + _insert_page_into_cache(empty_levels, payload); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + allocate_dict_buffer(); + + // Cache page decoder + std::unique_ptr page_decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + // Set the dictionary data + RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, + header->dictionary_page_header.num_values)); + _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] = std::move(page_decoder); + + _has_dict = true; + return Status::OK(); +} + +template +void ColumnChunkReader::_reserve_decompress_buf(size_t size) { + if (size > _decompress_buf_size) { + _decompress_buf_size = BitUtil::next_power_of_two(size); + _decompress_buf = make_unique_buffer(_decompress_buf_size); + } +} + +template +void ColumnChunkReader::_insert_page_into_cache( + const std::vector& level_bytes, const Slice& payload) { + StoragePageCache::CacheKey key = + _page_reader->make_page_cache_key(_page_reader->header_start_offset()); + const std::vector& header_bytes = _page_reader->header_bytes(); + size_t total = header_bytes.size() + level_bytes.size() + payload.size; + auto page = std::make_unique(total, true, segment_v2::DATA_PAGE); + size_t pos = 0; + memcpy(page->data() + pos, header_bytes.data(), header_bytes.size()); + pos += header_bytes.size(); + if (!level_bytes.empty()) { + memcpy(page->data() + pos, level_bytes.data(), level_bytes.size()); + pos += level_bytes.size(); + } + if (payload.size > 0) { + memcpy(page->data() + pos, payload.data, payload.size); + pos += payload.size; + } + page->reset_size(total); + PageCacheHandle handle; + StoragePageCache::instance()->insert(key, page.get(), &handle, segment_v2::DATA_PAGE); + page.release(); + _chunk_statistics.page_cache_write_counter += 1; +} + +template +Status ColumnChunkReader::skip_values(size_t num_values, + bool skip_data) { + if (UNLIKELY(_remaining_num_values < num_values)) { + return Status::IOError("Skip too many values in current page. {} vs. {}", + _remaining_num_values, num_values); + } + if (skip_data) { + if (UNLIKELY(_empty_value_section && num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + RETURN_IF_ERROR(_page_decoder->skip_values(num_values)); + } + // Commit logical page progress only after the physical decoder accepted the whole request. + _remaining_num_values -= num_values; + return Status::OK(); +} + +template +Status ColumnChunkReader::materialize_values( + MutableColumnPtr& doris_column, const DataTypeSerDe& serde, ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector) { + if (select_vector.num_values() == 0) { + return Status::OK(); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + const size_t physical_values = select_vector.num_values() - select_vector.num_nulls(); + if (UNLIKELY(_empty_value_section && physical_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + physical_values); + } + if (UNLIKELY((doris_column->is_column_dictionary() || context.dictionary_index_only) && + !_has_dict && physical_values != 0)) { + return Status::IOError("Not dictionary coded"); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); + Status status; + if (select_vector.has_filter()) { + if (select_vector.num_nulls() == 0) { + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_non_null_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else if (visit_nullable_expandable_column(*doris_column, [](auto&) {})) { + // Parquet omits NULL leaf values from the physical stream. For example, the logical + // DATE sequence [d0, NULL, d1, NULL, d2] is physically encoded as [d0, d1, d2]. The + // generic fallback follows the logical selection runs, so those NULLs split one + // contiguous physical span into decode(d0), insert-NULL, decode(d1), insert-NULL, + // decode(d2). On nullable sparse scans this repeatedly enters SerDe and turns decimal + // scaling, date conversion, timestamp/timezone conversion, or dictionary-ID + // materialization into many tiny calls even though the physical values are adjacent. + // + // Build selected non-NULL ranges in physical coordinates instead. In the example this + // decodes [d0, d1, d2] compactly with one SerDe consumer, records [0, 1, 0, 1, 0] as + // the logical NULL layout, and expands the final nested column backwards so unread + // compact values cannot be overwritten. Apply this to every expandable V2 scalar, + // including ordinary PLAIN primitives: otherwise nullable numeric predicates still + // fragment a physical span into millions of SerDe calls and defeat sparse decoding. + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_nullable_values( + *doris_column, serde, *_page_decoder, context, state, select_vector, + _nullable_selection_nulls, &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else { + ++_chunk_statistics.hybrid_selection_null_fallback_batches; + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + } + } else { + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, + &_chunk_statistics.materialization_time); + } + RETURN_IF_ERROR(status); + _remaining_num_values -= select_vector.num_values(); + return Status::OK(); +} + +template +bool ColumnChunkReader::can_filter_fixed_width_values( + const VExprSPtrs& conjuncts, int column_id) const { + if (conjuncts.empty() || + !supports_raw_fixed_filter_encoding(_current_encoding, _metadata.type)) { + return false; + } + const auto primitive_type = remove_nullable(_field_schema->data_type)->get_primitive_type(); + const bool has_identity_width = + (_metadata.type == tparquet::Type::INT32 && primitive_type == TYPE_INT) || + (_metadata.type == tparquet::Type::INT64 && primitive_type == TYPE_BIGINT) || + (_metadata.type == tparquet::Type::FLOAT && primitive_type == TYPE_FLOAT) || + (_metadata.type == tparquet::Type::DOUBLE && primitive_type == TYPE_DOUBLE); + if (!has_identity_width) { + // Raw predicates consume the physical Parquet width. Logical conversions such as UINT32 + // to BIGINT must stay on the typed path or a four-byte value is interpreted as eight bytes. + return false; + } + return std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + }); +} + +template +Status ColumnChunkReader::filter_fixed_width_values( + const VExprSPtrs& conjuncts, int column_id, ColumnSelectVector& select_vector, + NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) { + DORIS_CHECK(selected_nulls != nullptr); + DORIS_CHECK(physical_matches != nullptr); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + *used_filter = false; + row_filter->clear(); + if (!can_filter_fixed_width_values(conjuncts, column_id)) { + return Status::OK(); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + + ParquetSelection selection; + selected_nulls->clear(); + selected_nulls->reserve(select_vector.num_values() - select_vector.num_filtered()); + size_t physical_cursor = 0; + auto build_selection = [&]() { + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == + physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls->resize_fill(selected_nulls->size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls->resize_fill(selected_nulls->size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + }; + if (select_vector.has_filter()) { + build_selection.template operator()(); + } else { + build_selection.template operator()(); + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls->size(), + select_vector.num_values() - select_vector.num_filtered()); + if (UNLIKELY(_empty_value_section && selection.total_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + selection.total_values); + } + + physical_matches->clear(); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(_page_decoder->skip_values(selection.total_values)); + } else { + FixedWidthPredicateConsumer consumer(conjuncts, _field_schema->data_type, column_id, + physical_matches, projected_column); + RETURN_IF_ERROR(_page_decoder->decode_selected_fixed_values(selection, consumer)); + DORIS_CHECK_EQ(physical_matches->size(), selection.selected_values); + } + + row_filter->reserve(selected_nulls->size()); + size_t physical_row = 0; + for (const uint8_t is_null : *selected_nulls) { + row_filter->push_back(is_null != 0 ? 0 : (*physical_matches)[physical_row++]); + } + DORIS_CHECK_EQ(physical_row, physical_matches->size()); + // Commit logical progress only after both the raw comparison and NULL remapping succeed. + _remaining_num_values -= select_vector.num_values(); + *used_filter = true; + return Status::OK(); +} + +template +Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { + if constexpr (OFFSET_INDEX) { + if (_page_reader->has_active_offset_index()) { + while (true) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + break; + } else if (has_next_page()) { + RETURN_IF_ERROR(next_page()); + _current_row = _page_reader->start_row(); + } else [[unlikely]] { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } + + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + return Status::OK(); + } + } + + while (true) { + RETURN_IF_ERROR(parse_page_header()); + if (_page_reader->is_header_v2() || !IN_COLLECTION) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + RETURN_IF_ERROR(load_page_data()); + // this page contain this row. + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + break; + } + + _current_row = _page_reader->end_row(); + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } else { + RETURN_IF_ERROR(load_page_data()); + std::vector rep_levels; + std::vector def_levels; + bool cross_page = false; + + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row, &result_rows, + &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + bool need_load_next_page = true; + while (cross_page) { + need_load_next_page = false; + rep_levels.clear(); + def_levels.clear(); + RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + } + if (left_row == _current_row) { + break; + } + if (need_load_next_page) { + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } + } + }; + + return Status::OK(); +} + +template +Status ColumnChunkReader::_skip_nested_rows_in_page(size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + + std::vector rep_levels; + std::vector def_levels; + + bool cross_page = false; + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + DCHECK(cross_page == false); + if (num_rows != result_rows) [[unlikely]] { + return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows, + result_rows); + } + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_nested_rows( + std::vector& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) { + if (_state != DATA_LOADED) [[unlikely]] { + return Status::IOError("Should load page data first to load nested rows"); + } + *cross_page = false; + *result_rows = 0; + // Reserve only the requested row frontier. One nested row may legitimately contain more + // values and grow the vector incrementally, but a forged page count must not allocate gigabytes + // before the level stream proves those values exist. + const size_t requested_frontier = + max_rows == std::numeric_limits::max() ? max_rows : max_rows + 1; + rep_levels.reserve(rep_levels.size() + + std::min(_remaining_rep_nums, requested_frontier)); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (IN_COLLECTION) { + // A continuation level is valid across later V1 pages only after this chunk has seen + // a row start; accepting it on the first sequential page invents an orphan parent row. + if (!_page_reader->has_active_offset_index() && !_nested_row_started && + rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + if (!_page_reader->has_active_offset_index()) { + _nested_row_started = true; + } + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + if (*result_rows == max_rows) { // this page contain max_rows, page no end. + _current_row += max_rows; + _rep_level_rewind_one(); + return Status::OK(); + } + (*result_rows)++; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + _current_row += *result_rows; + + if ((_page_reader->is_header_v2() || _page_reader->has_active_offset_index()) && + UNLIKELY(_current_row != _page_reader->end_row())) { + // V2 and OffsetIndex advertise an exact logical row span. A page that exhausts its + // repetition levels without that many row starts would otherwise make the caller retry + // the same row forever. + return Status::Corruption( + "Parquet nested data page ended at row {}, expected page end row {}", _current_row, + _page_reader->end_row()); + } + + auto need_check_cross_page = [&]() -> bool { + return IN_COLLECTION && !_page_reader->has_active_offset_index() && + _remaining_rep_nums == 0 && !_page_reader->is_header_v2() && has_next_page(); + }; + *cross_page = need_check_cross_page(); + return Status::OK(); +}; + +template +Status ColumnChunkReader::load_cross_page_nested_row( + std::vector& rep_levels, bool* cross_page) { + RETURN_IF_ERROR(next_page()); + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + + *cross_page = has_next_page(); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (IN_COLLECTION) { + if (!_page_reader->has_active_offset_index() && !_nested_row_started && + rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + if (!_page_reader->has_active_offset_index()) { + _nested_row_started = true; + } + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + *cross_page = false; + _rep_level_rewind_one(); + break; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + return Status::OK(); +} + +template +int32_t ColumnChunkReader::_get_type_length() { + switch (_field_schema->physical_type) { + case tparquet::Type::INT32: + [[fallthrough]]; + case tparquet::Type::FLOAT: + return 4; + case tparquet::Type::INT64: + [[fallthrough]]; + case tparquet::Type::DOUBLE: + return 8; + case tparquet::Type::INT96: + return 12; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + return _field_schema->parquet_schema.type_length; + default: + return -1; + } +} + +/** + * Checks if the given column has a dictionary page. + * + * This function determines the presence of a dictionary page by checking the + * dictionary_page_offset field in the column metadata. The dictionary_page_offset + * must be set and greater than 0, and it must be less than the data_page_offset. + * + * The reason for these checks is based on the implementation in the Java version + * of ORC, where dictionary_page_offset is used to indicate the absence of a dictionary. + * Additionally, Parquet may write an empty row group, in which case the dictionary page + * content would be empty, and thus the dictionary page should not be read. + * + * See https://github.com/apache/arrow/pull/2667/files + */ +bool has_dict_page(const tparquet::ColumnMetaData& column) { + return column.__isset.dictionary_page_offset && column.dictionary_page_offset > 0 && + column.dictionary_page_offset < column.data_page_offset; +} + +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h new file mode 100644 index 00000000000000..102366f24949ee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -0,0 +1,416 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "util/slice.h" + +namespace doris { +class BlockCompressionCodec; +class DataTypeSerDe; +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris::format::parquet::native { +using ::doris::ColumnString; + +struct ColumnChunkRange { + size_t offset = 0; + size_t length = 0; +}; + +struct ParquetReaderCompat { + bool parquet_816_padding = false; + bool data_page_v2_always_compressed = false; +}; + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by); +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range); +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count); +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available); +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed); +Status validate_fixed_width_page_size(const tparquet::PageHeader& header, int32_t type_length, + level_t max_rep_level, level_t max_def_level, + bool schema_is_required = true); +Status validate_dictionary_page_size(const tparquet::PageHeader& header, int32_t type_length = -1); +Status validate_compressed_page_size(tparquet::CompressionCodec::type codec, + const Slice& compressed_data, + size_t expected_uncompressed_size); + +struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t materialization_time = 0; + int64_t hybrid_selection_batches = 0; + int64_t hybrid_selection_ranges = 0; + int64_t hybrid_selection_null_fallback_batches = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; +}; + +/** + * Read and decode parquet column data into doris block column. + *

Usage:

+ * // Create chunk reader + * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, + * tparquet::ColumnChunk* column_chunk, + * NativeFieldSchema* fieldSchema); + * // Initialize chunk reader + * chunk_reader.init(); + * while (chunk_reader.has_next_page()) { + * // Seek to next page header. Only read and parse the page header, not page data. + * chunk_reader.next_page(); + * // Load data to decoder. Load the page data into underlying container. + * // Or, we can call the chunk_reader.skip_page() to skip current page. + * chunk_reader.load_page_data(); + * // Decode values into column or slice. + * // Or, we can call chunk_reader.skip_values(num_values) to skip some values. + * chunk_reader.materialize_values(column, serde, context, state, selection); + * } + */ +template +class ColumnChunkReader { +public: + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_row, io::IOContext* io_ctx, + const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range = nullptr); + ~ColumnChunkReader() = default; + + // Initialize chunk reader, will generate the decoder and codec. + Status init(); + + // Whether the chunk reader has a more page to read. + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + if (_page_reader->has_active_offset_index()) { + return _page_reader->has_next_page(); + } + } + // A nested V1 page can invalidate its offset index after initialization. In that mode the + // validated byte range may contain writer padding, so logical values—not remaining bytes— + // are the authoritative chunk boundary. + return _chunk_parsed_values < _metadata.num_values; + } + + // Skip some values(will not read and parse) in current page if the values are filtered by predicates. + // when skip_data = false, the underlying decoder will not skip data, + // only used when maintaining the consistency of _remaining_num_values. + Status skip_values(size_t num_values, bool skip_data = true); + + // Load page data into the underlying container, + // and initialize the repetition and definition level decoder for current page data. + Status load_page_data(); + Status load_page_data_idempotent() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + return load_page_data(); + } + // The remaining number of values in current page(including null values). Decreased when reading or skipping. + uint32_t remaining_num_values() const { return _remaining_num_values; } + + // Apply one logical selection to the encoded page. Decoder advances physical payload cursors; + // SerDe interprets each selected raw span and materializes the destination Doris column. + Status materialize_values(MutableColumnPtr& doris_column, const DataTypeSerDe& serde, + ParquetDecodeContext& context, ParquetMaterializationState& state, + ColumnSelectVector& select_vector); + + static bool supports_raw_fixed_filter_encoding(tparquet::Encoding::type encoding, + tparquet::Type::type physical_type) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return physical_type == tparquet::Type::INT32 || + physical_type == tparquet::Type::INT64 || + physical_type == tparquet::Type::FLOAT || + physical_type == tparquet::Type::DOUBLE; + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return physical_type == tparquet::Type::INT32 || + physical_type == tparquet::Type::INT64 || + physical_type == tparquet::Type::FLOAT || + physical_type == tparquet::Type::DOUBLE; + case tparquet::Encoding::DELTA_BINARY_PACKED: + return physical_type == tparquet::Type::INT32 || physical_type == tparquet::Type::INT64; + default: + return false; + } + } + + // Evaluate selected fixed-width values and return one keep byte per selected logical row. + // NULL comparisons are false and therefore never enter the physical consumer; non-null + // matches are appended to projected_column when requested. + Status filter_fixed_width_values(const VExprSPtrs& conjuncts, int column_id, + ColumnSelectVector& select_vector, NullMap* selected_nulls, + IColumn::Filter* physical_matches, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter); + bool can_filter_fixed_width_values(const VExprSPtrs& conjuncts, int column_id) const; + + // Get the repetition level decoder of current page. + LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } + // Get the definition level decoder of current page. + LevelDecoder& def_level_decoder() { return _def_level_decoder; } + + level_t max_rep_level() const { return _max_rep_level; } + level_t max_def_level() const { return _max_def_level; } + + bool has_dict() const { return _has_dict; }; + + // Get page decoder + Decoder* get_page_decoder() { return _page_decoder; } + + void release_decoder_scratch(size_t max_retained_bytes) { + for (auto& [encoding, decoder] : _decoders) { + decoder->release_scratch(max_retained_bytes); + } + // Level decoders may batch-convert unsigned RLE values into Doris' signed level_t. + _rep_level_decoder.release_scratch(max_retained_bytes); + _def_level_decoder.release_scratch(max_retained_bytes); + if (_decompress_buf_size > max_retained_bytes) { + if (_page_uses_decompress_buf) { + // Keep the request until the page boundary because decoders still point into this + // allocation; dropping it now would trade retained memory for a use-after-free. + _decompress_release_pending = true; + _decompress_release_threshold = + std::min(_decompress_release_threshold, max_retained_bytes); + } else { + _decompress_buf.reset(); + _decompress_buf_size = 0; + _decompress_release_pending = false; + _decompress_release_threshold = std::numeric_limits::max(); + } + } + } + + size_t retained_decoder_scratch_bytes() const { + size_t bytes = _decompress_buf_size + _rep_level_decoder.retained_scratch_bytes() + + _def_level_decoder.retained_scratch_bytes(); + for (const auto& [encoding, decoder] : _decoders) { + bytes += decoder->retained_scratch_bytes(); + } + return bytes; + } + + size_t active_decoder_scratch_bytes() const { + // Only the current encoding is active. Old decoder instances retain reusable capacity but + // must not make the high-water policy treat their last batch as current working memory. + return _active_decompress_bytes + + (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + + _rep_level_decoder.active_scratch_bytes() + + _def_level_decoder.active_scratch_bytes(); + } + + tparquet::Encoding::type current_encoding() const { return _current_encoding; } + + ColumnChunkReaderStatistics& chunk_statistics() { + _chunk_statistics.decode_header_time = _page_reader->page_statistics().decode_header_time; + _chunk_statistics.skip_page_header_num = + _page_reader->page_statistics().skip_page_header_num; + _chunk_statistics.parse_page_header_num = + _page_reader->page_statistics().parse_page_header_num; + _chunk_statistics.read_page_header_time = + _page_reader->page_statistics().read_page_header_time; + // PageReader statistics are already cumulative. Snapshot them instead of folding the same + // totals into the chunk on every FileScannerV2 batch. Cache write counters are owned by + // ColumnChunkReader because insertion happens after decompression and therefore remain in + // the chunk accumulator above. + _chunk_statistics.page_read_counter = _page_reader->page_statistics().page_read_counter; + _chunk_statistics.data_page_read_counter = + _page_reader->page_statistics().data_page_read_counter; + _chunk_statistics.page_cache_hit_counter = + _page_reader->page_statistics().page_cache_hit_counter; + _chunk_statistics.page_cache_missing_counter = + _page_reader->page_statistics().page_cache_missing_counter; + _chunk_statistics.page_cache_compressed_hit_counter = + _page_reader->page_statistics().page_cache_compressed_hit_counter; + _chunk_statistics.page_cache_decompressed_hit_counter = + _page_reader->page_statistics().page_cache_decompressed_hit_counter; + return _chunk_statistics; + } + + Decoder* dictionary_decoder() { + return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)].get(); + } + + size_t page_start_row() const { return _page_reader->start_row(); } + + size_t page_end_row() const { return _page_reader->end_row(); } + + Status parse_page_header(); + Status next_page(); + + Status seek_to_nested_row(size_t left_row); + // Decode level slots without materializing their values. `read_levels` is the flat-column + // counterpart of load_page_nested_rows()+fill_def(): both advance definition/repetition and + // value streams together so a levels-only consumer cannot desynchronize the next data page. + Status read_levels(size_t num_values, std::vector* rep_levels, + std::vector* def_levels); + Status skip_nested_values(const std::vector& def_levels, size_t start_index = 0); + Status fill_def(std::vector& def_values) { + auto before_sz = def_values.size(); + auto append_sz = _remaining_def_nums - _remaining_rep_nums; + def_values.resize(before_sz + append_sz, 0); + if (max_def_level() != 0) { + auto ptr = def_values.data() + before_sz; + const size_t decoded = _def_level_decoder.get_levels(ptr, append_sz); + if (UNLIKELY(decoded != append_sz)) { + def_values.resize(before_sz); + return Status::Corruption( + "Parquet definition level stream ended after {} of {} slots", decoded, + append_sz); + } + } + _remaining_def_nums -= append_sz; + return Status::OK(); + } + + Status load_page_nested_rows(std::vector& rep_levels, size_t max_rows, + size_t* result_rows, bool* cross_page); + Status load_cross_page_nested_row(std::vector& rep_levels, bool* cross_page); + + Slice get_page_data() const { return _page_data; } + const Slice& v2_rep_levels() const { return _v2_rep_levels; } + const Slice& v2_def_levels() const { return _v2_def_levels; } + ColumnChunkReaderStatistics& statistics() { return chunk_statistics(); } + +private: + enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED }; + + // for check dict page. + Status _parse_first_page_header(); + Status _decode_dict_page(); + + void _reserve_decompress_buf(size_t size); + int32_t _get_type_length(); + void _insert_page_into_cache(const std::vector& level_bytes, const Slice& payload); + + Status _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _skip_nested_rows_in_page(size_t num_rows); + + level_t _rep_level_get_next() { + if constexpr (IN_COLLECTION) { + return _rep_level_decoder.get_next(); + } + return 0; + } + + void _rep_level_rewind_one() { + if constexpr (IN_COLLECTION) { + _rep_level_decoder.rewind_one(); + } + } + + ColumnChunkReaderState _state = NOT_INIT; + NativeFieldSchema* _field_schema = nullptr; + const level_t _max_rep_level; + const level_t _max_def_level; + + io::BufferedStreamReader* _stream_reader = nullptr; + tparquet::ColumnMetaData _metadata; + const tparquet::OffsetIndex* _offset_index = nullptr; + size_t _current_row = 0; + size_t _total_rows = 0; + io::IOContext* _io_ctx = nullptr; + + std::unique_ptr> _page_reader; + BlockCompressionCodec* _block_compress_codec = nullptr; + + ParquetPageReadContext _page_read_ctx; + ColumnChunkRange _chunk_range; + bool _has_validated_chunk_range = false; + + LevelDecoder _rep_level_decoder; + LevelDecoder _def_level_decoder; + size_t _chunk_parsed_values = 0; + // this page remaining rep/def nums + // if max_rep_level = 0 / max_def_level = 0, this value retail hava value. + uint32_t _remaining_rep_nums = 0; + uint32_t _remaining_def_nums = 0; + // this page remaining values to be processed (for read/skip). + // need parse this page header. + uint32_t _remaining_num_values = 0; + + Slice _page_data; + DorisUniqueBufferPtr _decompress_buf; + size_t _decompress_buf_size = 0; + bool _page_uses_decompress_buf = false; + size_t _active_decompress_bytes = 0; + bool _decompress_release_pending = false; + size_t _decompress_release_threshold = std::numeric_limits::max(); + Slice _v2_rep_levels; + Slice _v2_def_levels; + bool _dict_checked = false; + bool _has_dict = false; + bool _nested_row_started = false; + Decoder* _page_decoder = nullptr; + std::unique_ptr _empty_value_decoder; + bool _empty_value_section = false; + tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; + // Map: encoding -> Decoder + // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding + std::unordered_map> _decoders; + NullMap _nullable_selection_nulls; + ColumnChunkReaderStatistics _chunk_statistics; +}; + +bool has_dict_page(const tparquet::ColumnMetaData& column); + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp new file mode 100644 index 00000000000000..592e048846e137 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -0,0 +1,1902 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_reader.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet::native { +namespace { + +ParquetTimeUnit parquet_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; + } + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; + } + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; + } + return ParquetTimeUnit::UNKNOWN; +} + +template +bool release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + DORIS_CHECK(values != nullptr); + if (values->capacity() * sizeof(T) <= max_retained_bytes) { + return false; + } + std::vector().swap(*values); + return true; +} + +size_t retained_set_bytes(const std::unordered_set& values) { + return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); +} + +Status validate_decimal_physical_type(const NativeFieldSchema& field, int precision, int scale) { + if (precision <= 0 || scale < 0 || scale > precision) { + return Status::Corruption("Parquet decimal field {} has invalid precision {} and scale {}", + field.name, precision, scale); + } + switch (field.physical_type) { + case tparquet::Type::INT32: + if (precision <= 9) { + return Status::OK(); + } + break; + case tparquet::Type::INT64: + if (precision <= 18) { + return Status::OK(); + } + break; + case tparquet::Type::BYTE_ARRAY: + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + const int length = + field.parquet_schema.__isset.type_length ? field.parquet_schema.type_length : -1; + if (length > 0) { + const int64_t max_precision = (static_cast(length) * 8 - 1) * 30103 / 100000; + if (precision <= max_precision) { + return Status::OK(); + } + } + break; + } + default: + break; + } + return Status::Corruption("Parquet decimal field {} has incompatible physical type {}", + field.name, tparquet::to_string(field.physical_type)); +} + +Status validate_physical_annotation(const NativeFieldSchema& field) { + const auto& schema = field.parquet_schema; + if (field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + (!schema.__isset.type_length || schema.type_length <= 0)) { + return Status::Corruption("Parquet fixed-length field {} has invalid width {}", field.name, + schema.__isset.type_length ? schema.type_length : -1); + } + auto require = [&](bool valid, std::string_view annotation) -> Status { + if (valid) { + return Status::OK(); + } + return Status::Corruption("Parquet {} field {} is incompatible with physical type {}", + annotation, field.name, tparquet::to_string(field.physical_type)); + }; + + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "string"); + } + if (logical.__isset.DECIMAL) { + return validate_decimal_physical_type(field, logical.DECIMAL.precision, + logical.DECIMAL.scale); + } + if (logical.__isset.DATE) { + return require(field.physical_type == tparquet::Type::INT32, "date"); + } + if (logical.__isset.TIME) { + const auto unit = parquet_time_unit(logical.TIME.unit); + return require( + (unit == ParquetTimeUnit::MILLIS && + field.physical_type == tparquet::Type::INT32) || + ((unit == ParquetTimeUnit::MICROS || unit == ParquetTimeUnit::NANOS) && + field.physical_type == tparquet::Type::INT64), + "time"); + } + if (logical.__isset.TIMESTAMP) { + return require( + field.physical_type == tparquet::Type::INT64 && + parquet_time_unit(logical.TIMESTAMP.unit) != ParquetTimeUnit::UNKNOWN, + "timestamp"); + } + if (logical.__isset.INTEGER) { + const int width = logical.INTEGER.bitWidth; + return require(((width == 8 || width == 16 || width == 32) && + field.physical_type == tparquet::Type::INT32) || + (width == 64 && field.physical_type == tparquet::Type::INT64), + "integer"); + } + if (logical.__isset.UUID) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 16, + "UUID"); + } + if (logical.__isset.FLOAT16) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 2, + "FLOAT16"); + } + if (logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + // Geospatial WKB has no Doris logical mapping here; keep its BYTE_ARRAY payload raw so + // reader validation preserves the physical fallback chosen by native schema inference. + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "geospatial"); + } + if (logical.__isset.UNKNOWN) { + return Status::OK(); + } + return Status::Corruption("Unsupported Parquet logical annotation on field {}", field.name); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "converted string"); + case tparquet::ConvertedType::DECIMAL: + return validate_decimal_physical_type(field, + schema.__isset.precision ? schema.precision : -1, + schema.__isset.scale ? schema.scale : -1); + case tparquet::ConvertedType::DATE: + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + return require(field.physical_type == tparquet::Type::INT32, "converted INT32"); + case tparquet::ConvertedType::TIME_MICROS: + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_64: + return require(field.physical_type == tparquet::Type::INT64, "converted INT64"); + case tparquet::ConvertedType::INTERVAL: + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 12, + "interval"); + default: + return Status::Corruption("Unsupported Parquet converted annotation {} on field {}", + tparquet::to_string(schema.converted_type), field.name); + } +} + +IColumn::Filter* conversion_failure_map(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn::Filter* output_null_map, + IColumn::Filter* compatibility_scratch) { + const auto& schema = field.parquet_schema; + const bool is_utc_timestamp = + field.physical_type == tparquet::Type::INT96 || + (field.physical_type == tparquet::Type::INT64 && + ((schema.__isset.logicalType && schema.logicalType.__isset.TIMESTAMP && + schema.logicalType.TIMESTAMP.isAdjustedToUTC) || + (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS || + schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MICROS)))); + if (!strict_mode && output_null_map != nullptr && is_utc_timestamp && + remove_nullable(target_type)->get_primitive_type() == TYPE_DATETIMEV2) { + // Legacy UTC timestamp conversion kept out-of-range values as DATETIME defaults. Local + // timestamps intentionally keep non-strict NULL-on-overflow behavior because timezone + // conversion is not part of their representation. + compatibility_scratch->resize_fill(output_null_map->size(), 0); + return compatibility_scratch; + } + return output_null_map; +} + +void mark_local_timestamp_defaults(const NativeFieldSchema& field, const DataTypePtr& target_type, + bool strict_mode, IColumn& data_column, + IColumn::Filter* output_null_map, size_t start_row) { + const auto& schema = field.parquet_schema; + const bool is_local_timestamp = + field.physical_type == tparquet::Type::INT64 && schema.__isset.logicalType && + schema.logicalType.__isset.TIMESTAMP && !schema.logicalType.TIMESTAMP.isAdjustedToUTC; + if (strict_mode || output_null_map == nullptr || !is_local_timestamp || + remove_nullable(target_type)->get_primitive_type() != TYPE_DATETIMEV2) { + return; + } + auto& values = assert_cast(data_column).get_data(); + DORIS_CHECK_EQ(values.size(), output_null_map->size()); + for (size_t row = start_row; row < values.size(); ++row) { + if ((*output_null_map)[row] == 0 && !values[row].is_valid_date()) { + // Local timestamps before Doris' representable calendar can materialize as a zero + // date without a SerDe error. Preserve non-strict scan semantics by nulling only that + // sentinel; physical NULLs and valid local timestamps keep their original map bits. + (*output_null_map)[row] = 1; + } + } +} + +Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + DORIS_CHECK(context != nullptr); + // Annotation validation belongs before decoder construction: accepting an impossible pair + // lets the logical SerDe reinterpret a differently sized physical value stream. + RETURN_IF_ERROR(validate_physical_annotation(field)); + switch (field.physical_type) { + case tparquet::Type::BOOLEAN: + context->physical_type = ParquetPhysicalType::BOOLEAN; + break; + case tparquet::Type::INT32: + context->physical_type = ParquetPhysicalType::INT32; + break; + case tparquet::Type::INT64: + context->physical_type = ParquetPhysicalType::INT64; + break; + case tparquet::Type::INT96: + context->physical_type = ParquetPhysicalType::INT96; + break; + case tparquet::Type::FLOAT: + context->physical_type = ParquetPhysicalType::FLOAT; + break; + case tparquet::Type::DOUBLE: + context->physical_type = ParquetPhysicalType::DOUBLE; + break; + case tparquet::Type::BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::BYTE_ARRAY; + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY; + break; + default: + return Status::NotSupported("Unsupported Parquet physical type {}", + tparquet::to_string(field.physical_type)); + } + + const auto& schema = field.parquet_schema; + context->type_length = schema.__isset.type_length ? schema.type_length : -1; + context->decimal_precision = schema.__isset.precision ? schema.precision : -1; + context->decimal_scale = schema.__isset.scale ? schema.scale : -1; + context->timezone = ctz; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + context->logical_type = ParquetLogicalType::STRING; + } else if (logical.__isset.DECIMAL) { + context->logical_type = ParquetLogicalType::DECIMAL; + context->decimal_precision = logical.DECIMAL.precision; + context->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.DATE) { + context->logical_type = ParquetLogicalType::DATE; + } else if (logical.__isset.TIME) { + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = parquet_time_unit(logical.TIME.unit); + } else if (logical.__isset.TIMESTAMP) { + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = parquet_time_unit(logical.TIMESTAMP.unit); + context->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + } else if (logical.__isset.INTEGER) { + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_bit_width = logical.INTEGER.bitWidth; + context->logical_integer_is_signed = logical.INTEGER.isSigned; + } else if (logical.__isset.UUID) { + context->logical_type = ParquetLogicalType::UUID; + context->logical_uuid = true; + } else if (logical.__isset.FLOAT16) { + context->logical_type = ParquetLogicalType::FLOAT16; + context->logical_float16 = true; + } + if (context->logical_uuid && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 16)) { + return Status::Corruption("Parquet UUID field {} must be FIXED_LEN_BYTE_ARRAY(16)", + field.name); + } + if (context->logical_float16 && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 2)) { + return Status::Corruption("Parquet FLOAT16 field {} must be FIXED_LEN_BYTE_ARRAY(2)", + field.name); + } + return Status::OK(); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + context->logical_type = ParquetLogicalType::STRING; + break; + case tparquet::ConvertedType::DECIMAL: + context->logical_type = ParquetLogicalType::DECIMAL; + break; + case tparquet::ConvertedType::DATE: + context->logical_type = ParquetLogicalType::DATE; + break; + case tparquet::ConvertedType::TIME_MILLIS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MILLIS; + break; + case tparquet::ConvertedType::TIME_MICROS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MICROS; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MILLIS; + // Legacy converted timestamps are defined as UTC-adjusted, unlike an unannotated INT64. + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::TIMESTAMP_MICROS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MICROS; + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::INT_64: + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_is_signed = + schema.converted_type >= tparquet::ConvertedType::INT_8; + context->logical_integer_bit_width = + schema.converted_type == tparquet::ConvertedType::UINT_8 || + schema.converted_type == tparquet::ConvertedType::INT_8 + ? 8 + : schema.converted_type == tparquet::ConvertedType::UINT_16 || + schema.converted_type == tparquet::ConvertedType::INT_16 + ? 16 + : schema.converted_type == tparquet::ConvertedType::UINT_32 || + schema.converted_type == tparquet::ConvertedType::INT_32 + ? 32 + : 64; + break; + default: + break; + } + return Status::OK(); +} + +} // namespace + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + return init_decode_context(field, ctz, context); +} + +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode) { + IColumn::Filter output_null_map; + output_null_map.resize_fill(1, 0); + IColumn::Filter compatibility_scratch; + return conversion_failure_map(field, target_type, strict_mode, &output_null_map, + &compatibility_scratch) == &compatibility_scratch; +} + +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row) { + mark_local_timestamp_defaults(field, target_type, strict_mode, data_column, output_null_map, + start_row); +} +#endif + +static void fill_struct_null_map(NativeFieldSchema* field, NullMap& null_map, + const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = def_levels.size(); + DCHECK_EQ(num_levels, rep_levels.size()); + size_t origin_size = null_map.size(); + null_map.resize(origin_size + num_levels); + size_t pos = origin_size; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (def_levels[i] >= field->definition_level) { + null_map[pos++] = 0; + } else { + null_map[pos++] = 1; + } + } + null_map.resize(pos); +} + +static Status fill_array_offset(NativeFieldSchema* field, ColumnArray::Offsets64& offsets_data, + NullMap* null_map_ptr, const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = rep_levels.size(); + if (UNLIKELY(num_levels != def_levels.size())) { + return Status::Corruption("Parquet repetition and definition level counts differ"); + } + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_levels); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(origin_size + num_levels); + } + size_t offset_pos = origin_size - 1; + bool parent_opened = false; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (rep_levels[i] == field->repetition_level) { + // A continuation can extend only a parent opened by this aligned logical batch. + if (UNLIKELY(!parent_opened)) { + return Status::Corruption( + "Parquet collection starts with an orphan repetition continuation"); + } + offsets_data[offset_pos]++; + continue; + } + parent_opened = true; + offset_pos++; + offsets_data[offset_pos] = offsets_data[offset_pos - 1]; + if (def_levels[i] >= field->definition_level) { + offsets_data[offset_pos]++; + } + if (null_map_ptr != nullptr) { + if (def_levels[i] >= field->definition_level - 1) { + (*null_map_ptr)[offset_pos] = 0; + } else { + (*null_map_ptr)[offset_pos] = 1; + } + } + } + offsets_data.resize(offset_pos + 1); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(offset_pos + 1); + } + return Status::OK(); +} + +Status ColumnReader::create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection, + const std::set& column_ids, + const std::set& filter_column_ids, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode) { + size_t total_rows = row_group.num_rows; + if (field->data_type->get_primitive_type() == TYPE_ARRAY) { + std::unique_ptr element_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + element_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, enable_strict_mode)); + auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + element_reader->set_column_in_nested(); + RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); + array_reader->_filter_column_ids = filter_column_ids; + reader.reset(array_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_MAP) { + std::unique_ptr key_reader; + std::unique_ptr value_reader; + + if (column_ids.empty() || + column_ids.find(field->children[0].get_column_id()) != column_ids.end()) { + // Create key reader + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + key_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[0]); + key_reader = std::move(skip_reader); + } + + if (column_ids.empty() || + column_ids.find(field->children[1].get_column_id()) != column_ids.end()) { + // Create value reader + RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, + value_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[1]); + value_reader = std::move(skip_reader); + } + + auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + key_reader->set_column_in_nested(); + value_reader->set_column_in_nested(); + RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field)); + map_reader->_filter_column_ids = filter_column_ids; + reader.reset(map_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { + std::unordered_map> child_readers; + child_readers.reserve(field->children.size()); + int non_skip_reader_idx = -1; + for (int i = 0; i < field->children.size(); ++i) { + auto& child = field->children[i]; + std::unique_ptr child_reader; + if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) { + RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, + in_collection, column_ids, filter_column_ids, + page_cache_file_key, compat, enable_strict_mode)); + child_readers[child.name] = std::move(child_reader); + // Record the first non-SkippingReader + if (non_skip_reader_idx == -1) { + non_skip_reader_idx = i; + } + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &child); + skip_reader->_filter_column_ids = filter_column_ids; + child_readers[child.name] = std::move(skip_reader); + } + child_readers[child.name]->set_column_in_nested(); + } + // If all children are SkipReadingReader, force the first child to call create + if (non_skip_reader_idx == -1) { + std::unique_ptr child_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, in_collection, + column_ids, filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + child_reader->set_column_in_nested(); + child_readers[field->children[0].name] = std::move(child_reader); + } + auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field)); + struct_reader->_filter_column_ids = filter_column_ids; + reader.reset(struct_reader.release()); + } else { + auto physical_index = field->physical_column_index; + if (physical_index < 0 || static_cast(physical_index) >= row_group.columns.size()) { + // Keep the leaf-to-chunk invariant checked at this consumer too because unit callers + // can construct a reader without going through NativeParquetMetadata::init_schema(). + return Status::Corruption("Parquet physical column index {} is out of range {}", + physical_index, row_group.columns.size()); + } + const auto offset_it = col_offsets.find(physical_index); + const tparquet::OffsetIndex* offset_index = + offset_it != col_offsets.end() ? &offset_it->second : nullptr; + + const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet physical column {} has no chunk metadata", + physical_index); + } + if (in_collection) { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } else { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } + } + return Status::OK(); +} + +void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const { + result_ranges->add(page_row_range); + RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges); +} + +template +Status ScalarColumnReader::init( + io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + bool enable_strict_mode) { + _field_schema = field; + auto& chunk_meta = _chunk_meta.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(chunk_meta, file->size(), compat.parquet_816_padding, + &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_len = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_len, max_buf_size); + if ((typeid_cast(file.get()) && + typeid_cast( + ((doris::io::TracingFileReader*)(file.get()))->inner_reader().get())) || + typeid_cast(file.get())) { + // turn off prefetch data when using MergeRangeFileReader + prefetch_buffer_size = 0; + } + _stream_reader = std::make_unique(file, chunk_start, chunk_len, + prefetch_buffer_size); + ParquetPageReadContext ctx( + (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache, + page_cache_file_key, compat.data_page_v2_always_compressed); + + _chunk_reader = std::make_unique>( + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx, + &chunk_range); + _materialization_state.enable_strict_mode = enable_strict_mode; + RETURN_IF_ERROR(_chunk_reader->init()); + RETURN_IF_ERROR(init_decode_context(*field, _ctz, &_decode_context)); + return Status::OK(); +} + +template +void ScalarColumnReader::release_batch_scratch( + size_t max_retained_bytes) { + const size_t retained_bytes = retained_batch_scratch_bytes(); + const size_t active_bytes = active_batch_scratch_bytes(); + if (retained_bytes <= max_retained_bytes || active_bytes > max_retained_bytes) { + _oversized_scratch_idle_batches = 0; + return; + } + // An adaptive probe or one repeated outlier must not pin memory forever, but immediately + // dropping a large steady-state buffer makes every following batch allocate it again. Require + // three ordinary batches before treating an oversized capacity as idle. + constexpr uint8_t OVERSIZED_SCRATCH_IDLE_BATCHES = 3; + if (++_oversized_scratch_idle_batches < OVERSIZED_SCRATCH_IDLE_BATCHES) { + return; + } + _oversized_scratch_idle_batches = 0; + if (_chunk_reader != nullptr) { + // Persistent decoders also own batch-sized value/slice buffers, not only the reader. + _chunk_reader->release_decoder_scratch(max_retained_bytes); + } + bool release_selection = false; + release_selection |= release_vector_if_oversized(&_rep_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_def_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_null_run_lengths, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_nested_filter_map_data, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.dictionary_indices, + max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.selection.ranges, + max_retained_bytes); + if (retained_set_bytes(_ancestor_null_indices) > max_retained_bytes) { + std::unordered_set().swap(_ancestor_null_indices); + release_selection = true; + } + if (release_selection) { + _select_vector = ColumnSelectVector(); + } +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->retained_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.capacity() * sizeof(level_t) + + _def_levels.capacity() * sizeof(level_t) + + _null_run_lengths.capacity() * sizeof(uint16_t) + + _nested_filter_map_data.capacity() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + + retained_set_bytes(_ancestor_null_indices); +} + +template +size_t ScalarColumnReader::active_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->active_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.size() * sizeof(level_t) + + _def_levels.size() * sizeof(level_t) + _null_run_lengths.size() * sizeof(uint16_t) + + _nested_filter_map_data.size() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.size() * sizeof(uint32_t) + + _materialization_state.selection.ranges.size() * sizeof(ParquetSelectionRange) + + _ancestor_null_indices.size() * sizeof(size_t); +} + +#ifdef BE_TEST +template +void ScalarColumnReader::reserve_batch_scratch_for_test( + size_t elements) { + _rep_levels.reserve(elements); + _def_levels.reserve(elements); + _null_run_lengths.reserve(elements); + _nested_filter_map_data.reserve(elements); + _materialization_state.dictionary_indices.reserve(elements); + _materialization_state.selection.ranges.reserve(elements); + _ancestor_null_indices.reserve(elements); +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes_for_test() + const { + return retained_batch_scratch_bytes(); +} +#endif + +template +Status ScalarColumnReader::_skip_values(size_t num_values) { + if (num_values == 0) { + return Status::OK(); + } + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t skipped = 0; + size_t null_size = 0; + size_t nonnull_size = 0; + while (skipped < num_values) { + level_t def_level = -1; + size_t loop_skip = def_decoder.get_next_run(&def_level, num_values - skipped); + if (loop_skip == 0) { + return Status::Corruption("Parquet definition level stream ended while skipping"); + } + if (def_level < _field_schema->definition_level) { + null_size += loop_skip; + } else { + nonnull_size += loop_skip; + } + skipped += loop_skip; + } + if (null_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false)); + } + if (nonnull_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true)); + } + } else { + RETURN_IF_ERROR(_chunk_reader->skip_values(num_values)); + } + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_values(size_t num_values, + ColumnPtr& doris_column, + const DataTypePtr& type, + FilterMap& filter_map, + bool is_dict_filter) { + if (num_values == 0) { + return Status::OK(); + } + MutableColumnPtr data_column; + _null_run_lengths.clear(); + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level; + size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while materializing"); + } + + bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + prev_is_null = is_null; + has_read += loop_read; + } + } + } else { + if (_chunk_reader->max_def_level() > 0) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (_null_run_lengths.empty()) { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + } + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, map_data_column, + &filter_map, _filter_map_index)); + _filter_map_index += num_values; + } + DORIS_CHECK(_serde != nullptr); + // Keep selected-row cardinality stable: non-strict conversion failures append a nested + // default and mark this matching nullable output row instead of shortening the column. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = + conversion_failure_map(*_field_schema, type, _materialization_state.enable_strict_mode, + map_data_column, &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + return status; +} + +/** + * Load the nested column data of complex type. + * A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that + * whether the reader should read the remaining value of the last row in previous page. + */ +template +Status ScalarColumnReader::_read_nested_column( + ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter) { + _rep_levels.clear(); + _def_levels.clear(); + + // Handle nullable columns + MutableColumnPtr data_column; + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + _nested_filter_map_data.clear(); + + auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + if (filter_map.has_filter()) { + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + } else { + RETURN_IF_ERROR(_nested_filter_map.init( + nullptr, _rep_levels.size() - before_rep_level_sz, false)); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), + _null_run_lengths, _ancestor_null_indices)); + + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init( + _null_run_lengths, + _rep_levels.size() - before_rep_level_sz - _ancestor_null_indices.size(), + map_data_column, &_nested_filter_map, 0, &_ancestor_null_indices)); + } + + DORIS_CHECK(_serde != nullptr); + // Nested materialization must preserve the same value/null-map row alignment invariant. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = conversion_failure_map( + *_field_schema, type, _materialization_state.enable_strict_mode, map_data_column, + &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values( + data_column, *_serde, _decode_context, _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + RETURN_IF_ERROR(status); + if (!_ancestor_null_indices.empty()) { + RETURN_IF_ERROR(_chunk_reader->skip_values(_ancestor_null_indices.size(), false)); + } + if (filter_map.has_filter()) { + auto new_rep_sz = before_rep_level_sz; + for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) { + if (_nested_filter_map_data[idx - before_rep_level_sz]) { + _rep_levels[new_rep_sz] = _rep_levels[idx]; + _def_levels[new_rep_sz] = _def_levels[idx]; + new_rep_sz++; + } + } + _rep_levels.resize(new_rep_sz); + _def_levels.resize(new_rep_sz); + } + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + size_t right_row = std::min(left_row + batch_size - *read_rows, + (size_t)_row_ranges.get_range_to(_current_range_idx)); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + size_t load_rows = 0; + bool cross_page = false; + size_t before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &load_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && load_rows == 0 && !cross_page)) { + // A bounded V2/indexed page must advance at least one logical row; zero progress would + // leave both range cursors unchanged and spin forever on corrupt repetition levels. + return Status::Corruption("Parquet nested reader made no row progress"); + } + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); + _filter_map_index += load_rows; + while (cross_page) { + before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1)); + } + *read_rows += load_rows; + _current_row_index += load_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_fixed_width_filter_values( + size_t num_values, const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + IColumn* projected_column, IColumn::Filter* row_filter) { + DORIS_CHECK(row_filter != nullptr); + _null_run_lengths.clear(); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level = -1; + const size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while filtering fixed-width values"); + } + const bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + prev_is_null = is_null; + has_read += loop_read; + } + } else { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + } + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, nullptr, &filter_map, + _filter_map_index)); + _filter_map_index += num_values; + bool used_filter = false; + RETURN_IF_ERROR(_chunk_reader->filter_fixed_width_values( + conjuncts, column_id, _select_vector, &_fixed_width_predicate_nulls, + &_fixed_width_predicate_matches, projected_column, row_filter, &used_filter)); + // Chunk encodings are prevalidated before any definition level is consumed, so a false result + // here would make a materializing fallback observe an advanced level cursor. + DORIS_CHECK(used_filter); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_fixed_width_filter( + const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, size_t batch_size, + IColumn* projected_column, IColumn::Filter* row_filter, size_t* read_rows, bool* eof, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + if (_in_nested || conjuncts.empty()) { + return Status::OK(); + } + if (!std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + })) { + return Status::OK(); + } + // Validate every advertised value encoding before touching either cursor. A late fallback + // cannot rewind definition levels or a previously decoded fixed-width page. + const bool supported_encodings = std::ranges::all_of( + _chunk_meta.meta_data.encodings, [&](const tparquet::Encoding::type encoding) { + return ColumnChunkReader:: + supports_raw_fixed_filter_encoding(encoding, + _chunk_meta.meta_data.type) || + encoding == tparquet::Encoding::RLE || + encoding == tparquet::Encoding::BIT_PACKED; + }); + if (!supported_encodings) { + return Status::OK(); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + _current_row_index = right_row; + } else { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + if (!_chunk_reader->can_filter_fixed_width_values(conjuncts, column_id)) { + return Status::OK(); + } + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); ++idx) { + const auto range = read_ranges.get_range(idx); + const size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + const size_t values = + std::min(static_cast(range.to() - range.from()), batch_size - has_read); + IColumn::Filter fragment_filter; + RETURN_IF_ERROR(_read_fixed_width_filter_values( + values, conjuncts, column_id, filter_map, projected_column, &fragment_filter)); + row_filter->insert(row_filter->end(), fragment_filter.begin(), fragment_filter.end()); + has_read += values; + *read_rows += values; + _current_row_index += values; + if (has_read == batch_size) { + break; + } + } + } + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + *used_filter = true; + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_levels(FilterMap& filter_map, + size_t batch_size, + size_t* read_rows, + bool* eof) { + DORIS_CHECK(_in_nested); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + _rep_levels.clear(); + _def_levels.clear(); + *read_rows = 0; + + auto consume_level_segment = [&](size_t level_start, size_t filter_map_index) -> Status { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + // Advance the encoded value stream without constructing a temporary Doris column. The + // definition levels identify the physical values that actually exist; dictionary skips + // still validate every index. + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(_def_levels, level_start)); + if (!filter_map.has_filter()) { + return Status::OK(); + } + + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, level_start, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + size_t write_index = level_start; + for (size_t read_index = level_start; read_index < _rep_levels.size(); ++read_index) { + if (_nested_filter_map_data[read_index - level_start] != 0) { + _rep_levels[write_index] = _rep_levels[read_index]; + _def_levels[write_index] = _def_levels[read_index]; + ++write_index; + } + } + _rep_levels.resize(write_index); + _def_levels.resize(write_index); + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + const size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + const size_t right_row = + std::min(left_row + batch_size - *read_rows, + static_cast(_row_ranges.get_range_to(_current_range_idx))); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + + size_t loaded_rows = 0; + bool cross_page = false; + size_t level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &loaded_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && loaded_rows == 0 && !cross_page)) { + // Keep the levels-only path under the same forward-progress invariant as value reads. + return Status::Corruption("Parquet nested level reader made no row progress"); + } + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index)); + _filter_map_index += loaded_rows; + while (cross_page) { + level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index - 1)); + } + + *read_rows += loaded_rows; + _current_row_index += loaded_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Result +ScalarColumnReader::materialize_dictionary_values( + const ColumnInt32* dict_column, const DataTypePtr& target_type) { + DORIS_CHECK(dict_column != nullptr); + DORIS_CHECK(target_type != nullptr); + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + DORIS_CHECK(dictionary_decoder != nullptr); + // Materialize the dictionary in the file-local logical type once. Both predicate evaluation + // and matched-ID gathering must observe the same decimal scale, timestamp unit, and binary + // interpretation as ordinary data-page decoding. + const DataTypePtr dictionary_type = remove_nullable(target_type); + const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); + // The probe is the first typed read for this reader. Publish its SerDe identity now so the + // first dictionary-ID batch does not mistake initialization for a type change and drop cache. + _serde_type = dictionary_type.get(); + _serde = dictionary_serde; + if (_materialization_state.dictionary_generation != + dictionary_decoder->dictionary_generation()) { + _materialization_state.typed_dictionary = dictionary_type->create_column(); + ParquetDecodeContext dictionary_context = _decode_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + dictionary_context.dictionary_index_only = false; + auto status = dictionary_serde->read_parquet_dictionary( + *_materialization_state.typed_dictionary, *dictionary_decoder, dictionary_context); + if (!status.ok()) { + return ResultError(std::move(status)); + } + DORIS_CHECK_EQ(_materialization_state.typed_dictionary->size(), + dictionary_decoder->dictionary_size()); + _materialization_state.dictionary_generation = dictionary_decoder->dictionary_generation(); +#ifdef BE_TEST + ++_dictionary_materialization_count; +#endif + } + + auto result = _materialization_state.typed_dictionary->clone_empty(); + const auto& source_indices = dict_column->get_data(); + auto& indices = _materialization_state.dictionary_indices; + indices.resize(source_indices.size()); + for (size_t row = 0; row < source_indices.size(); ++row) { + if (UNLIKELY(source_indices[row] < 0 || + static_cast(source_indices[row]) >= + _materialization_state.typed_dictionary->size())) { + return ResultError(Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + source_indices[row], row, _materialization_state.typed_dictionary->size())); + } + indices[row] = static_cast(source_indices[row]); + } + result->insert_indices_from(*_materialization_state.typed_dictionary, indices.data(), + indices.data() + indices.size()); + return result; +} + +template +Result ScalarColumnReader::dictionary_values( + const DataTypePtr& target_type) { + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + if (dictionary_decoder == nullptr || dictionary_decoder->dictionary_size() == 0) { + return ResultError(Status::NotSupported("Parquet column has no reusable dictionary")); + } + auto ids = ColumnInt32::create(); + auto& data = ids->get_data(); + data.resize(dictionary_decoder->dictionary_size()); + for (size_t dictionary_id = 0; dictionary_id < data.size(); ++dictionary_id) { + data[dictionary_id] = cast_set(dictionary_id); + } + // Materialize the typed dictionary once and keep it in _materialization_state. Later row-level + // filtering decodes only ids and flattens surviving values from this same dictionary. + return materialize_dictionary_values(ids.get(), target_type); +} + +template +Status ScalarColumnReader::_try_load_dict_page(bool* loaded, + bool* has_dict) { + // _chunk_reader init will load first page header to check whether has dict page + *loaded = true; + *has_dict = _chunk_reader->has_dict(); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + const DataTypePtr materialization_type = remove_nullable(type); + const DataTypePtr file_type = remove_nullable(_field_schema->data_type); + if (!is_dict_filter && !file_type->equals(*materialization_type)) { + // File-to-table casts belong to ColumnMapper. Reaching this reader with a table type would + // also evaluate file-local predicates against values produced under the wrong type. + return Status::InternalError( + "Native Parquet reader type mismatch for file column '{}': file={}, requested={}", + _field_schema->name, file_type->get_name(), materialization_type->get_name()); + } + + const DataTypePtr serde_type = is_dict_filter ? file_type : materialization_type; + const bool serde_type_changed = _serde_type != serde_type.get(); + if (serde_type_changed || _dictionary_index_only != is_dict_filter) { + _serde_type = serde_type.get(); + _serde = serde_type->get_serde(); + _dictionary_index_only = is_dict_filter; + // Switching between typed values and dictionary IDs does not invalidate the dictionary + // materialized by the pruning probe. Preserve it unless the logical SerDe type changed. + if (serde_type_changed) { + _materialization_state.reset_dictionary(); + } + } + _decode_context.dictionary_index_only = is_dict_filter; + + _def_levels.clear(); + _rep_levels.clear(); + *read_rows = 0; + + if (_in_nested) { + return _read_nested_column(doris_column, materialization_type, filter_map, batch_size, + read_rows, eof, is_dict_filter); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + + do { + // generate the row ranges that should be read + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + // skip the whole page + _current_row_index = right_row; + } else { + bool skip_whole_batch = false; + // Determining whether to skip page or batch will increase the calculation time. + // When the filtering effect is greater than 60%, it is possible to skip the page or batch. + if (filter_map.has_filter() && filter_map.filter_ratio() > 0.6) { + // lazy read + size_t remaining_num_values = read_ranges.count(); + if (batch_size >= remaining_num_values && + filter_map.can_filter_all(remaining_num_values, _filter_map_index)) { + // We can skip the whole page if the remaining values are filtered by predicate columns + _filter_map_index += remaining_num_values; + _current_row_index = right_row; + *read_rows = remaining_num_values; + break; + } + skip_whole_batch = batch_size <= remaining_num_values && + filter_map.can_filter_all(batch_size, _filter_map_index); + if (skip_whole_batch) { + _filter_map_index += batch_size; + } + } + // load page data to decode or skip values + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); idx++) { + auto range = read_ranges.get_range(idx); + // generate the skipped values + size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + // generate the read values + size_t read_values = + std::min((size_t)(range.to() - range.from()), batch_size - has_read); + if (skip_whole_batch) { + RETURN_IF_ERROR(_skip_values(read_values)); + } else { + RETURN_IF_ERROR(_read_values(read_values, doris_column, materialization_type, + filter_map, is_dict_filter)); + } + has_read += read_values; + *read_rows += read_values; + _current_row_index += read_values; + if (has_read == batch_size) { + break; + } + } + } + } while (false); + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + + return Status::OK(); +} + +Status ArrayColumnReader::init(std::unique_ptr element_reader, + NativeFieldSchema* field) { + _field_schema = field; + _element_reader = std::move(element_reader); + return Status::OK(); +} + +Status ArrayColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_ARRAY) { + return Status::Corruption( + "Wrong data type for column '{}', expected Array type, actual type: {}.", + _field_schema->name, type->get_name()); + } + + ColumnPtr& element_column = assert_cast(*data_column).get_data_ptr(); + const DataTypePtr& element_type = + (assert_cast(remove_nullable(type).get()))->get_nested_type(); + // read nested column + RETURN_IF_ERROR(_element_reader->read_column_data(element_column, element_type, + root_node->element(), filter_map, batch_size, + read_rows, eof, is_dict_filter)); + if (*read_rows == 0) { + return Status::OK(); + } + + ColumnArray::Offsets64& offsets_data = assert_cast(*data_column).get_offsets(); + // fill offset and null map + RETURN_IF_ERROR(fill_array_offset(_field_schema, offsets_data, null_map_ptr, + _element_reader->get_rep_level(), + _element_reader->get_def_level())); + if (UNLIKELY(element_column->size() != offsets_data.back())) { + return Status::Corruption("Parquet array element count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field) { + _field_schema = field; + _key_reader = std::move(key_reader); + _value_reader = std::move(value_reader); + return Status::OK(); +} + +Status MapColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_MAP) { + return Status::Corruption( + "Wrong data type for column '{}', expected Map type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& map = assert_cast(*data_column); + const DataTypePtr& key_type = + assert_cast(remove_nullable(type).get())->get_key_type(); + const DataTypePtr& value_type = + assert_cast(remove_nullable(type).get())->get_value_type(); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + + size_t key_rows = 0; + size_t value_rows = 0; + bool key_eof = false; + bool value_eof = false; + int64_t orig_col_column_size = key_column->size(); + + RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->key(), + filter_map, batch_size, &key_rows, &key_eof, + is_dict_filter)); + + while (value_rows < key_rows && !value_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_value_reader->read_column_data( + value_column, value_type, root_node->value(), filter_map, key_rows - value_rows, + &loop_rows, &value_eof, is_dict_filter, key_column->size() - orig_col_column_size)); + value_rows += loop_rows; + } + if (UNLIKELY(key_rows != value_rows)) { + // MAP children share one logical-row boundary; EOF in one sibling is file corruption. + return Status::Corruption("Parquet map value reader returned {} rows for {} key rows", + value_rows, key_rows); + } + *read_rows = key_rows; + *eof = key_eof; + + if (*read_rows == 0) { + return Status::OK(); + } + + const auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + // MAP siblings may have child-local collections and nullness. At this boundary, + // only the outer entry distribution and whether that entry exists must agree. + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + if (UNLIKELY(_key_reader->get_rep_level().size() != _key_reader->get_def_level().size() || + _value_reader->get_rep_level().size() != _value_reader->get_def_level().size())) { + return Status::Corruption("Parquet map sibling level counts differ"); + } + if (UNLIKELY(parent_shape(*_key_reader) != parent_shape(*_value_reader))) { + return Status::Corruption("Parquet map key/value outer entry shapes differ"); + } + + if (UNLIKELY(key_column->size() != value_column->size())) { + return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", + key_column->size(), value_column->size()); + } + // Non-strict conversion can persist nullable Doris MAP keys, so the native reader must + // preserve writer output instead of reclassifying an otherwise valid file as corrupt. + const auto& key_rep_levels = _key_reader->get_rep_level(); + // fill offset and null map + // The key leaf is the canonical outer MAP shape. A nested value has additional repetition + // levels for its own collections, so comparing the two leaves would reject valid MAP values. + RETURN_IF_ERROR(fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, + key_rep_levels, _key_reader->get_def_level())); + if (UNLIKELY(key_column->size() != map.get_offsets().back())) { + return Status::Corruption("Parquet map entry count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + DORIS_CHECK(dynamic_cast(_key_reader.get()) == nullptr); + return _key_reader->read_column_levels(filter_map, batch_size, read_rows, eof); +} + +Status StructColumnReader::init( + std::unordered_map>&& child_readers, + NativeFieldSchema* field) { + _field_schema = field; + _child_readers = std::move(child_readers); + return Status::OK(); +} + +Status StructColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + _read_column_names.clear(); + for (const auto& child : _field_schema->children) { + auto reader = _child_readers.find(child.name); + DORIS_CHECK(reader != _child_readers.end()); + if (dynamic_cast(reader->second.get()) != nullptr) { + continue; + } + _read_column_names.emplace_back(child.name); + return reader->second->read_column_levels(filter_map, batch_size, read_rows, eof); + } + return Status::InternalError("Struct {} has no physical reader for levels", + _field_schema->name); +} +Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_STRUCT) { + return Status::Corruption( + "Wrong data type for column '{}', expected Struct type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& doris_struct = assert_cast(*data_column); + const auto* doris_struct_type = assert_cast(remove_nullable(type).get()); + + int64_t not_missing_column_id = -1; + size_t not_missing_orig_column_size = 0; + std::vector missing_column_idxs {}; + std::vector skip_reading_column_idxs {}; + std::vector> reference_parent_shape; + + auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + // Deeper repetitions belong to a nested child; only starts visible at this STRUCT's + // boundary and the optional parent's presence determine how siblings are paired. + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + + _read_column_names.clear(); + + for (size_t i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& doris_field = doris_struct.get_column_ptr(i); + auto& doris_type = doris_struct_type->get_element(i); + auto& doris_name = doris_struct_type->get_element_name(i); + if (!root_node->has_child(doris_name)) { + missing_column_idxs.push_back(i); + VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i + << "], doris_name: " << doris_name << " (column not exists in root node)"; + continue; + } + auto file_name = root_node->file_child_name(doris_name); + + // Check if this is a SkipReadingReader - we should skip it when choosing reference column + // because SkipReadingReader doesn't know the actual data size in nested context + bool is_skip_reader = + dynamic_cast(_child_readers[file_name].get()) != nullptr; + + if (is_skip_reader) { + // Store SkipReadingReader columns to fill them later based on reference column size + skip_reading_column_idxs.push_back(i); + continue; + } + + // Only add non-SkipReadingReader columns to _read_column_names + // This ensures get_rep_level() and get_def_level() return valid levels + _read_column_names.emplace_back(file_name); + + size_t field_rows = 0; + bool field_eof = false; + if (not_missing_column_id == -1) { + not_missing_column_id = i; + not_missing_orig_column_size = doris_field->size(); + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, batch_size, + &field_rows, &field_eof, is_dict_filter)); + *read_rows = field_rows; + *eof = field_eof; + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + reference_parent_shape = parent_shape(*_child_readers[file_name]); + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column + * (since `_read_nested_column` is not called repeatedly). + * + * It is worth mentioning that, theoretically, any sub-column can be chosen to fill the null_map, + * and selecting the shortest one will offer better performance + */ + } else { + while (field_rows < *read_rows && !field_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + *read_rows - field_rows, &loop_rows, &field_eof, is_dict_filter)); + field_rows += loop_rows; + } + if (UNLIKELY(*read_rows != field_rows)) { + // STRUCT siblings must advance the same logical rows before any result is exposed. + return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", + file_name, field_rows, *read_rows); + } + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + if (UNLIKELY(parent_shape(*_child_readers[file_name]) != reference_parent_shape)) { + return Status::Corruption( + "Parquet struct child '{}' has a different repeated-parent shape", + file_name); + } + // DCHECK_EQ(*eof, field_eof); + } + } + + int64_t missing_column_sz = -1; + + if (not_missing_column_id == -1) { + // All queried columns are missing in the file (e.g., all added after schema change) + // We need to pick a column from _field_schema children that exists in the file for RL/DL reference + std::string reference_file_column_name; + std::unique_ptr* reference_reader = nullptr; + + for (const auto& child : _field_schema->children) { + auto it = _child_readers.find(child.name); + if (it != _child_readers.end()) { + // Skip SkipReadingReader as they don't have valid RL/DL + bool is_skip_reader = dynamic_cast(it->second.get()) != nullptr; + if (!is_skip_reader) { + reference_file_column_name = child.name; + reference_reader = &(it->second); + break; + } + } + } + + if (reference_reader != nullptr) { + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR( + (*reference_reader) + ->read_column_levels(filter_map, batch_size, &field_rows, &field_eof)); + + *read_rows = field_rows; + *eof = field_eof; + _read_column_names.emplace_back(reference_file_column_name); + missing_column_sz = 0; + const auto& rep_levels = (*reference_reader)->get_rep_level(); + const auto& def_levels = (*reference_reader)->get_def_level(); + DORIS_CHECK_EQ(rep_levels.size(), def_levels.size()); + for (size_t level_index = 0; level_index < def_levels.size(); ++level_index) { + if (def_levels[level_index] >= _field_schema->repeated_parent_def_level && + rep_levels[level_index] <= _field_schema->repetition_level) { + ++missing_column_sz; + } + } + } else { + return Status::Corruption( + "Cannot read struct '{}': all queried columns are missing and no reference " + "column found in file", + _field_schema->name); + } + } + + // This missing_column_sz is not *read_rows. Because read_rows returns the number of rows. + // For example: suppose we have a column array>, + // where b is a newly added column, that is, a missing column. + // There are two rows of data in this column, + // [{1,null},{2,null},{3,null}] + // [{4,null},{5,null}] + // When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2. + // You should insert 5 records into subcolumn b instead of 2. + if (missing_column_sz == -1) { + missing_column_sz = doris_struct.get_column(not_missing_column_id).size() - + not_missing_orig_column_size; + } + + // Fill SkipReadingReader columns with the correct amount of data based on reference column + // Let SkipReadingReader handle the data filling through its read_column_data method + for (auto idx : skip_reading_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = const_cast(doris_struct_type->get_element(idx)); + auto& doris_name = const_cast(doris_struct_type->get_element_name(idx)); + auto file_name = root_node->file_child_name(doris_name); + + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz)); + } + + // Fill truly missing columns (not in root_node) with null or default value + for (auto idx : missing_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = doris_struct_type->get_element(idx); + DCHECK(doris_type->is_nullable()); + doris_field = IColumn::mutate(std::move(doris_field)); + auto mutable_column = doris_field->assert_mutable(); + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); + } + + if (null_map_ptr != nullptr) { + fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(), + this->get_def_level()); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h new file mode 100644 index 00000000000000..6ea2787f9f0211 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -0,0 +1,662 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/common.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "storage/segment/row_ranges.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { +using ::doris::ColumnString; +using segment_v2::RowRange; +using segment_v2::RowRanges; + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context); +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode); +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row); +#endif + +class ColumnReader { +public: + struct ColumnStatistics { + ColumnStatistics() + : page_index_read_calls(0), + decompress_time(0), + decompress_cnt(0), + decode_header_time(0), + decode_value_time(0), + materialization_time(0), + hybrid_selection_batches(0), + hybrid_selection_ranges(0), + hybrid_selection_null_fallback_batches(0), + decode_dict_time(0), + decode_level_time(0), + decode_null_map_time(0), + skip_page_header_num(0), + parse_page_header_num(0), + read_page_header_time(0), + page_read_counter(0), + page_cache_write_counter(0), + page_cache_compressed_write_counter(0), + page_cache_decompressed_write_counter(0), + page_cache_hit_counter(0), + page_cache_missing_counter(0), + page_cache_compressed_hit_counter(0), + page_cache_decompressed_hit_counter(0) {} + + ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time) + : page_index_read_calls(0), + decompress_time(cs.decompress_time), + decompress_cnt(cs.decompress_cnt), + decode_header_time(cs.decode_header_time), + decode_value_time(cs.decode_value_time), + materialization_time(cs.materialization_time), + hybrid_selection_batches(cs.hybrid_selection_batches), + hybrid_selection_ranges(cs.hybrid_selection_ranges), + hybrid_selection_null_fallback_batches(cs.hybrid_selection_null_fallback_batches), + decode_dict_time(cs.decode_dict_time), + decode_level_time(cs.decode_level_time), + decode_null_map_time(null_map_time), + skip_page_header_num(cs.skip_page_header_num), + parse_page_header_num(cs.parse_page_header_num), + read_page_header_time(cs.read_page_header_time), + page_read_counter(cs.page_read_counter), + page_cache_write_counter(cs.page_cache_write_counter), + page_cache_compressed_write_counter(cs.page_cache_compressed_write_counter), + page_cache_decompressed_write_counter(cs.page_cache_decompressed_write_counter), + page_cache_hit_counter(cs.page_cache_hit_counter), + page_cache_missing_counter(cs.page_cache_missing_counter), + page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter), + page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter), + leaf_page_read_counters {cs.data_page_read_counter} {} + + int64_t page_index_read_calls; + int64_t decompress_time; + int64_t decompress_cnt; + int64_t decode_header_time; + int64_t decode_value_time; + int64_t materialization_time; + int64_t hybrid_selection_batches; + int64_t hybrid_selection_ranges; + int64_t hybrid_selection_null_fallback_batches; + int64_t decode_dict_time; + int64_t decode_level_time; + int64_t decode_null_map_time; + int64_t skip_page_header_num; + int64_t parse_page_header_num; + int64_t read_page_header_time; + int64_t page_read_counter; + int64_t page_cache_write_counter; + int64_t page_cache_compressed_write_counter; + int64_t page_cache_decompressed_write_counter; + int64_t page_cache_hit_counter; + int64_t page_cache_missing_counter; + int64_t page_cache_compressed_hit_counter; + int64_t page_cache_decompressed_hit_counter; + // Preserve per-leaf identity when complex readers aggregate their other counters. + std::vector leaf_page_read_counters; + + void merge(ColumnStatistics& col_statistics) { + page_index_read_calls += col_statistics.page_index_read_calls; + decompress_time += col_statistics.decompress_time; + decompress_cnt += col_statistics.decompress_cnt; + decode_header_time += col_statistics.decode_header_time; + decode_value_time += col_statistics.decode_value_time; + materialization_time += col_statistics.materialization_time; + hybrid_selection_batches += col_statistics.hybrid_selection_batches; + hybrid_selection_ranges += col_statistics.hybrid_selection_ranges; + hybrid_selection_null_fallback_batches += + col_statistics.hybrid_selection_null_fallback_batches; + decode_dict_time += col_statistics.decode_dict_time; + decode_level_time += col_statistics.decode_level_time; + decode_null_map_time += col_statistics.decode_null_map_time; + skip_page_header_num += col_statistics.skip_page_header_num; + parse_page_header_num += col_statistics.parse_page_header_num; + read_page_header_time += col_statistics.read_page_header_time; + page_read_counter += col_statistics.page_read_counter; + leaf_page_read_counters.insert(leaf_page_read_counters.end(), + col_statistics.leaf_page_read_counters.begin(), + col_statistics.leaf_page_read_counters.end()); + page_cache_write_counter += col_statistics.page_cache_write_counter; + page_cache_compressed_write_counter += + col_statistics.page_cache_compressed_write_counter; + page_cache_decompressed_write_counter += + col_statistics.page_cache_decompressed_write_counter; + page_cache_hit_counter += col_statistics.page_cache_hit_counter; + page_cache_missing_counter += col_statistics.page_cache_missing_counter; + page_cache_compressed_hit_counter += col_statistics.page_cache_compressed_hit_counter; + page_cache_decompressed_hit_counter += + col_statistics.page_cache_decompressed_hit_counter; + } + }; + + ColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} + virtual ~ColumnReader() = default; + virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter, + int64_t real_column_size = -1) = 0; + + // Evaluate a predicate scalar directly from a supported fixed-width page encoding. The default + // is a non-consuming fallback for nested and synthetic readers. + virtual Status read_fixed_width_filter(const VExprSPtrs&, int, FilterMap&, size_t, IColumn*, + IColumn::Filter* row_filter, size_t* read_rows, + bool* eof, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + return Status::OK(); + } + + // Consume a nested batch while retaining only definition/repetition levels. This is used when + // schema evolution makes every projected STRUCT child synthetic: the parent still needs one + // physical leaf's shape, but decoding that leaf's strings or other payload would be wasted. + virtual Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) = 0; + + virtual Result materialize_dictionary_values(const ColumnInt32* dict_column, + const DataTypePtr& target_type) { + throw Exception( + Status::FatalError("Method materialize_dictionary_values is not supported")); + } + virtual Result dictionary_values(const DataTypePtr& target_type) { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); + } + + static Status create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection = false, + const std::set& column_ids = {}, + const std::set& filter_column_ids = {}, + const std::string& page_cache_file_key = {}, + const ParquetReaderCompat& compat = {}, bool enable_strict_mode = false); + virtual const std::vector& get_rep_level() const = 0; + virtual const std::vector& get_def_level() const = 0; + virtual ColumnStatistics column_statistics() = 0; + virtual void close() = 0; + + // A repeated parent can expand one logical-row batch into millions of leaf values. Keep + // ordinary batch scratch for reuse, but let the top-level adapter release exceptional + // high-water allocations after every parent offset/null-map consumer has finished. + virtual void release_batch_scratch(size_t max_retained_bytes) = 0; + + virtual void reset_filter_map_index() = 0; + + NativeFieldSchema* get_field_schema() const { return _field_schema; } + void set_column_in_nested() { _in_nested = true; } + +protected: + void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const; + + NativeFieldSchema* _field_schema = nullptr; + const RowRanges& _row_ranges; + size_t _total_rows = 0; + const cctz::time_zone* _ctz = nullptr; + io::IOContext* _io_ctx = nullptr; + int64_t _current_row_index = 0; + int64_t _decode_null_map_time = 0; + + size_t _filter_map_index = 0; + std::set _filter_column_ids; + + // _in_nested: column in struct/map/array + // IN_COLLECTION : column in map/array + bool _in_nested = false; +}; + +template +class ScalarColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ScalarColumnReader) +public: + ScalarColumnReader(const RowRanges& row_ranges, size_t total_rows, + const tparquet::ColumnChunk& chunk_meta, + const tparquet::OffsetIndex* offset_index, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx), + _chunk_meta(chunk_meta), + _offset_index(offset_index) {} + ~ScalarColumnReader() override { close(); } + Status init(io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, + RuntimeState* state, const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_fixed_width_filter(const VExprSPtrs& conjuncts, int column_id, + FilterMap& filter_map, size_t batch_size, + IColumn* projected_column, IColumn::Filter* row_filter, + size_t* read_rows, bool* eof, bool* used_filter) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + Result materialize_dictionary_values(const ColumnInt32* dict_column, + const DataTypePtr& target_type) override; + Result dictionary_values(const DataTypePtr& target_type) override; + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { + return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time); + } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override; + +#ifdef BE_TEST + void reserve_batch_scratch_for_test(size_t elements); + size_t retained_batch_scratch_bytes_for_test() const; + size_t dictionary_materialization_count_for_test() const { + return _dictionary_materialization_count; + } +#endif + + void reset_filter_map_index() override { + _filter_map_index = 0; // nested + } + +private: + tparquet::ColumnChunk _chunk_meta; + const tparquet::OffsetIndex* _offset_index = nullptr; + std::unique_ptr _stream_reader; + std::unique_ptr> _chunk_reader; +#ifdef BE_TEST + size_t _dictionary_materialization_count = 0; +#endif + // rep def levels buffer. + std::vector _rep_levels; + std::vector _def_levels; + + size_t _current_range_idx = 0; + + Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx, + std::vector& null_map, + std::unordered_set& ancestor_null_indices) { + size_t has_read = level_start_idx; + null_map.emplace_back(0); + bool prev_is_null = false; + + while (has_read < level_end_idx) { + level_t def_level = _def_levels[has_read++]; + size_t loop_read = 1; + while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) { + has_read++; + loop_read++; + } + + if (def_level < _field_schema->repeated_parent_def_level) { + for (size_t i = 0; i < loop_read; i++) { + ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i); + } + continue; + } + + bool is_null = def_level < _field_schema->definition_level; + + if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { + null_map.back() += loop_read; + } else { + if (!(prev_is_null ^ is_null)) { + null_map.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + null_map.emplace_back(USHRT_MAX); + null_map.emplace_back(0); + remaining -= USHRT_MAX; + } + null_map.emplace_back((u_short)remaining); + prev_is_null = is_null; + } + } + return Status::OK(); + } + + Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx, + size_t level_end_idx, std::vector& nested_filter_map_data, + FilterMap* nested_filter_map) { + DORIS_CHECK(nested_filter_map != nullptr); + nested_filter_map_data.resize(level_end_idx - level_start_idx); + for (size_t idx = level_start_idx; idx < level_end_idx; idx++) { + if (idx != level_start_idx && _rep_levels[idx] == 0) { + filter_loc++; + } + nested_filter_map_data[idx - level_start_idx] = + filter_map.filter_map_data()[filter_loc]; + } + + return nested_filter_map->init(nested_filter_map_data.data(), nested_filter_map_data.size(), + false); + } + + DataTypeSerDeSPtr _serde; + const IDataType* _serde_type = nullptr; + ParquetDecodeContext _decode_context; + ParquetMaterializationState _materialization_state; + bool _dictionary_index_only = false; + // Normal-size batch scratch is retained by the persistent leaf reader. Oversized allocations + // are released only after the top-level parent has consumed this leaf's level plan. + std::vector _null_run_lengths; + std::unordered_set _ancestor_null_indices; + std::vector _nested_filter_map_data; + NullMap _fixed_width_predicate_nulls; + IColumn::Filter _fixed_width_predicate_matches; + FilterMap _nested_filter_map; + ColumnSelectVector _select_vector; + uint8_t _oversized_scratch_idle_batches = 0; + + size_t retained_batch_scratch_bytes() const; + size_t active_batch_scratch_bytes() const; + + Status _skip_values(size_t num_values); + Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, bool is_dict_filter); + Status _read_fixed_width_filter_values(size_t num_values, const VExprSPtrs& conjuncts, + int column_id, FilterMap& filter_map, + IColumn* projected_column, IColumn::Filter* row_filter); + Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter); + Status _try_load_dict_page(bool* loaded, bool* has_dict); +}; + +class ArrayColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ArrayColumnReader) +public: + ArrayColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~ArrayColumnReader() override { close(); } + Status init(std::unique_ptr element_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override { + return _element_reader->read_column_levels(filter_map, batch_size, read_rows, eof); + } + const std::vector& get_rep_level() const override { + return _element_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _element_reader->get_def_level(); + } + ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _element_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); } + +private: + std::unique_ptr _element_reader; +}; + +class MapColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(MapColumnReader) +public: + MapColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~MapColumnReader() override { close(); } + + Status init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + return _key_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _key_reader->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics kst = _key_reader->column_statistics(); + ColumnStatistics vst = _value_reader->column_statistics(); + kst.merge(vst); + return kst; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _key_reader->release_batch_scratch(max_retained_bytes); + _value_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { + _key_reader->reset_filter_map_index(); + _value_reader->reset_filter_map_index(); + } + +private: + std::unique_ptr _key_reader; + std::unique_ptr _value_reader; +}; + +class StructColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(StructColumnReader) +public: + StructColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~StructColumnReader() override { close(); } + + Status init(std::unordered_map>&& child_readers, + NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + if (!_read_column_names.empty()) { + // can't use _child_readers[*_read_column_names.begin()] + // because the operator[] of std::unordered_map is not const :( + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column, + * that is `_read_column_names.front()`. + */ + return _child_readers.find(_read_column_names.front())->second->get_rep_level(); + } + return _child_readers.begin()->second->get_rep_level(); + } + + const std::vector& get_def_level() const override { + if (!_read_column_names.empty()) { + return _child_readers.find(_read_column_names.front())->second->get_def_level(); + } + return _child_readers.begin()->second->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics st; + for (const auto& column_name : _read_column_names) { + auto reader = _child_readers.find(column_name); + if (reader != _child_readers.end()) { + ColumnStatistics cst = reader->second->column_statistics(); + st.merge(cst); + } + } + return st; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + for (const auto& reader : _child_readers) { + reader.second->release_batch_scratch(max_retained_bytes); + } + } + + void reset_filter_map_index() override { + for (const auto& reader : _child_readers) { + reader.second->reset_filter_map_index(); + } + } + +private: + std::unordered_map> _child_readers; + std::vector _read_column_names; + //Need to use vector instead of set,see `get_rep_level()` for the reason. +}; + +// A special reader that skips actual reading but provides empty data with correct structure +// This is used when a column is not needed but its structure is required (e.g., for map keys) +class SkipReadingReader : public ColumnReader { +public: + SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx, NativeFieldSchema* field_schema) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) { + _field_schema = field_schema; // Use inherited member from base class + VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: " + << _field_schema->name; + } + + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override { + VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: " + << _field_schema->name << ", batch_size: " << batch_size; + DCHECK(real_column_size >= 0); // real_column_size for filtered column size. + + // Simulate reading without actually reading data + // Fill with default/null values based on column type + doris_column = IColumn::mutate(std::move(doris_column)); + MutableColumnPtr data_column = doris_column->assert_mutable(); + + if (real_column_size > 0) { + if (is_column_nullable(*doris_column)) { + auto* nullable_column = static_cast(data_column.get()); + nullable_column->insert_many_defaults(real_column_size); + } else { + // For non-nullable columns, insert appropriate default values + for (size_t i = 0; i < real_column_size; ++i) { + data_column->insert_default(); + } + } + } + + *read_rows = batch_size; // Indicate we "read" batch_size rows + *eof = false; // We can always provide more empty data + + VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size + << " default values for field: " << _field_schema->name; + + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t*, bool*) override { + return Status::InternalError("Skip reader cannot provide Parquet levels for field {}", + _field_schema->name); + } + + static std::unique_ptr create_unique(const RowRanges& row_ranges, + size_t total_rows, cctz::time_zone* ctz, + io::IOContext* io_ctx, + NativeFieldSchema* field_schema) { + return std::make_unique(row_ranges, total_rows, ctz, io_ctx, + field_schema); + } + + // These methods should not be called for SkipReadingReader + // If they are called, it indicates a logic error in the code + const std::vector& get_rep_level() const override { + LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + const std::vector& get_def_level() const override { + LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + // Implement required pure virtual methods from base class + ColumnStatistics column_statistics() override { + return ColumnStatistics(); // Return empty statistics + } + + void close() override { + // Nothing to close for skip reading + } + + void release_batch_scratch(size_t) override {} + + void reset_filter_map_index() override { _filter_map_index = 0; } +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.cpp b/be/src/format_v2/parquet/reader/native/common.cpp new file mode 100644 index 00000000000000..f048d2b878ebee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/common.h" + +#include + +#include "core/types.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet::native { + +Status FilterMap::init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all) { + _filter_all = filter_all; + _filter_map_data = filter_map_data; + _filter_map_size = filter_map_size; + if (filter_all) { + _has_filter = true; + _filter_ratio = 1; + } else if (filter_map_data == nullptr) { + _has_filter = false; + _filter_ratio = 0; + } else { + const size_t filter_count = simd::count_zero_num( + reinterpret_cast(filter_map_data), filter_map_size); + if (filter_count == filter_map_size) { + _has_filter = true; + _filter_all = true; + _filter_ratio = 1; + } else if (filter_count > 0 && filter_map_size > 0) { + _has_filter = true; + _filter_ratio = static_cast(filter_count) / filter_map_size; + } else { + _has_filter = false; + _filter_ratio = 0; + } + } + return Status::OK(); +} + +bool FilterMap::can_filter_all(size_t remaining_num_values, size_t filter_map_index) { + if (!_has_filter) { + return false; + } + if (_filter_all) { + DCHECK_LE(remaining_num_values + filter_map_index, _filter_map_size); + return true; + } + if (remaining_num_values + filter_map_index > _filter_map_size) { + return false; + } + return simd::count_zero_num( + reinterpret_cast(_filter_map_data + filter_map_index), + remaining_num_values) == remaining_num_values; +} + +Status FilterMap::generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index) const { + if (!has_filter() || filter_all()) { + return Status::InternalError( + "Native FilterMap requires a partial filter: has_filter={}, filter_all={}", + has_filter(), filter_all()); + } + if (rep_levels.empty()) { + return Status::OK(); + } + + nested_filter_map_data.resize(rep_levels.size()); + size_t current_row = current_row_ptr != nullptr ? *current_row_ptr : 0; + for (size_t i = start_index; i < rep_levels.size(); ++i) { + if (i != start_index && rep_levels[i] == 0) { + ++current_row; + if (current_row >= _filter_map_size) { + return Status::InvalidArgument("Nested filter row {} exceeds filter map size {}", + current_row, _filter_map_size); + } + } + nested_filter_map_data[i] = _filter_map_data[current_row]; + } + if (current_row_ptr != nullptr) { + *current_row_ptr = current_row; + } + + auto new_filter = std::make_unique(); + RETURN_IF_ERROR( + new_filter->init(nested_filter_map_data.data(), nested_filter_map_data.size(), false)); + *nested_filter_map = std::move(new_filter); + return Status::OK(); +} + +Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices) { + _num_values = num_values; + _num_nulls = 0; + _read_index = 0; + size_t map_index = 0; + bool is_null = false; + _has_filter = filter_map->has_filter(); + + if (_has_filter) { + _data_map.resize(num_values); + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_NULL; + } + } else { + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_CONTENT; + } + } + is_null = !is_null; + } + + size_t num_read = 0; + size_t source_index = 0; + size_t valid_count = 0; + while (valid_count < num_values) { + DCHECK_LT(filter_map_index + source_index, filter_map->filter_map_size()); + if (skipped_indices != nullptr && + skipped_indices->contains(filter_map_index + source_index)) { + ++source_index; + continue; + } + if (filter_map->filter_map_data()[filter_map_index + source_index] != 0) { + _data_map[valid_count] = + _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; + ++num_read; + } + ++valid_count; + ++source_index; + } + _num_filtered = num_values - num_read; + + if (null_map != nullptr && num_read > 0) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_read); + if (_num_nulls == 0) { + memset(null_map->data() + null_map_index, 0, num_read); + } else if (_num_nulls == num_values) { + memset(null_map->data() + null_map_index, 1, num_read); + } else { + for (size_t i = 0; i < num_values; ++i) { + if (_data_map[i] == CONTENT) { + (*null_map)[null_map_index++] = static_cast(false); + } else if (_data_map[i] == NULL_DATA) { + (*null_map)[null_map_index++] = static_cast(true); + } + } + } + } + } else { + _num_filtered = 0; + _run_length_null_map = &run_length_null_map; + if (null_map != nullptr) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_values); + for (const auto run_length : run_length_null_map) { + memset(null_map->data() + null_map_index, is_null ? 1 : 0, run_length); + null_map_index += run_length; + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } else { + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.h b/be/src/format_v2/parquet/reader/native/common.h new file mode 100644 index 00000000000000..4ad204da2ff30d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" + +namespace doris::format::parquet::native { + +// These cursor types belong to the native v2 decoder so its execution path never reaches into +// the legacy Parquet reader for filtering or level materialization. +using level_t = int16_t; + +class FilterMap { +public: + Status init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all); + + Status generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index = 0) const; + + const uint8_t* filter_map_data() const { return _filter_map_data; } + size_t filter_map_size() const { return _filter_map_size; } + bool has_filter() const { return _has_filter; } + bool filter_all() const { return _filter_all; } + double filter_ratio() const { return _has_filter ? _filter_ratio : 0; } + + bool can_filter_all(size_t remaining_num_values, size_t filter_map_index); + +private: + bool _filter_all = false; + bool _has_filter = false; + const uint8_t* _filter_map_data = nullptr; + size_t _filter_map_size = 0; + double _filter_ratio = 0; +}; + +class ColumnSelectVector { +public: + enum DataReadType : uint8_t { CONTENT = 0, NULL_DATA, FILTERED_CONTENT, FILTERED_NULL }; + + Status init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices = nullptr); + + size_t num_values() const { return _num_values; } + size_t num_nulls() const { return _num_nulls; } + size_t num_filtered() const { return _num_filtered; } + bool has_filter() const { return _has_filter; } + + template + size_t get_next_run(DataReadType* data_read_type) { + DCHECK_EQ(_has_filter, has_filter); + if constexpr (has_filter) { + if (_read_index == _num_values) { + return 0; + } + const DataReadType type = _data_map[_read_index++]; + size_t run_length = 1; + while (_read_index < _num_values && _data_map[_read_index] == type) { + ++run_length; + ++_read_index; + } + *data_read_type = type; + return run_length; + } + + size_t run_length = 0; + while (run_length == 0) { + if (_read_index == _run_length_null_map->size()) { + return 0; + } + *data_read_type = _read_index % 2 == 0 ? CONTENT : NULL_DATA; + run_length = (*_run_length_null_map)[_read_index++]; + } + return run_length; + } + +private: + std::vector _data_map; + const std::vector* _run_length_null_map = nullptr; + bool _has_filter = false; + size_t _num_values = 0; + size_t _num_nulls = 0; + size_t _num_filtered = 0; + size_t _read_index = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp b/be/src/format_v2/parquet/reader/native/decoder.cpp new file mode 100644 index 00000000000000..5f6a978317bde7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/decoder.h" + +#include +#include + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_dict_decoder.hpp" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +namespace doris::format::parquet::native { +namespace { +Status unsupported_type(tparquet::Type::type type, tparquet::Encoding::type encoding) { + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); +} + +Status create_plain_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::PLAIN); + } +} + +Status create_dictionary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + return Status::InternalError("Boolean type cannot have a dictionary page"); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT96: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FLOAT: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::DOUBLE: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique>(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::RLE_DICTIONARY); + } +} + +Status create_delta_binary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + default: + return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); + } +} + +Status create_byte_stream_split_decoder(tparquet::Type::type type, + std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::BYTE_STREAM_SPLIT); + } +} +} // namespace + +Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return create_plain_decoder(type, decoder); + case tparquet::Encoding::RLE_DICTIONARY: + return create_dictionary_decoder(type, decoder); + case tparquet::Encoding::RLE: + if (type != tparquet::Type::BOOLEAN) { + return unsupported_type(type, encoding); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + return create_delta_binary_decoder(type, decoder); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY && type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) { + return Status::InternalError( + "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY) { + return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return create_byte_stream_split_decoder(type, decoder); + default: + return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", + tparquet::to_string(encoding), tparquet::to_string(type)); + } +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h new file mode 100644 index 00000000000000..379176d5e459e0 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __AVX2__ +#include +#endif + +#include "common/status.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/types.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { + +inline bool dictionary_indices_in_bounds(const uint32_t* indices, size_t count, + size_t dictionary_size) { + if (count == 0) { + return true; + } + if (dictionary_size == 0) { + return false; + } + uint32_t max_index = 0; + size_t row = 0; +#ifdef __AVX2__ + __m256i vector_max = _mm256_setzero_si256(); + for (; row + 8 <= count; row += 8) { + const auto values = _mm256_loadu_si256(reinterpret_cast(indices + row)); + vector_max = _mm256_max_epu32(vector_max, values); + } + alignas(32) uint32_t lanes[8]; + _mm256_store_si256(reinterpret_cast<__m256i*>(lanes), vector_max); + for (const auto lane : lanes) { + max_index = std::max(max_index, lane); + } +#endif + for (; row < count; ++row) { + max_index = std::max(max_index, indices[row]); + } + return static_cast(max_index) < dictionary_size; +} + +class Decoder : public ParquetDecodeSource { +public: + Decoder() = default; + virtual ~Decoder() = default; + + static Status get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder); + + // The type with fix length + void set_type_length(int32_t type_length) { _type_length = type_length; } + + // Set the data to be decoded + virtual Status set_data(Slice* data) { + _data = data; + _offset = 0; + return Status::OK(); + } + + // Page headers provide an upper bound before encoding-specific headers are trusted. + virtual void set_expected_values(size_t expected_values) { _expected_values = expected_values; } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + return Status::NotSupported("Fixed values are not supported by this Parquet decoder"); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + return Status::NotSupported("Binary values are not supported by this Parquet decoder"); + } + + Status skip_values(size_t num_values) override = 0; + + virtual void release_scratch(size_t max_retained_bytes) {} + virtual size_t retained_scratch_bytes() const { return 0; } + virtual size_t active_scratch_bytes() const { return 0; } + + virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + return Status::NotSupported("set_dict is not supported"); + } + +protected: + template + static void release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + if (values->capacity() * sizeof(T) > max_retained_bytes) { + std::vector().swap(*values); + } + } + + int32_t _type_length = -1; + Slice* _data = nullptr; + // Page offsets are host-sized so checked advances cannot wrap at the 4 GiB boundary. + size_t _offset = 0; + size_t _expected_values = std::numeric_limits::max(); +}; + +class BaseDictDecoder : public Decoder { +public: + BaseDictDecoder() = default; + ~BaseDictDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + if (UNLIKELY(data == nullptr || data->size == 0)) { + return Status::Corruption("Parquet dictionary index stream is empty"); + } + _data = data; + _offset = 0; + uint8_t bit_width = *data->data; + // Dictionary indices are uint32_t; wider external widths make repeated runs overwrite the + // decoder's four-byte state before any dictionary-bound check can run. + if (UNLIKELY(bit_width > 32)) { + return Status::Corruption("Parquet dictionary index bit width {} exceeds 32", + bit_width); + } + _index_batch_decoder = std::make_unique>( + reinterpret_cast(data->data) + 1, static_cast(data->size) - 1, + bit_width); + return Status::OK(); + } + + bool has_dictionary() const override { return true; } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + indices->resize(num_values); + const auto decoded = + _index_batch_decoder->GetBatch(indices->data(), cast_set(num_values)); + if (UNLIKELY(decoded != num_values)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data(), num_values, + num_dictionary_values))) { + // The SIMD common path only computes a bound; recover the exact corrupt row for the + // diagnostic after the batch has already been proven invalid. + for (size_t row = 0; row < num_values; ++row) { + if ((*indices)[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[row], row, num_dictionary_values); + } + } + return Status::OK(); + } + + Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + const size_t num_dictionary_values = dictionary_size(); + indices->resize(selection.selected_values); + size_t cursor = 0; + size_t output = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + const auto decoded = _index_batch_decoder->GetBatch(indices->data() + output, + cast_set(range.count)); + if (UNLIKELY(decoded != range.count)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data() + output, range.count, + num_dictionary_values))) { + for (size_t row = 0; row < range.count; ++row) { + if ((*indices)[output + row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[output + row], range.first + row, num_dictionary_values); + } + } + output += range.count; + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + RETURN_IF_ERROR(_decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values)); + DORIS_CHECK_EQ(output, selection.selected_values); + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + return _decode_dictionary_values(num_values, 0, dictionary_size(), consumer); + } + + Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) override { + const size_t num_dictionary_values = dictionary_size(); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + RETURN_IF_ERROR(_decode_dictionary_values(range.count, range.first, + num_dictionary_values, consumer)); + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + return _decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_skip_indices, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _skip_indices.capacity() * sizeof(uint32_t); + } + size_t active_scratch_bytes() const override { return _skip_indices.size() * sizeof(uint32_t); } + +protected: + Status skip_values(size_t num_values) override { + return _decode_and_validate_skipped(num_values, 0, dictionary_size()); + } + + Status _decode_and_validate_skipped(size_t num_values, size_t row_offset, + size_t num_dictionary_values) { + constexpr size_t kSkipBatchSize = 4096; + // Skipped dictionary ids are still external input and must be bounds-checked, but keeping + // only one bounded gap buffer avoids the page-sized scratch used by sparse selections. + _skip_indices.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped_values = 0; + while (skipped_values < num_values) { + const size_t batch_size = std::min(num_values - skipped_values, kSkipBatchSize); + const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), + static_cast(batch_size)); + if (UNLIKELY(skipped != batch_size)) { + return Status::IOError( + "Can't skip enough Parquet dictionary indices at row {}: {} of {}", + row_offset + skipped_values, skipped, batch_size); + } + // Filter gaps may be huge RLE runs; validate them in bounded SIMD-sized batches. + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch_size, + num_dictionary_values))) { + for (size_t row = 0; row < batch_size; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at skipped row {} exceeds dictionary " + "size {}", + _skip_indices[row], row_offset + skipped_values + row, + num_dictionary_values); + } + } + skipped_values += batch_size; + } + return Status::OK(); + } + + Status _decode_dictionary_values(size_t num_values, size_t row_offset, + size_t num_dictionary_values, + ParquetDictionaryValueConsumer& consumer) { + constexpr size_t kLiteralBatchSize = 1024; + size_t decoded_values = 0; + while (decoded_values < num_values) { + const int32_t repeats = _index_batch_decoder->NextNumRepeats(); + if (repeats > 0) { + const size_t run = std::min(repeats, num_values - decoded_values); + const uint32_t index = + _index_batch_decoder->GetRepeatedValue(cast_set(run)); + if (UNLIKELY(static_cast(index) >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + index, row_offset + decoded_values, num_dictionary_values); + } + RETURN_IF_ERROR(consumer.consume_repeated(index, run)); + decoded_values += run; + continue; + } + + const int32_t literals = _index_batch_decoder->NextNumLiterals(); + if (UNLIKELY(literals == 0)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t batch = std::min({static_cast(literals), + num_values - decoded_values, kLiteralBatchSize}); + _skip_indices.resize(batch); + if (UNLIKELY(!_index_batch_decoder->GetLiteralValues(cast_set(batch), + _skip_indices.data()))) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch, + num_dictionary_values))) { + for (size_t row = 0; row < batch; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + _skip_indices[row], row_offset + decoded_values + row, + num_dictionary_values); + } + } + RETURN_IF_ERROR(consumer.consume_indices(_skip_indices.data(), batch)); + decoded_values += batch; + } + return Status::OK(); + } + + // For dictionary encoding + DorisUniqueBufferPtr _dict; + std::unique_ptr> _index_batch_decoder; + std::vector _skip_indices; + uint64_t _dictionary_generation = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp new file mode 100644 index 00000000000000..c94aead8caa6b6 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" + +namespace doris::format::parquet::native { +Status DeltaLengthByteArrayDecoder::_init_lengths() { + auto length_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_len_decoder.set_bit_reader(length_reader)); + const uint32_t num_lengths = _len_decoder.valid_values_count(); + + DeltaBitPackDecoder length_locator; + length_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(length_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector lengths(std::min(num_lengths, kLocateBatchSize)); + size_t located = 0; + int64_t payload_size = 0; + while (located < num_lengths) { + const size_t batch_size = std::min(num_lengths - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR( + length_locator.decode(lengths.data(), static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + for (size_t i = 0; i < batch_size; ++i) { + if (UNLIKELY(lengths[i] < 0) || + common::add_overflow(payload_size, static_cast(lengths[i]), + payload_size)) { + return Status::Corruption("Invalid Parquet delta byte-array length"); + } + } + located += batch_size; + } + // The locator leaves the shared reader at the first payload byte without retaining every + // length; the independent decoder supplies bounded batches during materialization or skip. + if (UNLIKELY(payload_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + payload_size, _bit_reader->bytes_left()); + } + _num_valid_values = num_lengths; + return Status::OK(); +} + +Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, + int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + + int64_t data_size = 0; + _buffered_length.resize(max_values); + uint32_t lengths_decoded = 0; + RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), static_cast(max_values), + &lengths_decoded)); + if (UNLIKELY(lengths_decoded != max_values)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + const int32_t* length_ptr = _buffered_length.data(); + for (int i = 0; i < max_values; ++i) { + int32_t len = length_ptr[i]; + if (len < 0) [[unlikely]] { + return Status::InvalidArgument("Negative string delta length"); + } + buffer[i].size = len; + if (common::add_overflow(data_size, static_cast(len), data_size)) { + return Status::InvalidArgument("Excess expansion in DELTA_(LENGTH_)BYTE_ARRAY"); + } + } + // Every declared byte must exist in this page. Check before resize so a tiny malformed stream + // cannot reserve memory based only on attacker-controlled decoded lengths. + if (UNLIKELY(data_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + data_size, _bit_reader->bytes_left()); + } + _buffered_data.resize(data_size); + char* data_ptr = _buffered_data.data(); + for (int64_t j = 0; j < data_size; j++) { + if (!_bit_reader->GetValue(8, data_ptr + j)) { + return Status::IOError("Get length bytes EOF"); + } + } + + for (int i = 0; i < max_values; ++i) { + buffer[i].data = data_ptr; + data_ptr += buffer[i].size; + } + // this->num_values_ -= max_values; + _num_valid_values -= max_values; + *out_num_values = max_values; + return Status::OK(); +} + +Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = max_values; + return Status::OK(); + } + + int suffix_read; + RETURN_IF_ERROR(_suffix_decoder.decode(buffer, max_values, &suffix_read)); + if (suffix_read != max_values) [[unlikely]] { + return Status::IOError("Read {}, expecting {} from suffix decoder", + std::to_string(suffix_read), std::to_string(max_values)); + } + + int64_t data_size = 0; + _buffered_prefix_length.resize(max_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), static_cast(max_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != max_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + const int32_t* prefix_len_ptr = _buffered_prefix_length.data(); + size_t preceding_value_size = _last_value.size(); + for (int i = 0; i < max_values; ++i) { + if (prefix_len_ptr[i] < 0) [[unlikely]] { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(prefix_len_ptr[i]); + if (prefix_size > preceding_value_size) [[unlikely]] { + // Prefixes form a dependency chain, so validate each one before aggregate allocation. + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (common::add_overflow(prefix_size, buffer[i].size, reconstructed_size) || + reconstructed_size > static_cast(std::numeric_limits::max()) || + common::add_overflow(data_size, static_cast(reconstructed_size), data_size)) + [[unlikely]] { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + preceding_value_size = reconstructed_size; + } + _buffered_data.resize(data_size); + + std::string_view prefix {_last_value}; + + char* data_ptr = _buffered_data.data(); + for (int i = 0; i < max_values; ++i) { + memcpy(data_ptr, prefix.data(), prefix_len_ptr[i]); + // buffer[i] currently points to the string suffix + memcpy(data_ptr + prefix_len_ptr[i], buffer[i].data, buffer[i].size); + buffer[i].data = data_ptr; + buffer[i].size += prefix_len_ptr[i]; + data_ptr += buffer[i].size; + prefix = std::string_view {buffer[i].data, buffer[i].size}; + } + _num_valid_values -= max_values; + _last_value = std::string {prefix}; + + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + *out_num_values = max_values; + return Status::OK(); +} +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h new file mode 100644 index 00000000000000..db9efecaced470 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -0,0 +1,818 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "exec/common/arithmetic_overflow.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +namespace detail { + +inline Status checked_delta_padding_bits(uint32_t bit_width, uint32_t remaining_values, + int64_t* padding_bits) { + DORIS_CHECK(padding_bits != nullptr); + const uint64_t widened_bits = static_cast(bit_width) * remaining_values; + if (UNLIKELY(widened_bits > static_cast(std::numeric_limits::max()))) { + return Status::Corruption("Parquet delta miniblock padding is too large"); + } + *padding_bits = static_cast(widened_bits); + return Status::OK(); +} + +} // namespace detail + +class DeltaDecoder : public Decoder { +public: + DeltaDecoder() = default; + ~DeltaDecoder() override = default; +}; + +/** + * Format + * [header] [block 1] [block 2] ... [block N] + * Header + * [block size] [_mini_blocks_per_block] [_total_value_count] [first value] + * Block + * [min delta] [list of bitwidths of the mini blocks] [miniblocks] + */ +template +class DeltaBitPackDecoder final : public DeltaDecoder { +public: + using UT = std::make_unsigned_t; + + DeltaBitPackDecoder() = default; + ~DeltaBitPackDecoder() override = default; + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + uint32_t num_valid_values = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), static_cast(batch_size), + &num_valid_values)); + // Skips retain exact validation without allocating in proportion to a sparse gap. + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", + num_values, skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + _values.resize(num_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", num_values, + decoded_count); + } + return consumer.consume(reinterpret_cast(_values.data()), num_values, + sizeof(T)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + _values.resize(selection.total_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", + selection.total_values, decoded_count); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, range.count * sizeof(T)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(reinterpret_cast(_values.data()), output, + sizeof(T)); + } + + Status decode(T* buffer, uint32_t num_values, uint32_t* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + uint32_t valid_values_count() { + // _total_value_count in header ignores of null values + return _total_values_remaining; + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + // The bit-width table drives later miniblock transitions, so it is reclaimable only after + // the page is exhausted; _values contains completed batch output and is always disposable. + if (_total_values_remaining == 0) { + release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(T) + _delta_bit_widths.size() * sizeof(uint8_t); + } + + Status set_data(Slice* slice) override { + _bit_reader.reset( + new BitReader((const uint8_t*)slice->data, cast_set(slice->size))); + RETURN_IF_ERROR(_init_header()); + _data = slice; + _offset = 0; + return Status::OK(); + } + + // Set BitReader which is already initialized by DeltaLengthByteArrayDecoder or + // DeltaByteArrayDecoder + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_header()); + return Status::OK(); + } + +private: + static constexpr int kMaxDeltaBitWidth = static_cast(sizeof(T) * 8); + Status _init_header(); + Status _init_block(); + Status _init_mini_block(int bit_width); + Status _get_internal(T* buffer, uint32_t max_values, uint32_t* out_num_values); + + std::vector _values; + + std::shared_ptr _bit_reader; + uint32_t _values_per_block; + uint32_t _mini_blocks_per_block; + uint32_t _values_per_mini_block; + uint32_t _total_value_count; + + T _min_delta; + T _last_value; + + uint32_t _mini_block_idx; + std::vector _delta_bit_widths; + int _delta_bit_width; + // If the page doesn't contain any block, `_block_initialized` will + // always be false. Otherwise, it will be true when first block initialized. + bool _block_initialized; + + uint32_t _total_values_remaining; + // Remaining values in current mini block. If the current block is the last mini block, + // _values_remaining_current_mini_block may greater than _total_values_remaining. + uint32_t _values_remaining_current_mini_block; +}; + +class DeltaLengthByteArrayDecoder final : public DeltaDecoder { +public: + explicit DeltaLengthByteArrayDecoder() + : _len_decoder(), _buffered_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _len_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + int num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), static_cast(batch_size), &num_valid_values)); + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError( + "Expected to skip {} Parquet delta-length values, skipped {}", num_values, + skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + num_values, decoded_count); + } + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _values.resize(selection.total_values); + int decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + selection.total_values, decoded_count); + } + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + int valid_values_count() const { return _num_valid_values; } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_buffered_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _len_decoder.release_scratch(max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _buffered_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _len_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _buffered_length.size() * sizeof(int32_t) + _buffered_data.size() * sizeof(char) + + _len_decoder.active_scratch_bytes(); + } + + Status set_data(Slice* slice) override { + if (slice == nullptr || slice->size == 0) { + // Reused decoders must never retain lengths or payload pointers from the prior page. + _bit_reader.reset(); + _data = nullptr; + _offset = 0; + _num_valid_values = 0; + _buffered_length.clear(); + _buffered_data.clear(); + return Status::Corruption("Parquet delta-length page is empty"); + } + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + _data = slice; + _offset = 0; + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + +private: + Status _init_lengths(); + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _len_decoder; + + int _num_valid_values; + std::vector _buffered_length; + std::vector _buffered_data; +}; + +class DeltaByteArrayDecoder : public DeltaDecoder { +public: + explicit DeltaByteArrayDecoder() : _buffered_prefix_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _prefix_len_decoder.set_expected_values(expected_values); + _suffix_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + RETURN_IF_ERROR(_skip_slices(batch_size)); + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _selected_binary_values.clear(); + _selected_value_sizes.clear(); + _selected_value_sizes.reserve(selection.selected_values); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + size_t decoded = 0; + while (decoded < range.count) { + const size_t batch_size = _bounded_reconstruction_batch_size(range.count - decoded); + RETURN_IF_ERROR(_decode_slices(batch_size)); + for (const auto& value : _values) { + if (UNLIKELY(value.size > std::numeric_limits::max() - + _selected_binary_values.size())) { + return Status::IOError( + "Parquet delta-byte-array selection byte size overflows"); + } + if (value.size > 0) { + _selected_binary_values.insert(_selected_binary_values.end(), value.data, + value.data + value.size); + } + _selected_value_sizes.push_back(value.size); + } + decoded += batch_size; + } + cursor = range.first + range.count; + } + DORIS_CHECK(selection.total_values >= cursor); + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + + _string_refs.resize(selection.selected_values); + size_t byte_offset = 0; + for (size_t output = 0; output < _selected_value_sizes.size(); ++output) { + const size_t value_size = _selected_value_sizes[output]; + const char* value_data = + value_size == 0 ? nullptr : _selected_binary_values.data() + byte_offset; + _string_refs[output] = StringRef(value_data, value_size); + byte_offset += value_size; + } + DORIS_CHECK_EQ(_selected_value_sizes.size(), selection.selected_values); + DORIS_CHECK_EQ(byte_offset, _selected_binary_values.size()); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + const size_t byte_size = num_values * static_cast(_type_length); + _fixed_values.resize(byte_size); + for (size_t row = 0; row < num_values; ++row) { + memcpy(_fixed_values.data() + row * _type_length, _values[row].data, _type_length); + } + return consumer.consume(_fixed_values.data(), num_values, + static_cast(_type_length)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.selected_values > + std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet delta-byte-array selection byte size overflows"); + } + _fixed_values.resize(selection.selected_values * value_width); + size_t output = 0; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + // Skips validate widths too, so filtering cannot hide a malformed value after the + // decoder cursor has advanced past it. + RETURN_IF_ERROR(skip_values(range.first - cursor)); + size_t decoded = 0; + while (decoded < range.count) { + const size_t batch_size = _bounded_reconstruction_batch_size(range.count - decoded); + RETURN_IF_ERROR(_decode_slices(batch_size)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + for (const auto& value : _values) { + memcpy(_fixed_values.data() + output * value_width, value.data, value_width); + ++output; + } + decoded += batch_size; + } + cursor = range.first + range.count; + } + DORIS_CHECK(selection.total_values >= cursor); + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_fixed_values.data(), output, value_width); + } + + Status set_data(Slice* slice) override { + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + auto prefix_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(prefix_reader)); + + // get the number of encoded prefix lengths + int num_prefix = _prefix_len_decoder.valid_values_count(); + DeltaBitPackDecoder prefix_locator; + prefix_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(prefix_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector ignored_prefixes( + std::min(static_cast(num_prefix), kLocateBatchSize)); + size_t located = 0; + while (located < static_cast(num_prefix)) { + const size_t batch_size = + std::min(static_cast(num_prefix) - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR(prefix_locator.decode(ignored_prefixes.data(), + static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + located += batch_size; + } + _num_valid_values = num_prefix; + + // at this time, the decoder_ will be at the start of the encoded suffix data. + RETURN_IF_ERROR(_suffix_decoder.set_bit_reader(_bit_reader)); + if (UNLIKELY(_suffix_decoder.valid_values_count() != num_prefix)) { + return Status::Corruption("Parquet delta prefix and suffix counts differ"); + } + + // TODO: read corrupted files written with bug(PARQUET-246). _last_value should be set + // to _last_value_in_previous_page when decoding a new page(except the first page) + _last_value = ""; + return Status::OK(); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_fixed_values, max_retained_bytes); + release_vector_if_oversized(&_selected_binary_values, max_retained_bytes); + release_vector_if_oversized(&_selected_value_sizes, max_retained_bytes); + release_vector_if_oversized(&_buffered_prefix_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _prefix_len_decoder.release_scratch(max_retained_bytes); + _suffix_decoder.release_scratch(max_retained_bytes); + // Prefix reconstruction crosses batch boundaries, so the previous value remains semantic + // decoder state until every value in the current page has been consumed. + if (_num_valid_values == 0) { + if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); + if (_last_value_in_previous_page.capacity() > max_retained_bytes) { + std::string().swap(_last_value_in_previous_page); + } + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _fixed_values.capacity() * sizeof(uint8_t) + + _selected_binary_values.capacity() * sizeof(char) + + _selected_value_sizes.capacity() * sizeof(size_t) + + _buffered_prefix_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _last_value.capacity() + + _last_value_in_previous_page.capacity() + + _prefix_len_decoder.retained_scratch_bytes() + + _suffix_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _fixed_values.size() * sizeof(uint8_t) + + _selected_binary_values.size() * sizeof(char) + + _selected_value_sizes.size() * sizeof(size_t) + + _buffered_prefix_length.size() * sizeof(int32_t) + + _buffered_data.size() * sizeof(char) + _last_value.size() + + _last_value_in_previous_page.size() + _prefix_len_decoder.active_scratch_bytes() + + _suffix_decoder.active_scratch_bytes(); + } + +private: + size_t _bounded_reconstruction_batch_size(size_t remaining_values) const { + constexpr size_t kReconstructionByteBudget = 4UL << 20; + constexpr size_t kMaxReconstructionValues = 4096; + // Prefix expansion, rather than encoded page bytes, determines reconstruction memory. A + // count-based chunk could turn one long value into gigabytes of temporary copies, so first + // learn its width and then cap subsequent chunks by a byte budget. + if (_last_value.empty()) { + return 1; + } + const size_t budgeted_values = + std::max(1, kReconstructionByteBudget / _last_value.size()); + return std::min({remaining_values, kMaxReconstructionValues, budgeted_values}); + } + + Status _skip_slices(size_t num_values) { + _values.resize(num_values); + int suffixes_decoded = 0; + RETURN_IF_ERROR(_suffix_decoder.decode(_values.data(), cast_set(num_values), + &suffixes_decoded)); + if (UNLIKELY(suffixes_decoded != num_values)) { + return Status::IOError("Expected to skip {} Parquet delta suffixes, skipped {}", + num_values, suffixes_decoded); + } + _buffered_prefix_length.resize(num_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), cast_set(num_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != num_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + + // Only the preceding value is semantic decoder state. Rebuilding every skipped value into + // one aggregate buffer would amplify compact full-prefix references into page_size * rows. + for (size_t row = 0; row < num_values; ++row) { + const int32_t encoded_prefix_size = _buffered_prefix_length[row]; + if (UNLIKELY(encoded_prefix_size < 0)) { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(encoded_prefix_size); + if (UNLIKELY(prefix_size > _last_value.size())) { + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (UNLIKELY( + common::add_overflow(prefix_size, _values[row].size, reconstructed_size))) { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + if (_type_length > 0 && + UNLIKELY(reconstructed_size != static_cast(_type_length))) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + reconstructed_size, _type_length); + } + // resize() preserves the prefix already stored in _last_value, so only the suffix + // needs copying and repeated full-prefix values allocate no additional payload buffer. + _last_value.resize(reconstructed_size); + if (_values[row].size > 0) { + memcpy(_last_value.data() + prefix_size, _values[row].data, _values[row].size); + } + } + _num_valid_values -= cast_set(num_values); + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + return Status::OK(); + } + + Status _validate_fixed_width_values() const { + if (_type_length <= 0) { + return Status::OK(); + } + const size_t value_width = static_cast(_type_length); + for (const auto& value : _values) { + if (UNLIKELY(value.size != value_width)) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + value.size, value_width); + } + } + return Status::OK(); + } + + Status _decode_slices(size_t num_values) { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-byte-array values, decoded {}", + num_values, decoded_count); + } + return Status::OK(); + } + + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::vector _fixed_values; + std::vector _selected_binary_values; + std::vector _selected_value_sizes; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _prefix_len_decoder; + DeltaLengthByteArrayDecoder _suffix_decoder; + std::string _last_value; + // string buffer for last value in previous page + std::string _last_value_in_previous_page; + int _num_valid_values; + std::vector _buffered_prefix_length; + std::vector _buffered_data; +}; +} // namespace doris::format::parquet::native + +namespace doris::format::parquet::native { + +template +Status DeltaBitPackDecoder::_init_header() { + if (!_bit_reader->GetVlqInt(&_values_per_block) || + !_bit_reader->GetVlqInt(&_mini_blocks_per_block) || + !_bit_reader->GetVlqInt(&_total_value_count) || + !_bit_reader->GetZigZagVlqInt(&_last_value)) { + return Status::IOError("Init header eof"); + } + if (_values_per_block == 0) { + return Status::InvalidArgument("Cannot have zero value per block"); + } + if (_values_per_block % 128 != 0) { + return Status::InvalidArgument( + "the number of values in a block must be multiple of 128, but it's " + + std::to_string(_values_per_block)); + } + if (_mini_blocks_per_block == 0) { + return Status::InvalidArgument("Cannot have zero miniblock per block"); + } + // Parquet requires integral block geometry; truncating here would silently decode fewer + // values than the header declares and desynchronize the following block. + if (UNLIKELY(_values_per_block % _mini_blocks_per_block != 0)) { + return Status::Corruption("Parquet delta block size {} is not divisible by {} miniblocks", + _values_per_block, _mini_blocks_per_block); + } + _values_per_mini_block = _values_per_block / _mini_blocks_per_block; + if (_values_per_mini_block == 0) { + return Status::InvalidArgument("Cannot have zero value per miniblock"); + } + if (_values_per_mini_block % 32 != 0) { + return Status::InvalidArgument( + "The number of values in a miniblock must be multiple of 32, but it's " + + std::to_string(_values_per_mini_block)); + } + // Encoded counts are external ULEB32 values. Bound them by the page's advertised logical + // values before any vector uses the count; optional levels may only reduce this upper bound. + if (UNLIKELY(_total_value_count > _expected_values)) { + return Status::Corruption("Parquet delta header advertises {} values, page allows {}", + _total_value_count, _expected_values); + } + _total_values_remaining = _total_value_count; + _delta_bit_widths.clear(); + // init as empty property + _block_initialized = false; + _delta_bit_width = 0; + _values_remaining_current_mini_block = 0; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_block() { + DCHECK_GT(_total_values_remaining, 0) << "InitBlock called at EOF"; + if (!_bit_reader->GetZigZagVlqInt(&_min_delta)) { + return Status::IOError("Init block eof"); + } + + // One byte follows for each miniblock. Defer allocation until a block is actually consumed so + // a one-value page cannot turn an irrelevant malicious miniblock count into a large resize. + if (UNLIKELY(_mini_blocks_per_block > static_cast(_bit_reader->bytes_left()))) { + return Status::Corruption("Parquet delta miniblock count {} exceeds remaining {} bytes", + _mini_blocks_per_block, _bit_reader->bytes_left()); + } + _delta_bit_widths.resize(_mini_blocks_per_block); + + // read the bitwidth of each miniblock + uint8_t* bit_width_data = _delta_bit_widths.data(); + for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) { + if (!_bit_reader->GetAligned(1, bit_width_data + i)) { + return Status::IOError("Decode bit-width EOF"); + } + // Note that non-conformant bitwidth entries are allowed by the Parquet spec + // for extraneous miniblocks in the last block (GH-14923), so we check + // the bitwidths when actually using them (see InitMiniBlock()). + } + _mini_block_idx = 0; + _block_initialized = true; + RETURN_IF_ERROR(_init_mini_block(bit_width_data[0])); + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_mini_block(int bit_width) { + if (bit_width > kMaxDeltaBitWidth) [[unlikely]] { + return Status::InvalidArgument("delta bit width larger than integer bit width"); + } + _delta_bit_width = bit_width; + _values_remaining_current_mini_block = _values_per_mini_block; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_get_internal(T* buffer, uint32_t num_values, + uint32_t* out_num_values) { + num_values = std::min(num_values, _total_values_remaining); + if (num_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + uint32_t i = 0; + while (i < num_values) { + if (_values_remaining_current_mini_block == 0) [[unlikely]] { + if (!_block_initialized) [[unlikely]] { + buffer[i++] = _last_value; + DCHECK_EQ(i, 1); // we're at the beginning of the page + if (i == num_values) { + // When block is uninitialized and i reaches num_values we have two + // different possibilities: + // 1. _total_value_count == 1, which means that the page may have only + // one value (encoded in the header), and we should not initialize + // any block. + // 2. _total_value_count != 1, which means we should initialize the + // incoming block for subsequent reads. + if (_total_value_count != 1) { + RETURN_IF_ERROR(_init_block()); + } + break; + } + RETURN_IF_ERROR(_init_block()); + } else { + ++_mini_block_idx; + if (_mini_block_idx < _mini_blocks_per_block) { + RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx])); + } else { + RETURN_IF_ERROR(_init_block()); + } + } + } + + uint32_t values_decode = std::min(_values_remaining_current_mini_block, num_values - i); + for (uint32_t j = 0; j < values_decode; ++j) { + if (!_bit_reader->GetValue(_delta_bit_width, buffer + i + j)) { + return Status::IOError("Get batch EOF"); + } + } + for (int j = 0; j < values_decode; ++j) { + // Addition between min_delta, packed int and last_value should be treated as + // unsigned addition. Overflow is as expected. + buffer[i + j] = static_cast(_min_delta) + static_cast(buffer[i + j]) + + static_cast(_last_value); + _last_value = buffer[i + j]; + } + _values_remaining_current_mini_block -= values_decode; + i += values_decode; + } + _total_values_remaining -= num_values; + + if (_total_values_remaining == 0) [[unlikely]] { + int64_t padding_bits = 0; + // Footer-controlled miniblock counts can exceed 32-bit products. Widen before multiplying; + // BitReader::Advance then verifies that the full declared padding exists in the stream. + RETURN_IF_ERROR(detail::checked_delta_padding_bits(cast_set(_delta_bit_width), + _values_remaining_current_mini_block, + &padding_bits)); + if (!_bit_reader->Advance(padding_bits)) { + return Status::IOError("Skip padding EOF"); + } + _values_remaining_current_mini_block = 0; + } + *out_num_values = num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp new file mode 100644 index 00000000000000..1f1166421a4035 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +// Dictionary decoders retain only encoded physical values and the index-stream cursor. Logical +// interpretation is deliberately delegated to DataTypeSerDe through decode_dictionary(). +template +class FixLengthDictDecoder final : public BaseDictDecoder { +public: + FixLengthDictDecoder() = default; + ~FixLengthDictDecoder() override = default; + + size_t dictionary_size() const override { return _num_dictionary_values; } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return fixed_consumer.consume(_dict.get(), _num_dictionary_values, + static_cast(_type_length)); + } + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override { + if (UNLIKELY(_type_length <= 0 || length < 0 || + num_values > std::numeric_limits::max() / + static_cast(_type_length) || + num_values * static_cast(_type_length) != + static_cast(length))) { + return Status::Corruption("Wrong dictionary data for fixed length type"); + } + if (UNLIKELY(dict == nullptr)) { + return Status::Corruption("Fixed-length Parquet dictionary is null"); + } + _dict = std::move(dict); + _num_dictionary_values = num_values; + ++_dictionary_generation; + return Status::OK(); + } + +private: + size_t _num_dictionary_values = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp new file mode 100644 index 00000000000000..28d8154f9db10d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +#include + +#include "util/cpu_info.h" + +namespace doris::format::parquet::native { + +Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + // Truncated PLAIN pages retain the public error keyword used by corrupt-file regression + // checks, while the bounds check still prevents the native decoder from reading past data. + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + RETURN_IF_ERROR(consumer.consume(reinterpret_cast(_data->data) + _offset, + num_values, static_cast(_type_length))); + _offset += byte_size; + return Status::OK(); +} + +Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet plain selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError("Unexpected end of stream in Parquet plain selection decoder"); + } + const auto* values = reinterpret_cast(_data->data) + _offset; + _offset += input_bytes; + const size_t selected_bytes = selection.selected_values * value_width; + constexpr size_t MIN_FRAGMENTED_RANGES = 8; + constexpr size_t MAX_AVERAGE_RANGE_VALUES = 4; + const bool fragmented = + selection.ranges.size() >= MIN_FRAGMENTED_RANGES && + selection.selected_values <= selection.ranges.size() * MAX_AVERAGE_RANGE_VALUES; + if (fragmented && selected_bytes <= static_cast(CpuInfo::get_l2_cache_size())) { + _selected_values.resize(selected_bytes); + size_t output_offset = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first + range.count <= selection.total_values); + const size_t range_bytes = range.count * value_width; + memcpy(_selected_values.data() + output_offset, values + range.first * value_width, + range_bytes); + output_offset += range_bytes; + } + DORIS_CHECK_EQ(output_offset, selected_bytes); + // Tiny alternating runs make every consumer re-enter conversion and grow its destination + // column per range. A cache-resident gather keeps the page access sequential and restores + // one atomic conversion batch without retaining page-sized scratch for dense selections. + return consumer.consume(_selected_values.data(), selection.selected_values, value_width); + } + _selected_values.clear(); + // PLAIN pages are random-access fixed-width spans. Keep those page bytes pinned while the + // consumer gathers directly into the final column, otherwise sparse scans pay for a second + // selected-width buffer and copy before materialization. + return consumer.consume_selected(values, value_width, selection.ranges); +} + +Status FixLengthPlainDecoder::skip_values(size_t num_values) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || num_values > (_data->size - _offset) / value_width)) { + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + _offset += num_values * value_width; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h new file mode 100644 index 00000000000000..14913217e2e0de --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +class FixLengthPlainDecoder final : public Decoder { +public: + FixLengthPlainDecoder() = default; + ~FixLengthPlainDecoder() override = default; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_selected_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _selected_values.capacity(); } + size_t active_scratch_bytes() const override { return _selected_values.size(); } + +private: + std::vector _selected_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp new file mode 100644 index 00000000000000..618d6ebf9f6d45 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_decoder.h" + +#include + +#include +#include + +#include "common/cast_set.h" +#include "util/bit_stream_utils.inline.h" +#include "util/bit_util.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { + +static constexpr size_t V1_LEVEL_SIZE = 4; + +Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels) { + _encoding = encoding; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + switch (encoding) { + case tparquet::Encoding::RLE: { + if (slice->size < V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + + uint8_t* data = (uint8_t*)slice->data; + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > slice->size - V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + _rle_decoder = RleBatchDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); + + slice->data += V1_LEVEL_SIZE + num_bytes; + slice->size -= V1_LEVEL_SIZE + num_bytes; + break; + } + case tparquet::Encoding::BIT_PACKED: { + // Header counts are uint32_t and can overflow before the byte bound check. Widen first so + // a forged level count cannot wrap to a small slice and desynchronize following values. + const uint64_t num_bits = static_cast(num_levels) * _bit_width; + const size_t num_bytes = static_cast((num_bits + 7) / 8); + if (num_bytes > slice->size) { + return Status::Corruption("Wrong parquet level format"); + } + if (num_bytes > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet BIT_PACKED level stream is too large"); + } + _bit_packed_decoder = BitReader((uint8_t*)slice->data, cast_set(num_bytes)); + + slice->data += num_bytes; + slice->size -= num_bytes; + break; + } + default: + return Status::IOError("Unsupported encoding for parquet level"); + } + return Status::OK(); +} + +Status LevelDecoder::init_v2(const Slice& levels, level_t max_level, uint32_t num_levels) { + _encoding = tparquet::Encoding::RLE; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + size_t byte_length = levels.size; + _rle_decoder = RleBatchDecoder((uint8_t*)levels.data, cast_set(byte_length), + _bit_width); + return Status::OK(); +} + +size_t LevelDecoder::get_levels(level_t* levels, size_t n) { + _can_rewind = false; + // toto template. + if (_encoding == tparquet::Encoding::RLE) { + n = std::min((size_t)_num_levels, n); + size_t num_decoded = 0; + if (_has_buffered_level && n > 0) { + if (!accept_level(_buffered_level)) { + return 0; + } + levels[num_decoded++] = _buffered_level; + _has_buffered_level = false; + } + if (num_decoded < n) { + const size_t remaining = n - num_decoded; + _rle_scratch.resize(remaining); + const size_t batch_decoded = + _rle_decoder.GetBatch(_rle_scratch.data(), cast_set(remaining)); + for (size_t i = 0; i < batch_decoded; ++i) { + const level_t level = cast_set(_rle_scratch[i]); + if (!accept_level(level)) { + return 0; + } + levels[num_decoded + i] = level; + } + num_decoded += batch_decoded; + } + _num_levels -= num_decoded; + _can_rewind = false; + return num_decoded; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + n = std::min((size_t)_num_levels, n); + size_t decoded = 0; + for (; decoded < n; ++decoded) { + level_t level = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &level)) { + break; + } + if (!accept_level(level)) { + return 0; + } + levels[decoded] = level; + } + _num_levels -= decoded; + return decoded; + } + return 0; +} + +size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { + DORIS_CHECK(val != nullptr); + _can_rewind = false; + max_run = std::min(max_run, _num_levels); + if (max_run == 0) { + return 0; + } + if (_encoding == tparquet::Encoding::RLE) { + size_t decoded = 0; + if (_has_buffered_level) { + *val = _buffered_level; + _has_buffered_level = false; + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } else { + uint16_t first = 0; + if (_rle_decoder.GetBatch(&first, 1) != 1) { + return 0; + } + *val = cast_set(first); + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } + while (decoded < max_run) { + const int32_t repeats = _rle_decoder.NextNumRepeats(); + if (repeats > 0) { + const level_t repeated = cast_set(_rle_decoder.GetRepeatedValue(0)); + if (!accept_level(repeated)) return 0; + if (repeated != *val) break; + const int32_t consume = std::min(repeats, cast_set(max_run - decoded)); + _rle_decoder.GetRepeatedValue(consume); + decoded += consume; + continue; + } + if (_rle_decoder.NextNumLiterals() == 0) break; + uint16_t literal = 0; + if (!_rle_decoder.GetLiteralValues(1, &literal)) break; + const level_t next = cast_set(literal); + if (!accept_level(next)) return 0; + if (next != *val) { + // Batch RLE has no physical rewind; retain the one-value lookahead logically. + _buffered_level = next; + _has_buffered_level = true; + break; + } + ++decoded; + } + _num_levels -= decoded; + _can_rewind = false; + return decoded; + } + if (_encoding != tparquet::Encoding::BIT_PACKED || + !_bit_packed_decoder.GetValue(_bit_width, val)) { + return 0; + } + if (!accept_level(*val)) { + return 0; + } + size_t decoded = 1; + while (decoded < max_run) { + level_t next = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &next)) { + break; + } + if (!accept_level(next)) return 0; + if (next != *val) { + // The lookahead belongs to the following run, so cursor APIs must leave it unread. + _bit_packed_decoder.Rewind(_bit_width); + break; + } + ++decoded; + } + _num_levels -= decoded; + return decoded; +} + +level_t LevelDecoder::get_next() { + if (_num_levels == 0) { + return -1; + } + level_t next = -1; + bool decoded = false; + if (_encoding == tparquet::Encoding::RLE) { + if (_has_buffered_level) { + next = _buffered_level; + _has_buffered_level = false; + decoded = true; + } else { + uint16_t value = 0; + decoded = _rle_decoder.GetBatch(&value, 1) == 1; + next = cast_set(value); + } + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + decoded = _bit_packed_decoder.GetValue(_bit_width, &next); + } + if (!decoded) { + return -1; + } + if (!accept_level(next)) { + return -1; + } + --_num_levels; + _last_level = next; + _can_rewind = true; + return next; +} + +void LevelDecoder::rewind_one() { + if (_encoding == tparquet::Encoding::RLE) { + DORIS_CHECK(_can_rewind && !_has_buffered_level); + _buffered_level = _last_level; + _has_buffered_level = true; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + DORIS_CHECK(_can_rewind); + _bit_packed_decoder.Rewind(_bit_width); + } else { + return; + } + // Rewinding restores one advertised level as well as its encoded bits. + ++_num_levels; + _can_rewind = false; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h new file mode 100644 index 00000000000000..9dcff491eb0759 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/common.h" +#include "util/bit_stream_utils.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class LevelDecoder { +public: + LevelDecoder() = default; + ~LevelDecoder() = default; + + Status init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels); + + Status init_v2(const Slice& levels, level_t max_level, uint32_t num_levels); + + inline bool has_levels() const { return _num_levels > 0; } + + size_t get_levels(level_t* levels, size_t n); + + size_t get_next_run(level_t* val, size_t max_run); + + level_t get_next(); + + void rewind_one(); + + void release_scratch(size_t max_retained_bytes) { + if (_rle_scratch.capacity() * sizeof(uint16_t) > max_retained_bytes) { + std::vector().swap(_rle_scratch); + } + } + size_t retained_scratch_bytes() const { return _rle_scratch.capacity() * sizeof(uint16_t); } + size_t active_scratch_bytes() const { return _rle_scratch.size() * sizeof(uint16_t); } + +private: + bool accept_level(level_t level) { + if (level >= 0 && level <= _max_level) { + return true; + } + // Bit width rounds up to a power of two, so encoded values can fit the bit stream while + // still exceeding the schema maximum. Poison the decoder before any caller can use them. + _num_levels = 0; + _has_buffered_level = false; + _can_rewind = false; + return false; + } + + tparquet::Encoding::type _encoding; + level_t _bit_width = 0; + level_t _max_level = 0; + uint32_t _num_levels = 0; + RleBatchDecoder _rle_decoder; + std::vector _rle_scratch; + BitReader _bit_packed_decoder; + bool _has_buffered_level = false; + bool _can_rewind = false; + level_t _buffered_level = -1; + level_t _last_level = -1; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp new file mode 100644 index 00000000000000..06b71582d7a2be --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_reader.h" + +#include +#include + +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" + +namespace doris::format::parquet::native { + +class LevelReader::Impl { +public: + virtual ~Impl() = default; + + virtual Status init() = 0; + virtual Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) = 0; + virtual ColumnChunkReaderStatistics statistics() = 0; +}; + +template +class LevelReaderImpl final : public LevelReader::Impl { +public: + LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, std::string page_cache_file_key, + ParquetReaderCompat compat) + : _file(std::move(file)), + _column_chunk(std::move(column_chunk)), + _field(field), + _total_rows(total_rows), + _max_buffer_size(max_buffer_size), + _io_ctx(io_ctx), + _enable_page_cache(enable_page_cache), + _page_cache_file_key(std::move(page_cache_file_key)), + _compat(compat) {} + + Status init() override { + DORIS_CHECK(_file != nullptr); + DORIS_CHECK(_field != nullptr); + const auto& metadata = _column_chunk.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(metadata, _file->size(), + _compat.parquet_816_padding, &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_size = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_size, _max_buffer_size); + auto* tracing_reader = typeid_cast(_file.get()); + if ((tracing_reader != nullptr && + typeid_cast(tracing_reader->inner_reader().get()) != + nullptr) || + typeid_cast(_file.get()) != nullptr) { + prefetch_buffer_size = 0; + } + _stream = std::make_unique(_file, chunk_start, chunk_size, + prefetch_buffer_size); + _chunk_reader = std::make_unique>( + _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, + ParquetPageReadContext(_enable_page_cache, _page_cache_file_key, + _compat.data_page_v2_always_compressed), + &chunk_range); + return _chunk_reader->init(); + } + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) override { + DORIS_CHECK(repetition_levels != nullptr); + DORIS_CHECK(definition_levels != nullptr); + DORIS_CHECK(rows_read != nullptr); + repetition_levels->clear(); + definition_levels->clear(); + *rows_read = 0; + if (_current_row > _total_rows || rows > _total_rows - _current_row) { + return Status::Corruption("Parquet level reader requested rows [{}, {}) of {}", + _current_row, _current_row + rows, _total_rows); + } + if constexpr (IN_COLLECTION) { + RETURN_IF_ERROR( + read_nested_rows(rows, repetition_levels, definition_levels, rows_read)); + } else { + RETURN_IF_ERROR(read_flat_rows(rows, repetition_levels, definition_levels, rows_read)); + } + _current_row += *rows_read; + return Status::OK(); + } + + ColumnChunkReaderStatistics statistics() override { return _chunk_reader->statistics(); } + +private: + Status read_flat_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + const size_t read_now = std::min( + rows - *rows_read, static_cast(_chunk_reader->remaining_num_values())); + if (read_now == 0) { + // Zero-value data pages are legal and do not contribute to the chunk cardinality. + // Advance them before applying the no-progress guard used for truncated chunks. + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + continue; + } + return Status::Corruption("Parquet flat level reader made no progress"); + } + RETURN_IF_ERROR( + _chunk_reader->read_levels(read_now, repetition_levels, definition_levels)); + *rows_read += read_now; + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + return Status::OK(); + } + + Status read_nested_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(_current_row + *rows_read)); + const size_t start_level = definition_levels->size(); + size_t loaded_rows = 0; + bool crosses_page = false; + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows( + *repetition_levels, rows - *rows_read, &loaded_rows, &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(*definition_levels, start_level)); + while (crosses_page) { + const size_t continuation_start = definition_levels->size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(*repetition_levels, + &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR( + _chunk_reader->skip_nested_values(*definition_levels, continuation_start)); + } + if (loaded_rows == 0) { + return Status::Corruption("Parquet nested level reader made no progress"); + } + *rows_read += loaded_rows; + } + return Status::OK(); + } + + io::FileReaderSPtr _file; + tparquet::ColumnChunk _column_chunk; + NativeFieldSchema* _field = nullptr; + size_t _total_rows = 0; + size_t _max_buffer_size = 0; + io::IOContext* _io_ctx = nullptr; + bool _enable_page_cache = false; + std::string _page_cache_file_key; + ParquetReaderCompat _compat; + size_t _current_row = 0; + std::unique_ptr _stream; + std::unique_ptr> _chunk_reader; +}; + +Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, + std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(field != nullptr); + std::unique_ptr impl; + if (field->repetition_level > 0) { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } else { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } + RETURN_IF_ERROR(impl->init()); + reader->reset(new LevelReader(std::move(impl))); + return Status::OK(); +} + +LevelReader::LevelReader(std::unique_ptr impl) : _impl(std::move(impl)) {} + +LevelReader::~LevelReader() = default; + +Status LevelReader::read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + return _impl->read_rows(rows, repetition_levels, definition_levels, rows_read); +} + +Status LevelReader::skip_rows(size_t rows) { + size_t rows_read = 0; + RETURN_IF_ERROR( + read_rows(rows, &_skip_repetition_levels, &_skip_definition_levels, &rows_read)); + if (rows_read != rows) { + return Status::Corruption("Parquet level reader skipped {} of {} rows", rows_read, rows); + } + return Status::OK(); +} + +ColumnChunkReaderStatistics LevelReader::statistics() { + return _impl->statistics(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h new file mode 100644 index 00000000000000..11117d25dcb252 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { + +// A physical-leaf reader that advances all three Parquet streams (repetition levels, definition +// levels and encoded values) but retains only the two level streams. It is used for shape-only +// operations such as COUNT(nullable_col), where materializing a large BYTE_ARRAY value would be +// both unnecessary and potentially unbounded. +// +// This deliberately shares ColumnChunkReader with the value path. Page V1/V2 parsing, +// decompression, dictionary-page handling, page cache semantics and corruption checks therefore +// cannot drift between an aggregate shortcut and an ordinary scan. +class LevelReader { +public: + class Impl; + + static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + std::unique_ptr* reader); + + ~LevelReader(); + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read); + Status skip_rows(size_t rows); + ColumnChunkReaderStatistics statistics(); + +private: + explicit LevelReader(std::unique_ptr impl); + + std::unique_ptr _impl; + // COUNT range gaps can call skip repeatedly as the adaptive row cap changes. Keep these + // throw-away level buffers on the persistent reader so only their logical sizes are reset. + std::vector _skip_repetition_levels; + std::vector _skip_definition_levels; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp new file mode 100644 index 00000000000000..37f682cbd4d9d5 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/page_reader.h" + +#include +#include +#include +#include + +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/slice.h" +#include "util/thrift_util.h" + +namespace doris { +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { +static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; + +template +Status PageReader::_validate_page_header(uint32_t header_size) const { + if (UNLIKELY(_cur_page_header.compressed_page_size < 0 || + _cur_page_header.uncompressed_page_size < 0)) { + return Status::Corruption("Parquet page has a negative compressed or uncompressed size"); + } + if (UNLIKELY(header_size > _end_offset - _offset || + static_cast(_cur_page_header.compressed_page_size) > + _end_offset - _offset - header_size)) { + // Sizes are untrusted signed Thrift fields and must fit the column chunk before arithmetic. + return Status::Corruption("Parquet page payload exceeds its column chunk"); + } + + const bool has_v1 = _cur_page_header.__isset.data_page_header; + const bool has_v2 = _cur_page_header.__isset.data_page_header_v2; + const bool has_dictionary = _cur_page_header.__isset.dictionary_page_header; + const bool has_index = _cur_page_header.__isset.index_page_header; + bool matching_layout = false; + switch (_cur_page_header.type) { + case tparquet::PageType::DATA_PAGE: + matching_layout = has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DATA_PAGE_V2: + matching_layout = has_v2 && !has_v1 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DICTIONARY_PAGE: + matching_layout = has_dictionary && !has_v1 && !has_v2 && !has_index; + break; + case tparquet::PageType::INDEX_PAGE: + matching_layout = has_index && !has_v1 && !has_v2 && !has_dictionary; + break; + default: + // Forward-compatible auxiliary pages have no known page-specific member. Their bounded + // payload can be skipped safely; accepting a known member here would let the enum and + // decoder layout disagree. + matching_layout = !has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + } + if (UNLIKELY(!matching_layout)) { + // The type discriminant owns the only valid union member. Guessing from an optional header + // lets a malformed page use one layout for validation and another for decoding. + return Status::Corruption("Parquet page type does not match its page-header layout"); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + const auto& v2 = _cur_page_header.data_page_header_v2; + if (UNLIKELY(v2.num_values < 0 || v2.num_rows < 0 || v2.num_nulls < 0 || + v2.repetition_levels_byte_length < 0 || + v2.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has negative counts or level sizes"); + } + if (UNLIKELY(v2.num_nulls > v2.num_values || v2.num_rows > v2.num_values)) { + return Status::Corruption( + "Parquet data page v2 null or row count exceeds its value count"); + } + const uint64_t level_bytes = static_cast(v2.repetition_levels_byte_length) + + static_cast(v2.definition_levels_byte_length); + if (UNLIKELY(level_bytes > static_cast(_cur_page_header.compressed_page_size) || + level_bytes > + static_cast(_cur_page_header.uncompressed_page_size))) { + return Status::Corruption("Parquet data page v2 level bytes exceed the page payload"); + } + } else if (_cur_page_header.type == tparquet::PageType::DATA_PAGE && + UNLIKELY(_cur_page_header.data_page_header.num_values < 0)) { + return Status::Corruption("Parquet data page has a negative value count"); + } else if (_cur_page_header.type == tparquet::PageType::DICTIONARY_PAGE && + UNLIKELY(_cur_page_header.dictionary_page_header.num_values < 0)) { + return Status::Corruption("Parquet dictionary page has a negative value count"); + } + return Status::OK(); +} + +template +PageReader::PageReader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, + const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index) + : _reader(reader), + _io_ctx(io_ctx), + _offset(offset), + _start_offset(offset), + _end_offset(offset + length), + _total_rows(total_rows), + _metadata(metadata), + _page_read_ctx(page_read_ctx), + _offset_index(offset_index) { + _next_header_offset = _offset; + _state = INITIALIZED; + _page_cache_key_builder.init(_page_read_ctx.page_cache_file_key); + + if constexpr (OFFSET_INDEX) { + _end_row = _offset_index != nullptr && _offset_index->page_locations.size() >= 2 + ? _offset_index->page_locations[1].first_row_index + : _total_rows; + } +} + +template +void PageReader::_reconcile_offset_index_location( + uint64_t header_offset, uint32_t header_size) { + if constexpr (!OFFSET_INDEX) { + return; + } + if (_offset_index == nullptr || _page_index >= _offset_index->page_locations.size()) { + return; + } + const auto& location = _offset_index->page_locations[_page_index]; + const bool data_page = _cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2; + const uint64_t serialized_size = static_cast(header_size) + + static_cast(_cur_page_header.compressed_page_size); + const bool matches = data_page && location.offset >= 0 && location.compressed_page_size > 0 && + header_offset == static_cast(location.offset) && + serialized_size == static_cast(location.compressed_page_size); + if (!matches && (data_page || (location.offset >= 0 && + header_offset >= static_cast(location.offset)))) { + // OffsetIndex is optional. Once its data-page rectangle disagrees with the serialized + // page, continue sequentially so a shifted location cannot redirect the next seek. + _offset_index = nullptr; + } +} + +template +void PageReader::_update_sequential_row_range() { + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + return; + } + } + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } +} + +template +Status PageReader::parse_page_header() { + if (_state == HEADER_PARSED) { + return Status::OK(); + } + if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { + return Status::IOError("Out-of-bounds Access"); + } + if (UNLIKELY(_offset != _next_header_offset)) { + return Status::IOError("Wrong header position, should seek to a page header first"); + } + if (UNLIKELY(_state != INITIALIZED)) { + return Status::IOError("Should skip or load current page to get next page"); + } + + _page_statistics.page_read_counter += 1; + + // Parse page header from file; header bytes are saved for possible cache insertion + const uint8_t* page_header_buf = nullptr; + size_t max_size = _end_offset - _offset; + size_t header_size = std::min(INIT_PAGE_HEADER_SIZE, max_size); + const size_t MAX_PAGE_HEADER_SIZE = config::parquet_header_max_size_mb << 20; + uint32_t real_header_size = 0; + + // Try a header-only lookup in the page cache. Cached pages store + // header + optional v2 levels + uncompressed payload, so we can + // parse the page header directly from the cached bytes and avoid + // a file read for the header. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + PageCacheHandle handle; + StoragePageCache::CacheKey key = make_page_cache_key(static_cast(_offset)); + if (StoragePageCache::instance()->lookup(key, &handle, segment_v2::DATA_PAGE)) { + // Parse header directly from cached data + _page_cache_handle = std::move(handle); + Slice s = _page_cache_handle.data(); + real_header_size = cast_set(s.size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Thrift does not clear absent optional fields when reusing an output object. + _cur_page_header = tparquet::PageHeader {}; + auto st = deserialize_thrift_msg(reinterpret_cast(s.data), + &real_header_size, true, &_cur_page_header); + if (!st.ok()) return st; + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + _reconcile_offset_index_location(_offset, real_header_size); + _update_sequential_row_range(); + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + // Increment page cache counters for a true cache hit on header+payload + _page_statistics.page_cache_hit_counter += 1; + // Detect whether the cached payload is compressed or decompressed and record + bool is_cache_payload_decompressed = should_cache_decompressed( + &_cur_page_header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (is_cache_payload_decompressed) { + _page_statistics.page_cache_decompressed_hit_counter += 1; + } else { + _page_statistics.page_cache_compressed_hit_counter += 1; + } + + _is_cache_payload_decompressed = is_cache_payload_decompressed; + + // Save header bytes for later use (e.g., to insert updated cache entries) + _header_buf.assign(s.data, s.data + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); + } else { + _page_statistics.page_cache_missing_counter += 1; + // Clear any existing cache handle on miss to avoid holding stale handle + _page_cache_handle = PageCacheHandle(); + } + } + // NOTE: page cache lookup for *decompressed* page data is handled in + // ColumnChunkReader::load_page_data(). PageReader should only be + // responsible for parsing the header bytes from the file and saving + // them in `_header_buf` for possible later insertion into the cache. + while (true) { + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + header_size = std::min(header_size, max_size); + { + SCOPED_RAW_TIMER(&_page_statistics.read_page_header_time); + RETURN_IF_ERROR(_reader->read_bytes(&page_header_buf, _offset, header_size, _io_ctx)); + } + real_header_size = cast_set(header_size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Reset on every retry as a partial deserialize can otherwise leak a stale union member. + _cur_page_header = tparquet::PageHeader {}; + auto st = + deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header); + if (st.ok()) { + break; + } + if (_offset + header_size >= _end_offset || real_header_size > MAX_PAGE_HEADER_SIZE) { + return Status::IOError( + "Failed to deserialize parquet page header. offset: {}, " + "header size: {}, end offset: {}, real header size: {}", + _offset, header_size, _end_offset, real_header_size); + } + header_size <<= 2; + } + + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + _reconcile_offset_index_location(_offset, real_header_size); + _update_sequential_row_range(); + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + + // Save header bytes for possible cache insertion later + _header_buf.assign(page_header_buf, page_header_buf + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status PageReader::get_page_data(Slice& slice) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::IOError("Should generate page header first to load current page data"); + } + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + slice.size = _cur_page_header.compressed_page_size; + RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx)); + _offset += slice.size; + _state = DATA_LOADED; + return Status::OK(); +} + +template class PageReader; +template class PageReader; +template class PageReader; +template class PageReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h new file mode 100644 index 00000000000000..da2c066891bda7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "storage/cache/page_cache.h" +#include "util/block_compression.h" +namespace doris { +class BlockCompressionCodec; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +/** + * Use to deserialize parquet page header, and get the page data in iterator interface. + */ + +// Session-level options for parquet page reading/caching. +struct ParquetPageReadContext { + // A default-constructed context has no stable file identity, so cache lookup must stay off. + bool enable_parquet_file_page_cache = false; + std::string page_cache_file_key; + bool data_page_v2_always_compressed = false; + ParquetPageReadContext() = default; + ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key, + bool data_page_v2_always_compressed = false) + : enable_parquet_file_page_cache(enable_parquet_file_page_cache && + !page_cache_file_key.empty()), + page_cache_file_key(std::move(page_cache_file_key)), + data_page_v2_always_compressed(data_page_v2_always_compressed) {} +}; + +inline bool should_cache_decompressed(const tparquet::PageHeader* header, + const tparquet::ColumnMetaData& metadata, + bool data_page_v2_always_compressed = false) { + // Data Page V2 declares its payload representation independently of the column codec. A warm + // hit must never send an explicitly uncompressed cached payload through the codec again. + if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed && + !data_page_v2_always_compressed) { + return true; + } + if (header->compressed_page_size <= 0) return true; + if (metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) return true; + if (header->uncompressed_page_size == 0) return true; + + double ratio = static_cast(header->uncompressed_page_size) / + static_cast(header->compressed_page_size); + return ratio <= config::parquet_page_cache_decompress_threshold; +} + +class ParquetPageCacheKeyBuilder { +public: + void init(std::string file_key) { _file_key_prefix = std::move(file_key); } + StoragePageCache::CacheKey make_key(uint64_t end_offset, int64_t offset) const { + return StoragePageCache::CacheKey(_file_key_prefix, end_offset, offset); + } + +private: + std::string _file_key_prefix; +}; + +template +class PageReader { +public: + struct PageStatistics { + int64_t decode_header_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + }; + + PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index = nullptr); + ~PageReader() = default; + + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + return _page_index + 1 < _offset_index->page_locations.size(); + } + } + return _offset < _end_offset; + } + + Status parse_page_header(); + + Status next_page() { + _page_statistics.skip_page_header_num += _state == INITIALIZED; + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + if (UNLIKELY(_page_index + 1 >= _offset_index->page_locations.size())) { + return Status::Corruption("Parquet OffsetIndex has no next page location"); + } + _page_index++; + _start_row = _offset_index->page_locations[_page_index].first_row_index; + if (_page_index + 1 < _offset_index->page_locations.size()) { + _end_row = _offset_index->page_locations[_page_index + 1].first_row_index; + } else { + _end_row = _total_rows; + } + int64_t next_page_offset = _offset_index->page_locations[_page_index].offset; + _offset = next_page_offset; + _next_header_offset = next_page_offset; + _state = INITIALIZED; + return Status::OK(); + } + } + + if (UNLIKELY(_offset == _start_offset)) { + return Status::Corruption("should parse first page."); + } + if (is_header_v2()) { + _start_row += _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _start_row += _cur_page_header.data_page_header.num_values; + } + _offset = _next_header_offset; + _state = INITIALIZED; + + return Status::OK(); + } + + Status skip_auxiliary_page() { + if constexpr (OFFSET_INDEX) { + // OffsetIndex enumerates data pages only, so an auxiliary physical page must not + // consume a logical page-location entry while advancing to its payload end. + skip_page_data(); + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status dict_next_page() { + if constexpr (OFFSET_INDEX) { + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status get_page_header(const tparquet::PageHeader** page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + *page_header = &_cur_page_header; + return Status::OK(); + } + + Status get_page_data(Slice& slice); + + // Skip page data and update offset (used when data is loaded from cache) + void skip_page_data() { + if (_state == HEADER_PARSED) { + _offset += _cur_page_header.compressed_page_size; + _state = DATA_LOADED; + } + } + + const std::vector& header_bytes() const { return _header_buf; } + // header start offset for current page + int64_t header_start_offset() const { + return static_cast(_next_header_offset) - static_cast(_last_header_size) - + static_cast(_cur_page_header.compressed_page_size); + } + uint64_t file_end_offset() const { return _end_offset; } + bool cached_decompressed() const { + return should_cache_decompressed(&_cur_page_header, _metadata, + _page_read_ctx.data_page_v2_always_compressed); + } + + PageStatistics& page_statistics() { return _page_statistics; } + + bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; } + + // Returns whether the current page's cache payload is decompressed + bool is_cache_payload_decompressed() const { return _is_cache_payload_decompressed; } + + size_t start_row() const { return _start_row; } + + size_t end_row() const { return _end_row; } + + bool has_active_offset_index() const { + if constexpr (OFFSET_INDEX) { + return _offset_index != nullptr; + } + return false; + } + + void discard_offset_index() { + if constexpr (OFFSET_INDEX) { + _offset_index = nullptr; + } + } + + // Accessors for cache handle + bool has_page_cache_handle() const { return _page_cache_handle.cache() != nullptr; } + const doris::PageCacheHandle& page_cache_handle() const { return _page_cache_handle; } + StoragePageCache::CacheKey make_page_cache_key(int64_t offset) const { + return _page_cache_key_builder.make_key(_end_offset, offset); + } + +private: + Status _validate_page_header(uint32_t header_size) const; + void _reconcile_offset_index_location(uint64_t header_offset, uint32_t header_size); + void _update_sequential_row_range(); + + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; + PageReaderState _state = INITIALIZED; + PageStatistics _page_statistics; + + io::BufferedStreamReader* _reader = nullptr; + io::IOContext* _io_ctx = nullptr; + // current reader offset in file location. + uint64_t _offset = 0; + // this page offset in file location. + uint64_t _start_offset = 0; + uint64_t _end_offset = 0; + uint64_t _next_header_offset = 0; + // current page row range + size_t _start_row = 0; + size_t _end_row = 0; + // total rows in this column chunk + size_t _total_rows = 0; + // Column metadata for this column chunk + const tparquet::ColumnMetaData& _metadata; + // Session-level parquet page cache options + ParquetPageReadContext _page_read_ctx; + // for page index + size_t _page_index = 0; + const tparquet::OffsetIndex* _offset_index; + + tparquet::PageHeader _cur_page_header; + bool _is_cache_payload_decompressed = true; + + // Page cache members + ParquetPageCacheKeyBuilder _page_cache_key_builder; + doris::PageCacheHandle _page_cache_handle; + // stored header bytes when cache miss so we can insert header+payload into cache + std::vector _header_buf; + // last parsed header size in bytes + uint32_t _last_header_size = 0; +}; + +template +std::unique_ptr> create_page_reader( + io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length, + size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& ctx, const tparquet::OffsetIndex* offset_index = nullptr) { + return std::make_unique>( + reader, io_ctx, offset, length, total_rows, metadata, ctx, offset_index); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp new file mode 100644 index 00000000000000..233b37c096e300 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -0,0 +1,772 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native_column_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "runtime/runtime_state.h" + +namespace doris::format::parquet { +namespace { + +constexpr size_t MAX_RETAINED_BATCH_SCRATCH_BYTES = 4UL << 20; + +DataTypePtr projected_type(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection) { + if (!format::is_partial_projection(projection)) { + return schema.type; + } + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + return schema.type; + case ParquetColumnSchemaKind::STRUCT: { + DataTypes child_types; + Strings child_names; + child_types.reserve(projection->children.size()); + child_names.reserve(projection->children.size()); + for (const auto& child_projection : projection->children) { + const auto child_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != schema.children.end()); + child_types.push_back(make_nullable(projected_type(**child_it, &child_projection))); + child_names.push_back((*child_it)->name); + } + DataTypePtr type = std::make_shared(child_types, child_names); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto* child_projection = + format::find_child_projection(projection, schema.children[0]->local_id); + DORIS_CHECK(child_projection != nullptr); + DataTypePtr type = std::make_shared( + projected_type(*schema.children[0], child_projection)); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::MAP: { + DORIS_CHECK(schema.children.size() == 2); + const auto* value_projection = + format::find_child_projection(projection, schema.children[1]->local_id); + DORIS_CHECK(value_projection != nullptr); + DataTypePtr type = std::make_shared( + make_nullable(schema.children[0]->type), + make_nullable(projected_type(*schema.children[1], value_projection))); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + } + DORIS_CHECK(false); + return nullptr; +} + +const NativeFieldSchema* find_child_field(const NativeFieldSchema& parent, + const ParquetColumnSchema& child) { + auto field_it = std::ranges::find_if(parent.children, [&](const NativeFieldSchema& field) { + return (child.parquet_field_id >= 0 && field.field_id == child.parquet_field_id) || + field.name == child.name; + }); + return field_it == parent.children.end() ? nullptr : &*field_it; +} + +void collect_physical_subtree_ids(const NativeFieldSchema& field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + ids->insert(field.get_column_id()); + for (const auto& child : field.children) { + collect_physical_subtree_ids(child, ids); + } +} + +void collect_projected_ids(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const NativeFieldSchema& native_field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + if (!format::is_partial_projection(projection)) { + return; + } + for (const auto& child_projection : projection->children) { + const auto schema_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(schema_it != schema.children.end()); + const NativeFieldSchema* child_field = find_child_field(native_field, **schema_it); + DORIS_CHECK(child_field != nullptr); + if (format::is_full_projection(&child_projection)) { + // A full child path is a request for its complete physical subtree. Keeping only the + // child group id makes its grandchildren SkipReadingReaders and silently defaults + // STRUCT fields (or breaks ARRAY/MAP shape invariants). + collect_physical_subtree_ids(*child_field, ids); + } else { + ids->insert(child_field->get_column_id()); + collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + } + } + if (schema.kind == ParquetColumnSchemaKind::MAP) { + DORIS_CHECK(!native_field.children.empty()); + // MAP entry existence and offsets are owned by the key stream even for value-only + // projections. Keep the key reader live so the native complex reader can validate + // key/value entry alignment while constructing offsets. + ids->insert(native_field.children[0].get_column_id()); + } +} + +Status append_non_null_dictionary_values(MutableColumnPtr& target, MutableColumnPtr values) { + DORIS_CHECK(target); + DORIS_CHECK(values); + const size_t value_count = values->size(); + if (auto* nullable = check_and_get_column(*target); nullable != nullptr) { + nullable->get_nested_column().insert_range_from(*values, 0, value_count); + auto& null_map = nullable->get_null_map_data(); + null_map.resize_fill(null_map.size() + value_count, 0); + return Status::OK(); + } + target->insert_range_from(*values, 0, value_count); + return Status::OK(); +} + +} // namespace + +NativeColumnReader::NativeColumnReader(const ParquetColumnSchema& schema, + DataTypePtr projected_type, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(schema, std::move(projected_type), profile), + _nested(schema.kind != ParquetColumnSchemaKind::PRIMITIVE) {} + +NativeColumnReader::~NativeColumnReader() { + (void)sync_native_profile(); +} + +Status NativeColumnReader::create( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + if (reader == nullptr) { + return Status::InvalidArgument("Native parquet reader result is null"); + } + if (file == nullptr || metadata == nullptr) { + return Status::InvalidArgument("Native parquet file context is not initialized"); + } + if (row_group_id < 0 || + row_group_id >= static_cast(metadata->to_thrift().row_groups.size())) { + return Status::InvalidArgument("Invalid native parquet row group {}", row_group_id); + } + const auto& native_schema = metadata->schema(); + if (column_schema.local_id < 0 || column_schema.local_id >= native_schema.size()) { + return Status::InvalidArgument("Invalid native parquet top-level column id {} for {}", + column_schema.local_id, column_schema.name); + } + auto* field = const_cast(native_schema.get_column(column_schema.local_id)); + DORIS_CHECK(field != nullptr); + if (field->name != column_schema.name && + !(field->field_id >= 0 && field->field_id == column_schema.parquet_field_id)) { + return Status::Corruption( + "Native/metadata parquet schema mismatch at column {}: native={}, arrow={}", + column_schema.local_id, field->name, column_schema.name); + } + + auto type = projected_type(column_schema, projection); + std::shared_ptr schema_node; + RETURN_IF_ERROR(build_native_schema_node(type, column_schema, &schema_node)); + std::set projected_ids; + collect_projected_ids(column_schema, projection, *field, &projected_ids); + + auto native_reader = std::unique_ptr( + new NativeColumnReader(column_schema, std::move(type), profile)); + RETURN_IF_ERROR(native_reader->init( + std::move(file), metadata, row_group_id, field, std::move(schema_node), + std::move(projected_ids), selected_ranges, offset_indexes, timezone, io_ctx, + runtime_state, enable_page_cache, page_cache_file_key, enable_dictionary_filter)); + *reader = std::move(native_reader); + return Status::OK(); +} + +Status NativeColumnReader::init( + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(field != nullptr); + DORIS_CHECK(schema_node != nullptr); + const auto& row_group = metadata->to_thrift().row_groups[row_group_id]; + DORIS_CHECK(row_group.num_rows > 0); + _row_group_rows = row_group.num_rows; + _selected_ranges = selected_ranges; + DORIS_CHECK(!_selected_ranges.empty()); + for (const auto& range : _selected_ranges) { + DORIS_CHECK(range.start >= 0); + DORIS_CHECK(range.length > 0); + DORIS_CHECK(range.start + range.length <= _row_group_rows); + _row_ranges.add(segment_v2::RowRange(range.start, range.start + range.length)); + } + // Offset indexes are immutable row-group metadata owned by ParquetScanScheduler. Sharing them + // avoids retaining the full N-column page-location map once per projected reader. + _offset_indexes = &offset_indexes; + _schema_node = std::move(schema_node); + _projected_column_ids = std::move(projected_column_ids); + _dictionary_filter_enabled = enable_dictionary_filter; + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + const size_t max_buffer_size = std::min(max_group_buffer, max_column_buffer); + RuntimeState* native_runtime_state = runtime_state; + const bool runtime_page_cache_enabled = + runtime_state == nullptr || + runtime_state->query_options().enable_parquet_file_page_cache; + if (runtime_page_cache_enabled != enable_page_cache) { + TQueryOptions query_options = + runtime_state == nullptr ? TQueryOptions() : runtime_state->query_options(); + query_options.__set_enable_parquet_file_page_cache(enable_page_cache); + _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); + native_runtime_state = _page_cache_runtime_state.get(); + } + const auto& thrift_metadata = metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::ColumnReader::create( + std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, + max_buffer_size, *_offset_indexes, native_runtime_state, false, _projected_column_ids, + _filter_column_ids, page_cache_file_key, compat, + runtime_state != nullptr && runtime_state->enable_strict_mode())); + DORIS_CHECK(_native_reader != nullptr); + _skip_column = _type->create_column(); + return Status::OK(); +} + +Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, MutableColumnPtr& column, + const DataTypePtr& output_type, bool dictionary_ids, + int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(column); + DORIS_CHECK(output_type != nullptr); + DORIS_CHECK(rows_read != nullptr); + *rows_read = 0; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + ColumnPtr native_column(std::move(column)); + bool eof = false; + int64_t native_calls = 0; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + ++native_calls; + size_t loop_rows = 0; + RETURN_IF_ERROR(_native_reader->read_column_data( + native_column, output_type, _schema_node, filter, + static_cast(rows - *rows_read), &loop_rows, &eof, dictionary_ids)); + if (loop_rows == 0 && !eof) { + // A selected RowRanges plan may reject the current data page completely. V1 advances + // the page cursor and deliberately returns zero rows so the caller can request the + // next page. Bound consecutive empty transitions by the Row Group row count to retain + // a deterministic corruption exit if a decoder ever stops advancing. + if (++consecutive_empty_calls > _row_group_rows + 1) { + column = IColumn::mutate(std::move(native_column)); + return Status::Corruption("Native parquet reader made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + column = IColumn::mutate(std::move(native_column)); + if (_profile.native_read_calls != nullptr) { + COUNTER_UPDATE(_profile.native_read_calls, native_calls); + } + if (_nested && _profile.nested_batches != nullptr) { + COUNTER_UPDATE(_profile.nested_batches, 1); + } + release_batch_scratch_if_needed(); + if (*rows_read != rows) { + return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + return Status::OK(); +} + +Status NativeColumnReader::read_with_fixed_width_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, + IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(rows_read != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *rows_read = 0; + *used_filter = false; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + bool eof = false; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + size_t loop_rows = 0; + IColumn::Filter loop_filter; + bool loop_used = false; + RETURN_IF_ERROR(_native_reader->read_fixed_width_filter( + conjuncts, column_id, filter, static_cast(rows - *rows_read), + projected_column, &loop_filter, &loop_rows, &eof, &loop_used)); + if (!loop_used) { + if (UNLIKELY(*rows_read != 0)) { + // Footer encoding lists are untrusted. Once a prior page advanced the cursor, a + // typed fallback would restart the request at the wrong row, so reject the file + // instead of terminating the BE or returning shifted results. + return Status::Corruption( + "Parquet fixed-width predicate encoding changed after {} rows for column " + "{}", + *rows_read, _name); + } + row_filter->clear(); + return Status::OK(); + } + row_filter->insert(row_filter->end(), loop_filter.begin(), loop_filter.end()); + if (loop_rows == 0 && !eof) { + if (++consecutive_empty_calls > _row_group_rows + 1) { + return Status::Corruption( + "Native parquet fixed-width predicate made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + if (*rows_read != rows) { + return Status::Corruption( + "Native parquet fixed-width predicate returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + *used_filter = true; + release_batch_scratch_if_needed(); + return Status::OK(); +} + +void NativeColumnReader::release_batch_scratch_if_needed() { + // PLAIN predicate batches bypass materialization but share the same persistent decoder tree, + // so both read paths must advance the retained-capacity aging clock. + constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; + if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + _batches_since_scratch_check = 0; + } +} + +Status NativeColumnReader::validate_selected_span(int64_t rows) { + DORIS_CHECK(rows >= 0); + while (_selected_range_idx < _selected_ranges.size()) { + const auto& range = _selected_ranges[_selected_range_idx]; + const int64_t range_end = range.start + range.length; + if (_logical_row_position < range_end) { + break; + } + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + return Status::Corruption("Native parquet read past selected ranges for column {}", _name); + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start || + rows > range.start + range.length - _logical_row_position) { + return Status::Corruption( + "Native parquet read [{}, {}) crosses selected range [{}, {}) for column {}", + _logical_row_position, _logical_row_position + rows, range.start, + range.start + range.length, _name); + } + return Status::OK(); +} + +void NativeColumnReader::advance_selected_span(int64_t rows) { + _logical_row_position += rows; + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } +} + +Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(validate_selected_span(rows)); + RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); + advance_selected_span(*rows_read); + update_reader_read_rows(*rows_read); + return Status::OK(); +} + +Status NativeColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + DORIS_CHECK(_logical_row_position <= _row_group_rows - rows); + int64_t remaining = rows; + int64_t native_skipped_rows = 0; + while (remaining > 0) { + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + _logical_row_position += remaining; + break; + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start) { + const int64_t gap = std::min(remaining, range.start - _logical_row_position); + _logical_row_position += gap; + remaining -= gap; + continue; + } + const int64_t selected_rows = detail::bounded_native_lazy_skip_rows( + std::min(remaining, range.start + range.length - _logical_row_position)); + _skip_column->clear(); + // Pending skips can span many filtered batches and are replayed for every lazy column. + // Chunking here bounds each dense bitmap while preserving one logical scheduler skip. + _filter_scratch.assign(static_cast(selected_rows), 0); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, + _type, false, &rows_read)); + DORIS_CHECK(_skip_column->empty()); + DORIS_CHECK(rows_read == selected_rows); + _logical_row_position += rows_read; + native_skipped_rows += rows_read; + remaining -= rows_read; + } + update_reader_skip_rows(native_skipped_rows); + return Status::OK(); +} + +Status NativeColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column) { + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const size_t old_size = column->size(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, column, _type, + false, &rows_read)); + advance_selected_span(rows_read); + if (column->size() != old_size + selected_rows) { + return Status::Corruption( + "Native parquet selection appended {} rows, expected {} for column {}", + column->size() - old_size, selected_rows, _name); + } + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + *used_filter = false; + row_filter->clear(); + if (!_dictionary_filter_enabled) { + return Status::OK(); + } + *used_filter = true; + + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const bool nullable = _type->is_nullable(); + DataTypePtr id_type = std::make_shared(); + if (nullable) { + id_type = make_nullable(id_type); + } + if (!_dictionary_id_column) { + _dictionary_id_column = id_type->create_column(); + } + _dictionary_id_column->clear(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, + _dictionary_id_column, id_type, true, &rows_read)); + advance_selected_span(rows_read); + if (_dictionary_id_column->size() != selected_rows) { + return Status::Corruption( + "Native parquet dictionary reader appended {} rows, expected {} for {}", + _dictionary_id_column->size(), selected_rows, _name); + } + + const ColumnInt32* ids = nullptr; + const NullMap* null_map = nullptr; + if (const auto* nullable_ids = check_and_get_column(*_dictionary_id_column); + nullable_ids != nullptr) { + ids = check_and_get_column(nullable_ids->get_nested_column()); + null_map = &nullable_ids->get_null_map_data(); + } else { + ids = check_and_get_column(*_dictionary_id_column); + } + DORIS_CHECK(ids != nullptr); + + if (!_matched_dictionary_ids) { + _matched_dictionary_ids = ColumnInt32::create(); + } + _matched_dictionary_ids->clear(); + auto& matched_ids = assert_cast(*_matched_dictionary_ids).get_data(); + row_filter->reserve(selected_rows); + const auto& id_data = ids->get_data(); + for (size_t row = 0; row < selected_rows; ++row) { + bool keep = false; + if (null_map == nullptr || (*null_map)[row] == 0) { + const int32_t dictionary_id = id_data[row]; + if (dictionary_id < 0 || + static_cast(dictionary_id) >= dictionary_filter.size()) { + return Status::Corruption( + "Invalid parquet dictionary id {} for column {} with {} entries", + dictionary_id, _name, dictionary_filter.size()); + } + keep = dictionary_filter[static_cast(dictionary_id)] != 0; + if (keep) { + matched_ids.push_back(dictionary_id); + } + } + row_filter->push_back(keep ? 1 : 0); + } + + const auto* matched_id_column = check_and_get_column(*_matched_dictionary_ids); + DORIS_CHECK(matched_id_column != nullptr); + auto matched_values = + DORIS_TRY(_native_reader->materialize_dictionary_values(matched_id_column, _type)); + RETURN_IF_ERROR(append_non_null_dictionary_values(column, std::move(matched_values))); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(cast_set(matched_ids.size())); + update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); + return Status::OK(); +} + +Status NativeColumnReader::select_with_fixed_width_filter( + const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_fixed_width_filter(batch_rows, filter_data, selected_rows == 0, + conjuncts, column_id, projected_column, row_filter, + &rows_read, used_filter)); + if (!*used_filter) { + return Status::OK(); + } + DORIS_CHECK_EQ(rows_read, batch_rows); + if (row_filter->size() != selected_rows) { + return Status::Corruption( + "Native parquet fixed-width predicate returned {} selected rows, expected {} for " + "{}", + row_filter->size(), selected_rows, _name); + } + advance_selected_span(rows_read); + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +void NativeColumnReader::flush_profile() { + record_page_fragments(sync_native_profile()); +} + +bool NativeColumnReader::crossed_page_since_last_batch() { + if (_native_reader == nullptr) { + return false; + } + const auto stats = _native_reader->column_statistics(); + bool crossed_page = false; + if (stats.leaf_page_read_counters.size() == _batch_leaf_page_read_counters.size()) { + for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { + crossed_page |= + stats.leaf_page_read_counters[leaf] - _batch_leaf_page_read_counters[leaf] > 1; + } + } else { + // The first snapshot covers the first batch; later snapshots must keep the stable tree shape. + for (const int64_t page_reads : stats.leaf_page_read_counters) { + crossed_page |= page_reads > 1; + } + } + _batch_leaf_page_read_counters = stats.leaf_page_read_counters; + return crossed_page; +} + +Result NativeColumnReader::dictionary_values() { + DORIS_CHECK(_native_reader != nullptr); + return _native_reader->dictionary_values(_type); +} + +void NativeColumnReader::record_page_fragments(int64_t page_fragments) { + if (_profile.native_page_fragments != nullptr) { + COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); + } +} + +int64_t NativeColumnReader::sync_native_profile() { + if (_native_reader == nullptr) { + return 0; + } + const auto stats = _native_reader->column_statistics(); + const auto& reported = _reported_native_stats; + if (_profile.decompress_time != nullptr) { + COUNTER_UPDATE(_profile.decompress_time, stats.decompress_time - reported.decompress_time); + } + if (_profile.decompress_count != nullptr) { + COUNTER_UPDATE(_profile.decompress_count, stats.decompress_cnt - reported.decompress_cnt); + } + if (_profile.decode_header_time != nullptr) { + COUNTER_UPDATE(_profile.decode_header_time, + stats.decode_header_time - reported.decode_header_time); + } + if (_profile.decode_value_time != nullptr) { + COUNTER_UPDATE(_profile.decode_value_time, + stats.decode_value_time - reported.decode_value_time); + } + if (_profile.decode_dictionary_time != nullptr) { + COUNTER_UPDATE(_profile.decode_dictionary_time, + stats.decode_dict_time - reported.decode_dict_time); + } + if (_profile.decode_level_time != nullptr) { + COUNTER_UPDATE(_profile.decode_level_time, + stats.decode_level_time - reported.decode_level_time); + } + if (_profile.decode_null_map_time != nullptr) { + COUNTER_UPDATE(_profile.decode_null_map_time, + stats.decode_null_map_time - reported.decode_null_map_time); + } + if (_profile.materialization_time != nullptr) { + COUNTER_UPDATE(_profile.materialization_time, + stats.materialization_time - reported.materialization_time); + } + if (_profile.hybrid_selection_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_batches, + stats.hybrid_selection_batches - reported.hybrid_selection_batches); + } + if (_profile.hybrid_selection_ranges != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_ranges, + stats.hybrid_selection_ranges - reported.hybrid_selection_ranges); + } + if (_profile.hybrid_selection_null_fallback_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_null_fallback_batches, + stats.hybrid_selection_null_fallback_batches - + reported.hybrid_selection_null_fallback_batches); + } + if (_profile.page_index_read_calls != nullptr) { + COUNTER_UPDATE(_profile.page_index_read_calls, + stats.page_index_read_calls - reported.page_index_read_calls); + } + if (_profile.skip_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.skip_page_header_count, + stats.skip_page_header_num - reported.skip_page_header_num); + } + if (_profile.parse_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.parse_page_header_count, + stats.parse_page_header_num - reported.parse_page_header_num); + } + if (_profile.read_page_header_time != nullptr) { + COUNTER_UPDATE(_profile.read_page_header_time, + stats.read_page_header_time - reported.read_page_header_time); + } + const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; + if (_profile.page_read_count != nullptr) { + COUNTER_UPDATE(_profile.page_read_count, page_read_delta); + } + if (_profile.page_cache_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_write_count, + stats.page_cache_write_counter - reported.page_cache_write_counter); + } + if (_profile.page_cache_compressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_write_count, + stats.page_cache_compressed_write_counter - + reported.page_cache_compressed_write_counter); + } + if (_profile.page_cache_decompressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_write_count, + stats.page_cache_decompressed_write_counter - + reported.page_cache_decompressed_write_counter); + } + if (_profile.page_cache_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_hit_count, + stats.page_cache_hit_counter - reported.page_cache_hit_counter); + } + if (_profile.page_cache_miss_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_miss_count, + stats.page_cache_missing_counter - reported.page_cache_missing_counter); + } + if (_profile.page_cache_compressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_hit_count, + stats.page_cache_compressed_hit_counter - + reported.page_cache_compressed_hit_counter); + } + if (_profile.page_cache_decompressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_hit_count, + stats.page_cache_decompressed_hit_counter - + reported.page_cache_decompressed_hit_counter); + } + _reported_native_stats = stats; + return page_read_delta; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h new file mode 100644 index 00000000000000..1d92735bfa19d3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format_v2/column_data.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/native/column_reader.h" + +namespace doris { +class RuntimeState; +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { + +class NativeParquetMetadata; + +namespace detail { +inline constexpr int64_t MAX_NATIVE_LAZY_SKIP_ROWS = std::numeric_limits::max(); + +inline int64_t bounded_native_lazy_skip_rows(int64_t rows) { + return std::min(rows, MAX_NATIVE_LAZY_SKIP_ROWS); +} +} // namespace detail + +// Production adapter from FileScannerV2's selection-oriented reader contract to Doris' native +// Parquet page/encoding reader. The owned native reader decodes page bytes directly into the final +// Doris column. It never creates an Arrow Array/Builder, DecodedColumnView, or intermediate nested +// values_column. +// +// Cursor contract: +// - read(rows) consumes and appends exactly `rows` logical top-level rows; +// - select(selection, batch_rows) consumes `batch_rows` and appends only selected rows; +// - skip(rows) consumes `rows` with an all-false FilterMap and appends no payload; +// - one adapter lives for one top-level column in one Row Group, so decoder dictionaries, +// decompression buffers, level buffers, converters, and destination capacity survive adaptive +// batch-size changes. +class NativeColumnReader final : public ParquetColumnReader { +public: + static Status create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, io::FileReaderSPtr file, + const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, + RuntimeState* runtime_state, bool enable_page_cache, + const std::string& page_cache_file_key, bool enable_dictionary_filter, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + ~NativeColumnReader() override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + MutableColumnPtr& column) override; + Status select_with_dictionary_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + bool* used_filter) override; + Status select_with_fixed_width_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, const VExprSPtrs& conjuncts, + int column_id, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) override; + void flush_profile() override; + bool crossed_page_since_last_batch() override; + Result dictionary_values() override; + +private: + NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, + ParquetColumnReaderProfile profile); + + Status init(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter); + + Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + MutableColumnPtr& column, const DataTypePtr& output_type, + bool dictionary_ids, int64_t* rows_read); + Status read_with_fixed_width_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter); + void release_batch_scratch_if_needed(); + int64_t sync_native_profile(); + void record_page_fragments(int64_t page_fragments); + Status validate_selected_span(int64_t rows); + void advance_selected_span(int64_t rows); + + // Native ParquetColumnReader keeps a reference to RowRanges; declare it before the reader. + segment_v2::RowRanges _row_ranges; + std::set _projected_column_ids; + std::set _filter_column_ids; + const std::unordered_map* _offset_indexes = nullptr; + std::shared_ptr _schema_node; + std::unique_ptr _native_reader; + std::unique_ptr _page_cache_runtime_state; + std::vector _selected_ranges; + size_t _selected_range_idx = 0; + int64_t _logical_row_position = 0; + int64_t _row_group_rows = 0; + + bool _dictionary_filter_enabled = false; + bool _nested = false; + // The native tree exposes cumulative statistics. Keep the last reported snapshot so each + // FileScannerV2 batch contributes only its delta to RuntimeProfile. + native::ColumnReader::ColumnStatistics _reported_native_stats; + // Page-crossing is sampled at every scheduler batch, independently of amortized profile flushes. + std::vector _batch_leaf_page_read_counters; + std::vector _filter_scratch; + size_t _batches_since_scratch_check = 0; + MutableColumnPtr _skip_column; + MutableColumnPtr _dictionary_id_column; + MutableColumnPtr _matched_dictionary_ids; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp deleted file mode 100644 index e06b7eaaf317e7..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/nested_column_materializer.h" - -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" - -namespace doris::format::parquet { - -ColumnArray* array_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnMap* map_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return &nullable_column->get_null_map_data(); - } - return nullptr; -} - -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts) { - offsets.reserve(offsets.size() + entry_counts.size()); - uint64_t current_offset = offsets.empty() ? 0 : offsets.back(); - for (const auto entry_count : entry_counts) { - current_offset += entry_count; - offsets.push_back(current_offset); - } -} - -void append_parent_nulls(NullMap* dst, const NullMap& src) { - if (dst == nullptr) { - return; // target column is not nullable; no null marker is needed - } - dst->insert(src.begin(), src.end()); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.h b/be/src/format_v2/parquet/reader/nested_column_materializer.h deleted file mode 100644 index 90fac01eb2f5e5..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/column/column.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_struct.h" - -namespace doris::format::parquet { - -// ============================================================================ -// ============================================================================ - -ColumnArray* array_column_from_output(MutableColumnPtr& column); - -ColumnMap* map_column_from_output(MutableColumnPtr& column); - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column); - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column); - -// offsets[i] = offsets[i-1] + entry_counts[i]. -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts); - -void append_parent_nulls(NullMap* dst, const NullMap& src); - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp deleted file mode 100644 index fd261ef5219d27..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ /dev/null @@ -1,803 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "core/string_ref.h" -#include "runtime/runtime_profile.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { - switch (time_unit) { - case ParquetTimeUnit::MILLIS: - return DecodedTimeUnit::MILLIS; - case ParquetTimeUnit::MICROS: - return DecodedTimeUnit::MICROS; - case ParquetTimeUnit::NANOS: - return DecodedTimeUnit::NANOS; - case ParquetTimeUnit::UNKNOWN: - default: - return DecodedTimeUnit::UNKNOWN; - } -} - -Status decoded_fixed_value_size(const std::string& column_name, DecodedValueKind value_kind, - size_t* value_size) { - switch (value_kind) { - case DecodedValueKind::BOOL: - *value_size = sizeof(bool); - return Status::OK(); - case DecodedValueKind::INT32: - *value_size = sizeof(int32_t); - return Status::OK(); - case DecodedValueKind::UINT32: - *value_size = sizeof(uint32_t); - return Status::OK(); - case DecodedValueKind::INT64: - *value_size = sizeof(int64_t); - return Status::OK(); - case DecodedValueKind::UINT64: - *value_size = sizeof(uint64_t); - return Status::OK(); - case DecodedValueKind::INT96: - *value_size = 12; - return Status::OK(); - case DecodedValueKind::FLOAT: - *value_size = sizeof(float); - return Status::OK(); - case DecodedValueKind::DOUBLE: - *value_size = sizeof(double); - return Status::OK(); - case DecodedValueKind::BINARY: - case DecodedValueKind::FIXED_BINARY: - return Status::InvalidArgument("Parquet binary value kind has no fixed value size for {}", - column_name); - } - return Status::InternalError("Unknown decoded value kind for column {}", column_name); -} - -Status get_binary_chunks(const std::string& column_name, - ::parquet::internal::RecordReader& record_reader, - std::vector>* chunks) { - if (auto* dictionary_reader = - dynamic_cast<::parquet::internal::DictionaryRecordReader*>(&record_reader); - dictionary_reader != nullptr) { - auto chunked = dictionary_reader->GetResult(); - if (chunked == nullptr) { - return Status::Corruption( - "Parquet dictionary record reader returned null result for column {}", - column_name); - } - *chunks = chunked->chunks(); - return Status::OK(); - } - auto* binary_reader = dynamic_cast<::parquet::internal::BinaryRecordReader*>(&record_reader); - if (binary_reader == nullptr) { - return Status::InternalError("Parquet binary record reader is not available for column {}", - column_name); - } - *chunks = binary_reader->GetBuilderChunks(); - return Status::OK(); -} - -Status append_dictionary_binary_values(const std::string& column_name, - const ::arrow::DictionaryArray& dictionary_array, - std::vector* values) { - DORIS_CHECK(values != nullptr); - const auto& dictionary = dictionary_array.dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - column_name); - } - auto append_value = [&](int64_t dictionary_index) -> Status { - if (dictionary_index < 0 || dictionary_index >= dictionary->length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(dictionary.get())) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(dictionary.get())) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back( - reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); - }; - for (int64_t row_idx = 0; row_idx < dictionary_array.length(); ++row_idx) { - if (dictionary_array.IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - RETURN_IF_ERROR(append_value(dictionary_array.GetValueIndex(row_idx))); - } - return Status::OK(); -} - -Status build_binary_values(const std::string& column_name, - const std::vector>& chunks, - int64_t records_read, const NullMap* null_map, - bool read_dense_for_nullable, std::vector* binary_values) { - std::vector compact_values; - auto* values = read_dense_for_nullable ? &compact_values : binary_values; - values->reserve(records_read); - for (const auto& chunk : chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < binary_array->length(); ++row_idx) { - if (binary_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(row_idx, &length); - values->emplace_back(reinterpret_cast(value), length); - } - } else if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < fixed_array->length(); ++row_idx) { - if (fixed_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(row_idx)), - fixed_array->byte_width()); - } - } else if (auto* dictionary_array = dynamic_cast<::arrow::DictionaryArray*>(chunk.get())) { - RETURN_IF_ERROR( - append_dictionary_binary_values(column_name, *dictionary_array, values)); - } else { - return Status::InternalError("Unexpected Arrow binary array type for column {}", - column_name); - } - } - if (read_dense_for_nullable) { - if (null_map == nullptr || null_map->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid dense nullable parquet null map for column {}: rows={}, null_map={}", - column_name, records_read, null_map == nullptr ? 0 : null_map->size()); - } - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (compact_values.size() != static_cast(non_null_count)) { - return Status::Corruption( - "Invalid dense nullable parquet binary values for column {}: values={}, " - "records={}, nulls={}", - column_name, compact_values.size(), records_read, - records_read - non_null_count); - } - binary_values->reserve(records_read); - size_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - binary_values->emplace_back(static_cast(nullptr), 0); - continue; - } - binary_values->emplace_back(compact_values[value_idx++]); - } - return Status::OK(); - } - if (binary_values->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid parquet binary record read result for column {}: rows={}, records={}", - column_name, binary_values->size(), records_read); - } - return Status::OK(); -} - -float half_to_float(uint16_t value) { - const uint32_t sign = (value & 0x8000U) << 16; - const uint32_t exponent = (value & 0x7C00U) >> 10; - const uint32_t mantissa = value & 0x03FFU; - - if (exponent == 0) { - if (mantissa == 0) { - return std::bit_cast(sign); - } - const float subnormal = std::ldexp(static_cast(mantissa), -24); - return sign == 0 ? subnormal : -subnormal; - } - if (exponent == 0x1FU) { - return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); - } - return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); -} - -Status build_float16_values(const std::string& column_name, - const ParquetTypeDescriptor& type_descriptor, - const std::vector& binary_values, int64_t row_count, - std::vector* float_values) { - if (type_descriptor.fixed_length != 2) { - return Status::Corruption("Invalid parquet Float16 length for column {}: {}", column_name, - type_descriptor.fixed_length); - } - if (binary_values.size() != static_cast(row_count)) { - return Status::Corruption( - "Invalid parquet Float16 value count for column {}: values={}, rows={}", - column_name, binary_values.size(), row_count); - } - float_values->resize(static_cast(row_count)); - for (int64_t row = 0; row < row_count; ++row) { - const auto& binary_value = binary_values[static_cast(row)]; - if (binary_value.data == nullptr && binary_value.size == 0) { - (*float_values)[static_cast(row)] = 0; - continue; - } - if (binary_value.data == nullptr || binary_value.size != 2) { - return Status::Corruption( - "Invalid parquet Float16 value for column {} at row {}: data={}, size={}", - column_name, row, binary_value.data == nullptr ? "null" : "non-null", - binary_value.size); - } - uint16_t raw_value = 0; - std::memcpy(&raw_value, binary_value.data, sizeof(raw_value)); - (*float_values)[static_cast(row)] = half_to_float(raw_value); - } - return Status::OK(); -} - -} // namespace - -Status ParquetLeafReader::collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - batch->_values_written = record_reader.values_written(); - - if (!batch->is_binary_value()) { - batch->_fixed_values = record_reader.values(); - return Status::OK(); - } - - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &batch->_binary_chunks)); - batch->_values_written = 0; - for (const auto& chunk : batch->_binary_chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", _name); - } - batch->_values_written += chunk->length(); - } - return Status::OK(); -} - -Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - - // Arrow's RecordReader::Reset() does not reset ByteArray/FLBA builders. GetBuilderChunks() - // (or DictionaryRecordReader::GetResult()) is the documented reset operation and must be - // called before the next ReadRecords(). Otherwise a levels-only skip followed by a normal read - // observes values from both batches; for example, skipping ARRAY ["a", "b"] and then - // reading ["c"] would report one current level but three values. Release the chunks here and - // let the temporary vector destroy them immediately. We deliberately do not inspect or copy - // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value - // materialization. - if (batch->is_binary_value()) { - std::vector> discarded_chunks; - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &discarded_chunks)); - } - - // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the - // RecordReader and are cleared by Reset(); binary values were released above solely to reset - // the Arrow builder. - batch->_values_written = 0; - return Status::OK(); -} - -// - FLOAT16: binary -> half_to_float -> float_values -Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, MutableColumnPtr& column) const { - std::vector binary_values; - std::vector spaced_values; - std::vector float_values; - DecodedColumnView view; - view.value_kind = batch._value_kind; - view.time_unit = decoded_time_unit(_type_descriptor.time_unit); - view.row_count = row_count; - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.decimal_precision = _type_descriptor.decimal_precision; - view.decimal_scale = _type_descriptor.decimal_scale; - view.fixed_length = _type_descriptor.fixed_length; - view.timestamp_is_adjusted_to_utc = _type_descriptor.timestamp_is_adjusted_to_utc; - view.timezone = _timezone; - view.enable_strict_mode = _enable_strict_mode; - view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); - const bool read_dense_for_nullable = batch._read_dense_for_nullable && view.null_map != nullptr; - - if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, binary_values, row_count, - &float_values)); - view.value_kind = DecodedValueKind::FLOAT; - view.values = reinterpret_cast(float_values.data()); - } else if (batch.is_binary_value()) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - view.binary_values = &binary_values; - } else if (read_dense_for_nullable) { - RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &spaced_values)); - view.values = spaced_values.data(); - } else { - view.values = batch._fixed_values; - } - - if (_decoded_value_appender != nullptr) { - return _decoded_value_appender(column, view); - } - - { - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& tmp_null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = tmp_null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - tmp_null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(tmp_null_map.data() + old_null_map_size, 0, - tmp_null_map.size() - old_null_map_size); - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } - return Status::OK(); -} - -bool ParquetLeafBatch::is_binary_value() const { - return _value_kind == DecodedValueKind::BINARY || _value_kind == DecodedValueKind::FIXED_BINARY; -} - -Status ParquetLeafReader::build_spaced_fixed_values(const ParquetLeafBatch& batch, - int64_t row_count, const NullMap* null_map, - std::vector* spaced_values) const { - DORIS_CHECK(null_map != nullptr); - DORIS_CHECK(spaced_values != nullptr); - size_t value_size = 0; - RETURN_IF_ERROR(decoded_fixed_value_size(_name, batch._value_kind, &value_size)); - spaced_values->resize(static_cast(row_count) * value_size); - const auto non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (batch._values_written != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet values for column {}: values={}, records={}, " - "nulls={}", - _name, batch._values_written, row_count, row_count - non_null_count); - } - auto* dst = spaced_values->data(); - int64_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < row_count; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - continue; // NULL row: skip it and keep the target slot zeroed - } - std::memcpy(dst + static_cast(record_idx) * value_size, - batch._fixed_values + static_cast(value_idx) * value_size, value_size); - ++value_idx; - } - return Status::OK(); -} - -ParquetLeafReader::ParquetLeafReader( - const ::parquet::ColumnDescriptor* descriptor, ParquetTypeDescriptor type_descriptor, - DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile, const cctz::time_zone* timezone, - bool enable_strict_mode, - std::function decoded_value_appender) - : _descriptor(descriptor), - _type_descriptor(type_descriptor), - _type(std::move(type)), - _name(std::move(name)), - _record_reader(std::move(record_reader)), - _profile(profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _decoded_value_appender(std::move(decoded_value_appender)) {} - -Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, - int64_t* rows_read) const { - if (batch == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet leaf batch result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - *rows_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet records for column {}: {}", _name, - e.what()); - } - if (*rows_read < 0 || *rows_read > batch_rows) { - return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, - *rows_read); - } - return collect_batch(*_record_reader, batch); -} - -Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const { - if (_descriptor->max_definition_level() == 0) { - return Status::OK(); - } - auto* def_levels = batch.def_levels(); - if (def_levels == nullptr && records_read > 0) { - return Status::Corruption( - "Parquet record reader returned null definition levels for nullable column {}", - _name); - } - const int16_t max_definition_level = _descriptor->max_definition_level(); - null_map->resize(records_read); - auto* __restrict dst = null_map->data(); - const auto* __restrict src = def_levels; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - dst[record_idx] = src[record_idx] != max_definition_level; - } - return Status::OK(); -} - -Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - ParquetLeafBatch leaf_batch; - int64_t records_read = 0; - RETURN_IF_ERROR(read_batch(batch_rows, &leaf_batch, &records_read)); - return build_nested_batch_from_leaf_batch(leaf_batch, records_read, value_slot_definition_level, - batch, value_slot_repetition_level); -} - -Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - int64_t records_read = 0; - ParquetLeafBatch leaf_batch; - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - records_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } - if (records_read < 0 || records_read > batch_rows) { - return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, - records_read); - } - RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &leaf_batch)); - return build_nested_levels_batch_from_leaf_batch(leaf_batch, records_read, batch); -} - -Status ParquetLeafReader::build_nested_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->value_slot_definition_level = value_slot_definition_level; - batch->value_slot_repetition_level = value_slot_repetition_level; - - batch->records_read = records_read; - if (_type->is_nullable() && leaf_batch.read_dense_for_nullable()) { - return Status::NotSupported( - "Dense nullable parquet nested reader is not supported for column {}", _name); - } - batch->levels_written = leaf_batch.consumed_level_count(); - const int64_t values_written = leaf_batch.values_written(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - if (batch->levels_written == 0 && batch->records_read > 0 && - values_written == batch->records_read && _descriptor->max_definition_level() == 0 && - _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read || values_written < 0 || - values_written > batch->levels_written) { - return Status::Corruption( - "Invalid nested parquet read result for column {}: rows={}, levels={}, values={}", - _name, batch->records_read, batch->levels_written, values_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - - const int16_t leaf_definition_level = _descriptor->max_definition_level(); - // Arrow's RecordReader may emit value placeholders for null ancestors that are below the - // Doris materialization threshold. Those slots must still advance the payload value index; - // otherwise the next defined child level points at the placeholder instead of its real value. - auto count_value_slots = [&](int16_t slot_definition_level) { - int64_t slot_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] >= slot_definition_level && - batch->rep_levels[level_idx] <= value_slot_repetition_level) { - ++slot_count; - } - } - return slot_count; - }; - - const int64_t value_slot_count = count_value_slots(value_slot_definition_level); - int16_t payload_slot_definition_level = value_slot_definition_level; - int64_t payload_value_slot_count = value_slot_count; - while (payload_slot_definition_level > 0 && payload_value_slot_count < values_written) { - --payload_slot_definition_level; - payload_value_slot_count = count_value_slots(payload_slot_definition_level); - } - - int64_t leaf_value_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < value_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - if (batch->def_levels[level_idx] == leaf_definition_level) { - ++leaf_value_count; - } - } - - enum class ValueLayout { LEVELS, VALUE_SLOTS, LEAF_VALUES, PAYLOAD_VALUE_SLOTS }; - ValueLayout value_layout = ValueLayout::LEAF_VALUES; - if (values_written == batch->levels_written) { - value_layout = ValueLayout::LEVELS; - } else if (values_written == value_slot_count) { - value_layout = ValueLayout::VALUE_SLOTS; - } else if (values_written == leaf_value_count) { - value_layout = ValueLayout::LEAF_VALUES; - } else if (values_written == payload_value_slot_count) { - value_layout = ValueLayout::PAYLOAD_VALUE_SLOTS; - } else { - return Status::Corruption( - "Nested parquet reader returned inconsistent value count for column {}: values={}, " - "levels={}, slots={}, leaf_values={}, payload_slots={}, " - "payload_slot_definition_level={}", - _name, values_written, batch->levels_written, value_slot_count, leaf_value_count, - payload_value_slot_count, payload_slot_definition_level); - } - - batch->value_indices.resize(static_cast(batch->levels_written), -1); - NullMap value_nulls(static_cast(values_written), 1); - int64_t value_idx = 0; - const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS - ? payload_slot_definition_level - : value_slot_definition_level; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < decoded_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - const bool has_leaf_value = batch->def_levels[level_idx] == leaf_definition_level; - int64_t decoded_value_idx = -1; - if (value_layout == ValueLayout::LEVELS) { - decoded_value_idx = level_idx; - } else if (value_layout == ValueLayout::VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else if (value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else { - if (!has_leaf_value) { - continue; - } - decoded_value_idx = value_idx++; - } - DORIS_CHECK(decoded_value_idx >= 0); - DORIS_CHECK(decoded_value_idx < values_written); - if (has_leaf_value) { - batch->value_indices[static_cast(level_idx)] = decoded_value_idx; - value_nulls[static_cast(decoded_value_idx)] = 0; - } - } - if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { - return Status::Corruption( - "Nested parquet reader value cursor stopped early for column {}: values={}, " - "visited={}", - _name, values_written, value_idx); - } - - const auto value_type = remove_nullable(_type); - batch->values_column = value_type->create_column(); - if (values_written > 0) { - ParquetLeafReader value_reader(_descriptor, _type_descriptor, value_type, _name, - _record_reader, _profile, _timezone, _enable_strict_mode); - RETURN_IF_ERROR(value_reader.append_values(leaf_batch, values_written, &value_nulls, - batch->values_column)); - } - return Status::OK(); -} - -Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->records_read = records_read; - batch->levels_written = leaf_batch.consumed_level_count(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - - // Required flat leaves do not have physical def/rep level buffers. Synthesize one level slot - // per top-level row so the COUNT(col) aggregation code can use the same shape loop. - if (batch->levels_written == 0 && batch->records_read > 0 && - _descriptor->max_definition_level() == 0 && _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read) { - return Status::Corruption( - "Invalid nested parquet levels result for column {}: rows={}, levels={}", _name, - batch->records_read, batch->levels_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - return Status::OK(); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h deleted file mode 100644 index b396b35fd1f32c..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ /dev/null @@ -1,173 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/status.h" -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace arrow { -class Array; -} // namespace arrow - -namespace doris::format::parquet { - -struct ParquetLeafReaderTestAccess; - -// Read result for a nested scalar leaf, separating Dremel-encoded shape from actual values. -// The COUNT(col) aggregation fast path consumes only records_read, levels_written, def_levels, and rep_levels. -// That path does not populate value_indices or values_column, so callers must not call build_nested_column() afterwards. -struct ParquetNestedScalarBatch { - int64_t records_read = 0; - int64_t levels_written = 0; - int16_t value_slot_definition_level = 0; - int16_t value_slot_repetition_level = std::numeric_limits::max(); - std::vector def_levels; - std::vector rep_levels; - std::vector value_indices; - MutableColumnPtr values_column; - - bool empty() const { return levels_written == 0; } -}; - -class ParquetLeafBatch { -public: - int64_t consumed_level_count() const { return _consumed_level_count; } - int64_t decoded_level_count() const { return _decoded_level_count; } - int64_t values_written() const { return _values_written; } - bool read_dense_for_nullable() const { return _read_dense_for_nullable; } - const int16_t* def_levels() const { return _def_levels; } - const int16_t* rep_levels() const { return _rep_levels; } - const std::vector>& binary_chunks() const { - return _binary_chunks; - } - -private: - friend class ParquetLeafReader; - - bool is_binary_value() const; - - DecodedValueKind _value_kind = DecodedValueKind::INT32; - int64_t _consumed_level_count = 0; - int64_t _decoded_level_count = 0; - int64_t _values_written = 0; - const int16_t* _def_levels = nullptr; - const int16_t* _rep_levels = nullptr; - const uint8_t* _fixed_values = nullptr; - bool _read_dense_for_nullable = false; - std::vector> _binary_chunks; -}; - -// read_batch() -> build_null_map() + append_values() -// read_nested_batch() -class ParquetLeafReader { -public: - ParquetLeafReader(const ::parquet::ColumnDescriptor* descriptor, - ParquetTypeDescriptor type_descriptor, DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile = {}, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - std::function - decoded_value_appender = nullptr); - - Status read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const; - - Status build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const; - - Status append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, - MutableColumnPtr& column) const; - - // LEVELS / VALUE_SLOTS / LEAF_VALUES / PAYLOAD_VALUE_SLOTS. - Status read_nested_batch( - int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level = std::numeric_limits::max()) const; - - // COUNT(col) and nested-skip shape-only read path. It still calls Arrow - // RecordReader::ReadRecords() to advance the Parquet cursor and obtain def/rep levels, but - // Doris only copies levels: - // - it does not build value_indices or values_column - // - it does not enter DataTypeSerde::read_column_from_decoded_values() - // - for Binary/FLBA, it releases and immediately discards Arrow builder chunks because that is - // the RecordReader's required reset operation; it never copies them into a Doris Column - // This lets COUNT(col) on MAP/ARRAY/STRUCT evaluate top-level NULL state and lets skip advance - // nested shape without Doris-side STRING/BINARY materialization. Arrow RecordReader does not - // expose a public levels-only API, so ReadRecords may still perform required page decoding. - Status read_nested_levels_batch(int64_t batch_rows, ParquetNestedScalarBatch* batch) const; - -private: - friend struct ParquetLeafReaderTestAccess; - - Status collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - // Levels-only variant of collect_batch(). It snapshots only def/rep level state and does not - // expose value buffers. Binary chunks are released only to reset Arrow's builder and are - // immediately discarded. Used by COUNT(col) and nested skip. - Status collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - Status build_spaced_fixed_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, - std::vector* spaced_values) const; - - Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const; - Status build_nested_levels_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - ParquetNestedScalarBatch* batch) const; - - const ::parquet::ColumnDescriptor* _descriptor = - nullptr; // Arrow column descriptor (physical_type, max_dl, max_rl) - ParquetTypeDescriptor - _type_descriptor; // type encoding information (decimal precision, timestamp unit, etc.) - DataTypePtr _type; // Doris target type - std::string _name; // column name for error messages - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader (shared ownership) - ParquetColumnReaderProfile _profile; // profile counters - const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion - bool _enable_strict_mode = false; // strict mode for type mismatch errors - std::function _decoded_value_appender; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp index 4e9a363b13c7cb..dcd601b003ae3c 100644 --- a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp @@ -20,6 +20,7 @@ #include "core/assert_cast.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" namespace doris::format::parquet { diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp deleted file mode 100644 index 520a3cb1e62e3b..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ /dev/null @@ -1,584 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/scalar_column_reader.h" - -#include -#include -#include - -#include -#include -#include - -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -class ParquetNestedScalarValueCursor { -public: - explicit ParquetNestedScalarValueCursor(const ParquetNestedScalarBatch* batch) { reset(batch); } - - void reset(const ParquetNestedScalarBatch* batch) { - DORIS_CHECK(batch != nullptr); - _batch = batch; - } - - Status value_index(const std::string& column_name, int64_t level_idx, int64_t* value_idx) { - DORIS_CHECK(_batch != nullptr); - DORIS_CHECK(value_idx != nullptr); - DORIS_CHECK(level_idx < _batch->levels_written); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(static_cast(level_idx) < _batch->value_indices.size()); - const int64_t computed_value_idx = _batch->value_indices[static_cast(level_idx)]; - if (computed_value_idx < 0) { - return Status::Corruption("Nested parquet value is absent for column {}", column_name); - } - DORIS_CHECK(_batch->values_column.get() != nullptr); - if (computed_value_idx >= _batch->values_column->size()) { - return Status::Corruption("Nested parquet value index is out of range for column {}", - column_name); - } - *value_idx = computed_value_idx; - return Status::OK(); - } - -private: - const ParquetNestedScalarBatch* _batch = nullptr; -}; - -Status append_scalar_batch_value(const ScalarColumnReader& column_reader, - const ParquetNestedScalarBatch& batch, int64_t level_idx, - ParquetNestedScalarValueCursor* value_cursor, - MutableColumnPtr& column) { - DORIS_CHECK(value_cursor != nullptr); - int64_t value_idx = -1; - RETURN_IF_ERROR(value_cursor->value_index(column_reader.name(), level_idx, &value_idx)); - auto* nullable_column = check_and_get_column(*column); - if (nullable_column != nullptr) { - nullable_column->get_nested_column().insert_from(*batch.values_column, - static_cast(value_idx)); - nullable_column->get_null_map_data().push_back(0); - return Status::OK(); - } - column->insert_from(*batch.values_column, static_cast(value_idx)); - return Status::OK(); -} - -Status append_arrow_binary_dictionary_value(const std::string& column_name, - const ::arrow::Array& dictionary, - int64_t dictionary_index, - std::vector* values) { - DORIS_CHECK(values != nullptr); - if (dictionary_index < 0 || dictionary_index >= dictionary.length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast(&dictionary)) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast(&dictionary)) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); -} - -} // namespace - -ScalarColumnReader::ScalarColumnReader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile profile) - : ParquetColumnReader(column_schema, column_schema.type, profile), - _descriptor(column_schema.descriptor), - _type_descriptor(column_schema.type_descriptor), - _record_reader(std::move(record_reader)), - _page_skip_plan(page_skip_plan), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _nested_batch(std::make_unique()) {} - -ScalarColumnReader::~ScalarColumnReader() = default; - -Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet column read result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - auto reader = leaf_reader(); - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(reader.read_batch(rows, &leaf_batch, rows_read)); - - NullMap null_map; - RETURN_IF_ERROR(reader.build_null_map(leaf_batch, *rows_read, &null_map)); - const auto value_kind = decoded_value_kind(_type_descriptor); - const bool is_binary_value = - value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; - if (!is_binary_value && leaf_batch.read_dense_for_nullable() && !null_map.empty()) { - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map.data()), null_map.size())); - const int64_t null_count = *rows_read - non_null_count; - if (leaf_batch.values_written() != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet record read result for column {}: values={}, " - "records={}, nulls={}", - _name, leaf_batch.values_written(), *rows_read, null_count); - } - } else if (!is_binary_value && !leaf_batch.read_dense_for_nullable() && - leaf_batch.values_written() != *rows_read) { - return Status::Corruption( - "Invalid parquet record read result for column {}: values={}, records={}", _name, - leaf_batch.values_written(), *rows_read); - } - - RETURN_IF_ERROR(reader.append_values(leaf_batch, *rows_read, &null_map, column)); - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::skip_records(int64_t rows) { - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - if (rows <= 0) { - return Status::OK(); - } - int64_t skipped_rows = 0; - SCOPED_TIMER(_profile.arrow_skip_records_time); - try { - _record_reader->Reset(); - while (skipped_rows < rows) { - const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); - if (skipped <= 0) { - return Status::Corruption( - "Failed to skip parquet records for column {}: skipped {} of {} rows", - _name, skipped_rows, rows); - } - skipped_rows += skipped; - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -int64_t ScalarColumnReader::page_filtered_rows_to_skip(int64_t rows) const { - if (_page_skip_plan == nullptr || rows <= 0) { - return 0; - } - const int64_t skip_end = _row_group_rows_read + rows; - int64_t filtered_rows = 0; - for (const auto& range : _page_skip_plan->skipped_ranges) { - const int64_t range_end = range.start + range.length; - if (range_end <= _row_group_rows_read) { - continue; - } - if (range.start >= skip_end) { - break; - } - const int64_t start = std::max(range.start, _row_group_rows_read); - const int64_t end = std::min(range_end, skip_end); - if (start < end) { - // Scheduler gap skips are derived from page-index selected_ranges. A page-filtered - // range can only overlap such a gap when the whole data page is outside every selected - // range, so partial overlap would mean the planner and scheduler are out of sync. - DORIS_CHECK(start == range.start); - DORIS_CHECK(end == range_end); - filtered_rows += end - start; - } - } - return filtered_rows; -} - -void ScalarColumnReader::advance_rows_read(int64_t rows) { - DORIS_CHECK(rows >= 0); - _row_group_rows_read += rows; -} - -Status ScalarColumnReader::skip(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - const int64_t page_filtered_rows = page_filtered_rows_to_skip(rows); - DORIS_CHECK(page_filtered_rows <= rows); - const int64_t record_reader_skip_rows = rows - page_filtered_rows; - RETURN_IF_ERROR(skip_records(record_reader_skip_rows)); - advance_rows_read(rows); - return Status::OK(); -} - -Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& sel, - uint16_t selected_rows, int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, - bool* used_filter) { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(used_filter != nullptr); - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); - *used_filter = false; - row_filter->clear(); - row_filter->reserve(selected_rows); - DORIS_CHECK(_record_reader != nullptr); - // A clean fallback is possible only before any range skip or read advances the record reader. - // Once dictionary selection starts, losing dictionary output is corruption rather than a - // reason to retry through the ordinary selected-read path with an already advanced stream. - if (!_record_reader->read_dictionary()) { - return Status::OK(); - } - *used_filter = true; - - const auto ranges = selection_to_ranges(sel, selected_rows); - int64_t cursor = 0; - for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument( - "Invalid parquet dictionary selection range [{}, {}) for column {}", - range.start, range.start + range.length, _name); - } - RETURN_IF_ERROR(skip(range.start - cursor)); - - int64_t range_rows_read = 0; - RETURN_IF_ERROR(read_range_with_dictionary_filter(range.length, dictionary_filter, column, - row_filter, &range_rows_read, - used_filter)); - if (!*used_filter) { - return Status::OK(); - } - if (range_rows_read != range.length) { - return Status::Corruption( - "Parquet dictionary selected read returned {} rows, expected {} rows for " - "column {}", - range_rows_read, range.length, _name); - } - cursor = range.start + range.length; - } - RETURN_IF_ERROR(skip(batch_rows - cursor)); - if (_profile.reader_select_rows != nullptr) { - COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); - } - return Status::OK(); -} - -Status ScalarColumnReader::read_range_with_dictionary_filter( - int64_t rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter) { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(rows_read != nullptr); - DORIS_CHECK(used_filter != nullptr); - DORIS_CHECK(_record_reader != nullptr); - if (!_record_reader->read_dictionary()) { - return Status::Corruption( - "Parquet dictionary reader became unavailable after selected reading started for " - "column {}", - _name); - } - - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(leaf_reader().read_batch(rows, &leaf_batch, rows_read)); - int64_t matched_rows = 0; - RETURN_IF_ERROR(append_dictionary_filtered_values(leaf_batch.binary_chunks(), dictionary_filter, - column, row_filter, &matched_rows, - used_filter)); - if (!*used_filter) { - return Status::Corruption( - "Parquet dictionary reader did not return dictionary batches for column {}", _name); - } - if (row_filter->size() < static_cast(*rows_read)) { - return Status::Corruption( - "Parquet dictionary filter produced too few row decisions for column {}: " - "filter={}, rows={}", - _name, row_filter->size(), *rows_read); - } - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(matched_rows != nullptr); - DORIS_CHECK(used_filter != nullptr); - *matched_rows = 0; - *used_filter = false; - - std::vector selected_values; - for (const auto& chunk : chunks) { - DORIS_CHECK(chunk != nullptr); - const auto* dict_array = dynamic_cast(chunk.get()); - if (dict_array == nullptr) { - // The caller has already consumed rows from a DictionaryRecordReader. Falling back to a - // normal selected read would desynchronize the Parquet stream, so absence of a - // DictionaryArray is reported as corruption by read_range_with_dictionary_filter(). - return Status::OK(); - } - *used_filter = true; - const auto& dictionary = dict_array->dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - _name); - } - - // Dictionary predicates are evaluated once against the dictionary page and produce a - // dictionary-entry bitmap. DATA_PAGE rows then only need an integer-index lookup. NULL rows - // do not have a dictionary entry and cannot satisfy the supported equality/IN predicates. - for (int64_t row = 0; row < dict_array->length(); ++row) { - bool keep = false; - if (!dict_array->IsNull(row)) { - const int64_t dictionary_index = dict_array->GetValueIndex(row); - if (dictionary_index < 0 || dictionary_index >= dictionary->length() || - dictionary_index >= static_cast(dictionary_filter.size())) { - return Status::Corruption( - "Invalid parquet dictionary index {} for column {}: dictionary={}, " - "filter={}", - dictionary_index, _name, dictionary->length(), - dictionary_filter.size()); - } - keep = dictionary_filter[static_cast(dictionary_index)] != 0; - if (keep) { - RETURN_IF_ERROR(append_arrow_binary_dictionary_value( - _name, *dictionary, dictionary_index, &selected_values)); - ++*matched_rows; - } - } - row_filter->push_back(keep ? 1 : 0); - } - } - - if (!*used_filter) { - return Status::OK(); - } - return append_decoded_binary_values(selected_values, column); -} - -Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const { - DecodedColumnView view; - view.value_kind = decoded_value_kind(_type_descriptor); - view.row_count = static_cast(values.size()); - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.fixed_length = _type_descriptor.fixed_length; - view.binary_values = &values; - - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(null_map.data() + old_null_map_size, 0, null_map.size() - old_null_map_size); - return Status::OK(); - } - return _type->get_serde()->read_column_from_decoded_values(*column, view); - } - - NullMap null_map(values.size(), 0); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - return _type->get_serde()->read_column_from_decoded_values(*column, view); -} - -// The value index stream must advance on those null slots, otherwise later payload values shift. -Status ScalarColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - RETURN_IF_ERROR(leaf_reader().read_nested_batch(rows, materialized_slot_definition_level, - _nested_batch.get(), _repetition_level)); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(leaf_reader().read_nested_levels_batch(rows, _nested_batch.get())); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar build result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - // The levels-only loader intentionally does not populate value-slot metadata or payload - // buffers. Derive the logical slot threshold from the schema, exactly as load_nested_batch() - // does, so this consumer works for both loaded batch forms. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_read = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_read < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - if (def_level == _definition_level) { - RETURN_IF_ERROR(append_scalar_batch_value(*this, *_nested_batch, current_level_idx, - &value_cursor, column)); - } else { - if (!_type->is_nullable() && def_level >= _nullable_definition_level) { - return Status::Corruption( - "Parquet scalar column {} contains null for non-nullable field", _name); - } - column->insert_default(); - } - ++*values_read; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - if (values_consumed == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar consume result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - // A levels-only batch intentionally has no value-slot metadata. Reconstruct the same logical - // slot threshold used by load_nested_batch(): a nullable leaf owns a slot at one definition - // level below a non-null value, while a required leaf owns a slot only at its full definition - // level. For example, an empty ARRAY boundary must not be consumed as a STRING value. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_consumed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_consumed < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - RETURN_IF_ERROR(validate_nested_value(current_level_idx, false)); - ++*values_consumed; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::append_nested_value(int64_t level_idx, MutableColumnPtr& column) const { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar append result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return append_scalar_batch_value(*this, *_nested_batch, level_idx, &value_cursor, column); - } - if (!_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable value", - _name); - } - column->insert_default(); - return Status::OK(); -} - -Status ScalarColumnReader::validate_nested_value(int64_t level_idx, bool require_non_null) const { - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return Status::OK(); - } - if (require_non_null || !_type->is_nullable()) { - return Status::Corruption("Parquet scalar column {} contains null for non-nullable field", - _name); - } - return Status::OK(); -} - -const std::vector& ScalarColumnReader::nested_definition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->def_levels; -} - -const std::vector& ScalarColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->rep_levels; -} - -int64_t ScalarColumnReader::nested_levels_written() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->levels_written; -} - -bool ScalarColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h deleted file mode 100644 index 5342baa803eca0..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.h +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/string_ref.h" -#include "format_v2/parquet/parquet_type.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris::format::parquet { - -struct ScalarColumnReaderTestAccess; - -// load_nested_batch() / build_nested_column() -class ScalarColumnReader final : public ParquetColumnReader { - friend class MapColumnReader; - friend struct ScalarColumnReaderTestAccess; - -public: - ScalarColumnReader(const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan = nullptr, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - ParquetColumnReaderProfile profile = {}); - ~ScalarColumnReader() override; - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - bool* used_filter) override; - - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - -private: - Status append_nested_value(int64_t level_idx, MutableColumnPtr& column) const; - Status validate_nested_value(int64_t level_idx, bool require_non_null) const; - Status read_range_with_dictionary_filter(int64_t rows, const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - int64_t* rows_read, bool* used_filter); - Status append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const; - Status append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const; - - const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } - - ParquetLeafReader leaf_reader() const { - return ParquetLeafReader(_descriptor, _type_descriptor, _type, _name, _record_reader, - _profile, _timezone, _enable_strict_mode); - } - - void advance_rows_read(int64_t rows); - Status skip_records(int64_t rows); - int64_t page_filtered_rows_to_skip(int64_t rows) const; - - const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor - ParquetTypeDescriptor _type_descriptor; // type encoding information - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader - const ParquetPageSkipPlan* _page_skip_plan = - nullptr; // page-index pruning result (may be nullptr) - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) - std::unique_ptr _nested_batch; // intermediate result for nested reads -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp deleted file mode 100644 index 5abe7abe75e9a2..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/struct_column_reader.h" - -#include -#include -#include -#include - -#include "core/column/column_struct.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { - -ParquetColumnReader* StructColumnReader::shape_source_reader() const { - for (const auto& child : _children) { - auto* child_reader = child.get(); - DORIS_CHECK(child_reader != nullptr); - if (!child_reader->is_or_has_repeated_child()) { - return child_reader; - } - } - if (_children.empty()) { - return nullptr; - } - return _children[0].get(); -} - -Status StructColumnReader::advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const { - DORIS_CHECK(child_reader != nullptr); - const int64_t next_child_cursor = parent_level_idx + 1; - if (auto* scalar_child = dynamic_cast(child_reader)) { - if (next_child_cursor > scalar_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - scalar_child->name(), _name); - } - scalar_child->set_nested_build_level_cursor( - std::max(scalar_child->nested_build_level_cursor(), next_child_cursor)); - return Status::OK(); - } - if (auto* struct_child = dynamic_cast(child_reader); - struct_child != nullptr && !struct_child->is_or_has_repeated_child()) { - if (next_child_cursor > struct_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - struct_child->name(), _name); - } - struct_child->set_nested_build_level_cursor( - std::max(struct_child->nested_build_level_cursor(), next_child_cursor)); - for (auto& grandchild : struct_child->_children) { - RETURN_IF_ERROR(struct_child->advance_child_past_null_parent(grandchild.get(), - parent_level_idx)); - } - return Status::OK(); - } - - int64_t child_cursor = child_reader->nested_build_level_cursor(); - const auto& child_rep_levels = child_reader->nested_repetition_levels(); - const int64_t child_levels_written = child_reader->nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!child_reader->is_or_has_repeated_child() || child_rep_level <= _repetition_level) { - break; - } - } - child_reader->set_nested_build_level_cursor(child_cursor); - return Status::OK(); -} - -Status StructColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status StructColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status StructColumnReader::load_nested_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::load_nested_levels_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_levels_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet struct build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status StructColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument( - "Invalid parquet struct process result pointer for column {}", _name); - } - if (_children.empty()) { - if (column != nullptr) { - (*column)->resize((*column)->size() + static_cast(length_upper_bound)); - } - *values_processed = length_upper_bound; - return Status::OK(); - } - ColumnStruct* struct_column = nullptr; - NullMap* parent_null_map = nullptr; - if (column != nullptr) { - struct_column = struct_column_from_output(*column); - DORIS_CHECK(struct_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - } - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - const auto& def_levels = shape_reader->nested_definition_levels(); - const auto& rep_levels = shape_reader->nested_repetition_levels(); - const int64_t levels_written = shape_reader->nested_levels_written(); - - NullMap parent_nulls; - std::vector parent_level_indices; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < levels_written) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = - !shape_reader->is_or_has_repeated_child() || rep_level <= _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (def_level < _repeated_ancestor_definition_level) { - continue; - } - if (shape_reader->is_or_has_repeated_child() && rep_level > _repetition_level) { - continue; - } - const bool parent_is_null = def_level < _nullable_definition_level; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption( - "Parquet STRUCT column {} contains null for non-nullable struct", _name); - } - parent_nulls.push_back(parent_is_null); - parent_level_indices.push_back(current_level_idx); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - std::vector child_columns; - if (column != nullptr) { - child_columns.reserve(struct_column->get_columns().size()); - for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { - child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); - } - } - for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { - const int output_idx = _child_output_indices[child_idx]; - if (column != nullptr && output_idx < 0) { - continue; - } - // STRUCT owns row alignment. Child readers consume only present parent rows from their - // level streams; null STRUCT parents become default placeholders in every child column. - // This mirrors Arrow's separation between struct validity and child array materialization, - // and avoids asking scalar/list/map children to invent values for an absent parent. - int64_t pending_present_rows = 0; - int64_t total_child_rows = 0; - auto flush_present_rows = [&]() -> Status { - if (pending_present_rows == 0) { - return Status::OK(); - } - int64_t child_rows = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_children[child_idx]->build_nested_column( - pending_present_rows, child_columns[output_idx], &child_rows)); - } else { - RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, - &child_rows)); - } - if (child_rows != pending_present_rows) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), child_rows, pending_present_rows, _name); - } - total_child_rows += child_rows; - pending_present_rows = 0; - return Status::OK(); - }; - for (size_t parent_idx = 0; parent_idx < parent_nulls.size(); ++parent_idx) { - const auto parent_is_null = parent_nulls[parent_idx]; - if (!parent_is_null) { - ++pending_present_rows; - continue; - } - RETURN_IF_ERROR(flush_present_rows()); - if (column != nullptr) { - child_columns[output_idx]->insert_default(); - } - RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), - parent_level_indices[parent_idx])); - ++total_child_rows; - } - RETURN_IF_ERROR(flush_present_rows()); - if (total_child_rows != *values_processed) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), total_child_rows, *values_processed, _name); - } - } - if (column != nullptr) { - for (size_t child_idx = 0; child_idx < child_columns.size(); ++child_idx) { - struct_column->get_column_ptr(child_idx) = std::move(child_columns[child_idx]); - } - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& StructColumnReader::nested_definition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_definition_levels(); -} - -const std::vector& StructColumnReader::nested_repetition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_repetition_levels(); -} - -int64_t StructColumnReader::nested_levels_written() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_levels_written(); -} - -bool StructColumnReader::is_or_has_repeated_child() const { - auto* shape_reader = shape_source_reader(); - return shape_reader != nullptr && shape_reader->is_or_has_repeated_child(); -} - -void StructColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - for (auto& child : _children) { - DORIS_CHECK(child != nullptr); - child->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - } -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h deleted file mode 100644 index 3c2d6904cb36f4..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.h +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class StructColumnReader final : public ParquetColumnReader { -public: - StructColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::vector> children, - std::vector child_output_indices, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _children(std::move(children)), - _child_output_indices(std::move(child_output_indices)) { - DCHECK_EQ(_children.size(), _child_output_indices.size()); - } - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - ParquetColumnReader* shape_source_reader() const; - Status advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const; - - std::vector> _children; // projected child readers - std::vector _child_output_indices; // child reader -> struct output position mapping -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 589154d4acc0e4..ab2bf93c785edc 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -33,11 +34,11 @@ struct RowRange { struct ParquetPageSkipPlan { int leaf_column_id = -1; // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes - // dictionary pages, matching Arrow PageReader::set_data_page_filter(). + // dictionary pages, matching the native page reader's ordinal domain. std::vector skipped_pages; std::vector skipped_page_compressed_sizes; - // Row ranges covered by skipped data pages. ScalarColumnReader uses these ranges to avoid - // calling RecordReader::SkipRecords() again for pages already skipped by Arrow. + // Row ranges covered by skipped data pages. NativeColumnReader uses these ranges to avoid + // consuming logical rows twice after the page reader has already skipped their payload. std::vector skipped_ranges; bool empty() const { return skipped_ranges.empty(); } @@ -66,6 +67,7 @@ class SelectionVector { _owned.clear(); _data = data; _size = count; + ++_generation; } void resize(size_t count) { @@ -75,12 +77,14 @@ class SelectionVector { for (size_t idx = 0; idx < count; ++idx) { _data[idx] = static_cast(idx); } + ++_generation; } void clear() { _owned.clear(); _data = nullptr; _size = 0; + ++_generation; } size_t size() const { return _size; } @@ -98,7 +102,29 @@ class SelectionVector { return _data[idx]; } - void set_index(size_t idx, Index value) { _data[idx] = value; } + void set_index(size_t idx, Index value) { + _data[idx] = value; + ++_generation; + } + + Status materialize_filter(size_t count, int64_t batch_rows, const uint8_t** filter) const { + DORIS_CHECK(filter != nullptr); + RETURN_IF_ERROR(verify(count, batch_rows)); + if (_filter_generation != _generation || _filter_count != count || + _filter_batch_rows != batch_rows) { + // Selection is shared by all readers in one scheduler batch. Cache its dense bitmap so + // a wide lazy projection does not rebuild the same O(batch_rows) filter per column. + _filter.assign(static_cast(batch_rows), 0); + for (size_t idx = 0; idx < count; ++idx) { + _filter[get_index(idx)] = 1; + } + _filter_generation = _generation; + _filter_count = count; + _filter_batch_rows = batch_rows; + } + *filter = _filter.data(); + return Status::OK(); + } Status verify(size_t count, int64_t batch_rows) const { if (batch_rows < 0) { @@ -135,13 +161,19 @@ class SelectionVector { std::vector _owned; Index* _data = nullptr; size_t _size = 0; + uint64_t _generation = 0; + mutable std::vector _filter; + mutable uint64_t _filter_generation = std::numeric_limits::max(); + mutable size_t _filter_count = 0; + mutable int64_t _filter_batch_rows = -1; }; -inline std::vector selection_to_ranges(const SelectionVector& selection, - uint16_t selected_rows) { - std::vector ranges; +inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, + std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); if (selected_rows == 0) { - return ranges; + return; } int64_t range_start = selection.get_index(0); @@ -152,11 +184,17 @@ inline std::vector selection_to_ranges(const SelectionVector& selectio previous = current; continue; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); range_start = current; previous = current; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); +} + +inline std::vector selection_to_ranges(const SelectionVector& selection, + uint16_t selected_rows) { + std::vector ranges; + selection_to_ranges(selection, selected_rows, &ranges); return ranges; } diff --git a/be/src/format_v2/table/hive_reader.cpp b/be/src/format_v2/table/hive_reader.cpp index 783ccbf95034fb..619e9959a1bfa4 100644 --- a/be/src/format_v2/table/hive_reader.cpp +++ b/be/src/format_v2/table/hive_reader.cpp @@ -60,7 +60,7 @@ bool is_file_column_position_slot(const TFileScanSlotInfo& slot_info, if (slot_info.__isset.is_file_slot) { return slot_info.is_file_slot; } - return true; + return !slot_info.__isset.category || slot_info.category != TColumnCategory::PARTITION_KEY; } bool is_hive1_orc_column_name(std::string_view name) { @@ -102,11 +102,16 @@ void add_name_mapping(std::vector* name_mapping, const std::string& } // namespace Status HiveReader::prepare_split(const format::SplitReadOptions& options) { - if (options.current_split_format != _format) { - return Status::InternalError( - "Hive scan expects all splits to use the same file format, " - "initialized_format={}, current_split_format={}", - static_cast(_format), static_cast(options.current_split_format)); + { + // Keep derived validation visible without overlapping the base scopes on the same timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + if (options.current_split_format != _format) { + return Status::InternalError( + "Hive scan expects all splits to use the same file format, " + "initialized_format={}, current_split_format={}", + static_cast(_format), static_cast(options.current_split_format)); + } } return format::TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 4294d51d043d9e..271f14a7b483eb 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -28,13 +28,20 @@ namespace doris::format::hudi { Status HudiReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.hudi_params && - options.current_range.table_format_params.hudi_params.__isset.schema_id) { - _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.hudi_params && + options.current_range.table_format_params.hudi_params.__isset.schema_id) { + _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -66,6 +73,8 @@ Status HudiHybridReader::init(format::TableReadOptions&& options) { } Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { + // A newly selected child initializes against the same scanner profile. Keep hybrid dispatch + // outside those shared counters so first-split initialization is counted exactly once. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -81,6 +90,11 @@ bool HudiHybridReader::current_split_pruned() const { return _current_split_reader->current_split_pruned(); } +bool HudiHybridReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_uses_metadata_count(); +} + Status HudiHybridReader::abort_split() { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->abort_split(); @@ -111,11 +125,47 @@ void HudiHybridReader::set_batch_size(size_t batch_size) { } } +Status HudiHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& HudiHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + +int64_t HudiHybridReader::condition_cache_hit_count() const { + // Keep the wrapper count cumulative across native/JNI dispatch so scanner-level delta + // accounting neither loses a child hit nor observes a counter reset on a split switch. + return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) + + (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()); +} + Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -123,7 +173,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt format::FileFormat file_format; RETURN_IF_ERROR(_to_file_format(*_scan_params, options.current_range, &file_format)); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::hudi::HudiReader::create_unique(); + } +#else _native_reader = format::hudi::HudiReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); @@ -139,12 +197,14 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, + .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index e22c6bd866f061..c13ac1215d72ee 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -60,9 +61,13 @@ class HudiHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; + int64_t condition_cache_hit_count() const override; #ifdef BE_TEST void TEST_install_batch_size_children() { @@ -72,6 +77,16 @@ class HudiHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) { + _native_reader->TEST_set_condition_cache_hit_count(native_hits); + _jni_reader->TEST_set_condition_cache_hit_count(jni_hits); + } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -87,6 +102,10 @@ class HudiHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle native parquet/orc splits std::unique_ptr _jni_reader; // handle MOR JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::hudi diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp new file mode 100644 index 00000000000000..6ab67caa77da03 --- /dev/null +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -0,0 +1,639 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/block/block.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/types.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_schema_utils.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { + +namespace { + +constexpr const char* kFilePathColumn = "file_path"; +constexpr const char* kPosColumn = "pos"; +constexpr const char* kRowColumn = "row"; +constexpr const char* kPartitionColumn = "partition"; +constexpr const char* kSpecIdColumn = "spec_id"; +constexpr const char* kDeleteFilePathColumn = "delete_file_path"; +constexpr const char* kContentOffsetColumn = "content_offset"; +constexpr const char* kContentSizeInBytesColumn = "content_size_in_bytes"; +constexpr int kPositionDeleteContent = 1; +constexpr int32_t kDeleteFilePathFieldId = 2147483546; +constexpr int32_t kDeleteFilePosFieldId = 2147483545; +constexpr int32_t kDeleteFileRowFieldId = 2147483544; + +bool block_has_row(const Block& block, size_t row) { + return block.columns() > 0 && row < block.rows(); +} + +void insert_int64_nullable(MutableColumnPtr& column, const int64_t* value) { + if (value == nullptr) { + parquet_utils::insert_null(column); + } else { + parquet_utils::insert_int64(column, *value); + } +} + +// Fail loudly if the filled output block is malformed. Each output column must be produced by a +// file slot; a projected system-table column that is not backed by a file slot would be skipped +// during fill and left shorter than the rest, producing a block with inconsistent column lengths. +void check_output_columns_aligned(const MutableColumns& columns) { + if (columns.empty()) { + return; + } + const size_t expected_rows = columns.front()->size(); + for (const auto& column : columns) { + DORIS_CHECK(column->size() == expected_rows) + << "Iceberg position delete system table output block has inconsistent column " + "sizes; a projected column is not backed by a file slot"; + } +} + +const IColumn* get_column(const Block& block, const std::string& name) { + auto pos = block.get_position_by_name(name); + if (pos < 0) { + return nullptr; + } + return block.get_by_position(pos).column.get(); +} + +const IColumn* nested_column(const IColumn* column) { + if (column == nullptr) { + return nullptr; + } + if (const auto* nullable = check_and_get_column(column)) { + return nullable->get_nested_column_ptr().get(); + } + return column; +} + +bool column_is_null_at(const IColumn* column, size_t row) { + const auto* nullable = check_and_get_column(column); + return nullable != nullptr && nullable->is_null_at(row); +} + +const ColumnString* get_string_column(const IColumn* column) { + return check_and_get_column(nested_column(column)); +} + +const ColumnInt64* get_int64_column(const IColumn* column) { + return check_and_get_column(nested_column(column)); +} + +ColumnDefinition build_delete_file_column(const std::string& name, DataTypePtr type) { + ColumnDefinition column; + column.identifier = Field::create_field(name); + column.name = name; + column.type = std::move(type); + return column; +} + +void set_iceberg_delete_field_id(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + if (column->name == kFilePathColumn) { + column->identifier = Field::create_field(kDeleteFilePathFieldId); + } else if (column->name == kPosColumn) { + column->identifier = Field::create_field(kDeleteFilePosFieldId); + } else if (column->name == kRowColumn && !column->has_identifier_field_id()) { + column->identifier = Field::create_field(kDeleteFileRowFieldId); + } +} + +class PositionDeleteFileTableReader final : public format::TableReader { +protected: + format::TableColumnMappingMode mapping_mode() const override { + const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) + ? schema_has_any_field_id(_data_reader.file_schema) + : schema_has_all_field_ids(_data_reader.file_schema); + if (!_data_reader.file_schema.empty() && has_field_ids) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; + } + + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->enable_row_lineage_virtual_columns = true; + // Parquet may preserve a selected complex wrapper without its own ID; position-delete row + // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. + options->allow_idless_complex_wrapper_projection = + supports_iceberg_scan_semantics_v1(_scan_params) && + _format == format::FileFormat::PARQUET; + } +}; + +} // namespace + +IcebergPositionDeleteSysTableV2Reader::~IcebergPositionDeleteSysTableV2Reader() = default; + +Status IcebergPositionDeleteSysTableV2Reader::prepare_split( + const format::SplitReadOptions& options) { + RETURN_IF_ERROR(close()); + RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + // This synthetic reader has no physical schema where a predicate can be localized, so every + // split predicate must run after its system-table columns have been materialized. + RETURN_IF_ERROR(_prepare_all_conjuncts_as_remaining()); + // The inner delete-file reader has distinct counters, so the outer preparation can safely + // contain its cache miss/open work without re-entering the same RuntimeProfile timer. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _current_range = options.current_range; + _has_split = true; + return _init_split(); +} + +Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.exec_timer); + _reset_materialized_block_stats(); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(eos != nullptr); + DORIS_CHECK(block->columns() == _projected_columns.size()); + block->clear_column_data(_projected_columns.size()); + + if (*eos) { + return Status::OK(); + } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + *eos = true; + return Status::OK(); + } + if (!_has_split) { + *eos = true; + return Status::OK(); + } + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + size_t read_rows = 0; + RETURN_IF_ERROR(_append_deletion_vector_block(block, &read_rows, eos)); + if (read_rows > 0) { + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + } + if (read_rows == 0) { + // Yield after one deletion-vector batch so cancellation and Scanner row budgets are + // observed even when residual predicates reject every synthesized row. + block->clear_column_data(_projected_columns.size()); + } + return Status::OK(); + } + + DORIS_CHECK(_position_reader != nullptr); + if (_batch_size > 0) { + _position_reader->set_batch_size(_batch_size); + } + + while (true) { + Block delete_block = _create_delete_block(); + bool position_reader_eof = false; + RETURN_IF_ERROR(_position_reader->get_block(&delete_block, &position_reader_eof)); + const size_t delete_rows = delete_block.rows(); + if (delete_rows > 0) { + size_t read_rows = 0; + RETURN_IF_ERROR( + _append_position_delete_block(block, delete_block, delete_rows, &read_rows)); + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + if (read_rows == 0) { + // A filtered materialized batch is still progress; return it to Scanner instead of + // consuming an unbounded number of position-delete batches in this call. + block->clear_column_data(_projected_columns.size()); + *eos = false; + return Status::OK(); + } + *eos = false; + return Status::OK(); + } + if (position_reader_eof) { + RETURN_IF_ERROR(close()); + *eos = true; + return Status::OK(); + } + } +} + +Status IcebergPositionDeleteSysTableV2Reader::close() { + Status close_status = Status::OK(); + if (_position_reader != nullptr) { + close_status = _position_reader->close(); + _position_reader.reset(); + } + auto base_status = format::TableReader::close(); + if (!base_status.ok() && close_status.ok()) { + close_status = std::move(base_status); + } + _iceberg_file_desc = nullptr; + _delete_file_desc = nullptr; + _read_columns.clear(); + _partition_value.reset(); + _next_dv_position.reset(); + _dv_positions = roaring::Roaring64Map(); + _has_split = false; + return close_status; +} + +std::string IcebergPositionDeleteSysTableV2Reader::debug_string() const { + std::ostringstream out; + out << "IcebergPositionDeleteSysTableV2Reader{base=" << format::TableReader::debug_string() + << ", has_split=" << _has_split << ", delete_file_kind=" + << (_delete_file_kind == DeleteFileKind::DELETION_VECTOR ? "DELETION_VECTOR" + : "POSITION_DELETE") + << ", read_column_count=" << _read_columns.size() + << ", dv_position_count=" << _dv_positions.cardinality() << "}"; + return out.str(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_init_split() { + if (_runtime_state == nullptr || _scanner_profile == nullptr || _scan_params == nullptr) { + return Status::InvalidArgument( + "invalid Iceberg position delete system table v2 reader context"); + } + if (_file_slot_descs == nullptr) { + return Status::InvalidArgument( + "Iceberg position delete system table v2 reader requires file slot descriptors"); + } + if (!_current_range.__isset.table_format_params || + !_current_range.table_format_params.__isset.iceberg_params) { + return Status::InternalError("Iceberg position delete system table range misses params"); + } + + _iceberg_file_desc = &_current_range.table_format_params.iceberg_params; + if (!_iceberg_file_desc->__isset.delete_files || _iceberg_file_desc->delete_files.size() != 1) { + return Status::InternalError( + "Iceberg position delete system table range should contain exactly one delete " + "file"); + } + _delete_file_desc = _iceberg_file_desc->delete_files.data(); + if (is_iceberg_deletion_vector(*_delete_file_desc)) { + _delete_file_kind = DeleteFileKind::DELETION_VECTOR; + } else if (_delete_file_desc->__isset.content && + _delete_file_desc->content == kPositionDeleteContent) { + _delete_file_kind = DeleteFileKind::POSITION_DELETE; + } else if (!_delete_file_desc->__isset.content) { + return Status::InternalError( + "Iceberg position delete system table delete file misses content"); + } else { + return Status::InternalError( + "Iceberg position delete system table does not support delete file content {}", + _delete_file_desc->content); + } + + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + return _init_deletion_vector_reader(); + } + return _init_position_delete_reader(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { + if (!_delete_file_desc->__isset.file_format) { + return Status::InternalError("Iceberg position delete file misses file format"); + } + format::FileFormat file_format; + if (_delete_file_desc->file_format == TFileFormatType::FORMAT_PARQUET) { + file_format = format::FileFormat::PARQUET; + } else if (_delete_file_desc->file_format == TFileFormatType::FORMAT_ORC) { + file_format = format::FileFormat::ORC; + } else { + return Status::NotSupported( + "Iceberg position delete system table v2 reader only supports Parquet and ORC " + "delete files, file_format={}", + _delete_file_desc->file_format); + } + + const bool read_row = _output_column_requested(kRowColumn); + _init_read_columns(read_row); + std::vector projected_columns; + RETURN_IF_ERROR(_build_delete_file_projected_columns(&projected_columns)); + + static constexpr const char* kPositionReaderProfile = "IcebergPositionDeleteFileReader"; + if (_position_reader_profile == nullptr) { + _position_reader_profile = _scanner_profile->get_child(kPositionReaderProfile); + if (_position_reader_profile == nullptr) { + // The outer system-table reader calls the inner reader synchronously. Giving both the + // same profile would nest identical counter pointers and double-count every timer. + _position_reader_profile = _scanner_profile->create_child(kPositionReaderProfile); + } + } + _position_reader = std::make_unique(); + RETURN_IF_ERROR(_position_reader->init({ + .projected_columns = std::move(projected_columns), + .conjuncts = {}, + .format = file_format, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _position_reader_profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::type::NONE, + .condition_cache_digest = 0, + })); + // Keep standalone-reader defaults for scanner-only fields that may be added to + // SplitReadOptions. + auto split_options = format::SplitReadOptions {}; + split_options.partition_values = {}; + split_options.partition_prune_conjuncts = {}; + split_options.cache = nullptr; + split_options.current_range = _current_range; + split_options.current_split_format = file_format; + split_options.global_rowid_context = std::nullopt; + RETURN_IF_ERROR(_position_reader->prepare_split(split_options)); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_init_deletion_vector_reader() { + if (!_delete_file_desc->__isset.referenced_data_file_path || + _delete_file_desc->referenced_data_file_path.empty()) { + return Status::InternalError("Iceberg deletion vector misses referenced data file path"); + } + if (_io_ctx == nullptr) { + return Status::InvalidArgument( + "Iceberg position delete system table v2 reader requires IO context"); + } + + IcebergDeleteFileReaderOptions options; + options.state = _runtime_state; + options.profile = _scanner_profile; + options.scan_params = _scan_params; + options.io_ctx = _io_ctx.get(); + if (_current_range.__isset.fs_name) { + options.fs_name = &_current_range.fs_name; + } + + _dv_positions = roaring::Roaring64Map(); + RETURN_IF_ERROR(read_iceberg_deletion_vector(*_delete_file_desc, options, &_dv_positions)); + _next_dv_position.emplace(_dv_positions.begin()); + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_position_delete_block( + Block* output_block, const Block& delete_block, size_t delete_rows, size_t* appended_rows) { + auto columns_guard = output_block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = output_block->get_name_to_pos_map(); + + for (size_t row = 0; row < delete_rows; ++row) { + for (const auto* slot : *_file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, &delete_block, row, 0)); + } + } + check_output_columns_aligned(columns); + *appended_rows = delete_rows; + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_deletion_vector_block(Block* block, + size_t* read_rows, + bool* eof) { + const size_t batch_size = std::max( + _batch_size > 0 ? _batch_size + : (_runtime_state == nullptr + ? static_cast(102400) + : static_cast(_runtime_state->batch_size())), + 1); + if (!_next_dv_position.has_value() || *_next_dv_position == _dv_positions.end()) { + *read_rows = 0; + *eof = true; + RETURN_IF_ERROR(close()); + return Status::OK(); + } + + auto columns_guard = block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + auto name_to_pos = block->get_name_to_pos_map(); + + size_t rows = 0; + while (rows < batch_size && *_next_dv_position != _dv_positions.end()) { + const uint64_t dv_pos = **_next_dv_position; + for (const auto* slot : *_file_slot_descs) { + auto it = name_to_pos.find(slot->col_name()); + if (it == name_to_pos.end()) { + continue; + } + RETURN_IF_ERROR(_append_sys_column(columns[it->second], *slot, nullptr, 0, dv_pos)); + } + ++(*_next_dv_position); + ++rows; + } + check_output_columns_aligned(columns); + *read_rows = rows; + _record_scan_rows(rows); + // FileScannerV2 treats eof=true as "advance to the next split" without returning the + // current block. Keep eof false after appending rows and report EOF on the next empty call. + *eof = false; + return Status::OK(); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_sys_column(MutableColumnPtr& column, + const SlotDescriptor& slot, + const Block* delete_block, + size_t source_row, + uint64_t dv_pos) { + const std::string& name = slot.col_name(); + if (name == kFilePathColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_string(column, _delete_file_desc->referenced_data_file_path); + return Status::OK(); + } + const auto* source_column = get_column(*delete_block, kFilePathColumn); + const auto* path_column = get_string_column(source_column); + if (path_column == nullptr || !block_has_row(*delete_block, source_row) || + column_is_null_at(source_column, source_row)) { + return Status::InternalError("Iceberg position delete file_path column is missing"); + } + parquet_utils::insert_string(column, path_column->get_data_at(source_row).to_string()); + return Status::OK(); + } + + if (name == kPosColumn) { + if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { + parquet_utils::insert_int64(column, cast_set(dv_pos)); + return Status::OK(); + } + const auto* source_column = get_column(*delete_block, kPosColumn); + const auto* pos_column = get_int64_column(source_column); + if (pos_column == nullptr || !block_has_row(*delete_block, source_row) || + column_is_null_at(source_column, source_row)) { + return Status::InternalError("Iceberg position delete pos column is missing"); + } + parquet_utils::insert_int64(column, pos_column->get_element(source_row)); + return Status::OK(); + } + + if (name == kRowColumn) { + if (delete_block != nullptr) { + auto row_pos = delete_block->get_position_by_name(kRowColumn); + if (row_pos >= 0) { + auto row_column = delete_block->get_by_position(row_pos) + .column->convert_to_full_column_if_const(); + if (source_row < row_column->size()) { + column->insert_from(*row_column, source_row); + return Status::OK(); + } + } + } + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (name == kPartitionColumn) { + return _append_partition_column(column, slot); + } + + if (name == kSpecIdColumn) { + if (_iceberg_file_desc->__isset.partition_spec_id) { + parquet_utils::insert_int32(column, + cast_set(_iceberg_file_desc->partition_spec_id)); + } else { + parquet_utils::insert_null(column); + } + return Status::OK(); + } + + if (name == kDeleteFilePathColumn) { + parquet_utils::insert_string(column, _delete_file_output_path()); + return Status::OK(); + } + + if (name == kContentOffsetColumn) { + const int64_t* value = _delete_file_desc->__isset.content_offset + ? &_delete_file_desc->content_offset + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + if (name == kContentSizeInBytesColumn) { + const int64_t* value = _delete_file_desc->__isset.content_size_in_bytes + ? &_delete_file_desc->content_size_in_bytes + : nullptr; + insert_int64_nullable(column, value); + return Status::OK(); + } + + return Status::InternalError("Unknown Iceberg position delete system table column: {}", name); +} + +Status IcebergPositionDeleteSysTableV2Reader::_append_partition_column(MutableColumnPtr& column, + const SlotDescriptor& slot) { + if (!_iceberg_file_desc->__isset.partition_data_json || + _iceberg_file_desc->partition_data_json.empty()) { + parquet_utils::insert_null(column); + return Status::OK(); + } + + if (!_partition_value) { + auto partition_value = slot.get_data_type_ptr()->create_column(); + auto serde = slot.get_data_type_ptr()->get_serde(); + StringRef partition_data(_iceberg_file_desc->partition_data_json.data(), + _iceberg_file_desc->partition_data_json.size()); + DataTypeSerDe::FormatOptions options = DataTypeSerDe::get_default_format_options(); + RETURN_IF_ERROR(serde->from_string(partition_data, *partition_value, options)); + DORIS_CHECK(partition_value->size() == 1); + _partition_value = std::move(partition_value); + } + column->insert_from(*_partition_value, 0); + return Status::OK(); +} + +Block IcebergPositionDeleteSysTableV2Reader::_create_delete_block() const { + Block block; + for (const auto& column : _read_columns) { + block.insert(ColumnWithTypeAndName(column.type->create_column(), column.type, column.name)); + } + return block; +} + +bool IcebergPositionDeleteSysTableV2Reader::_output_column_requested( + const std::string& name) const { + return std::any_of(_file_slot_descs->begin(), _file_slot_descs->end(), + [&name](const SlotDescriptor* slot) { return slot->col_name() == name; }); +} + +void IcebergPositionDeleteSysTableV2Reader::_init_read_columns(bool read_row) { + _read_columns.clear(); + _read_columns.push_back({kFilePathColumn, make_nullable(std::make_shared())}); + _read_columns.push_back({kPosColumn, make_nullable(std::make_shared())}); + if (read_row) { + for (const auto* slot : *_file_slot_descs) { + if (slot->col_name() == kRowColumn) { + _read_columns.push_back({kRowColumn, slot->get_data_type_ptr()}); + break; + } + } + } +} + +Status IcebergPositionDeleteSysTableV2Reader::_build_delete_file_projected_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_read_columns.size()); + for (const auto& column : _read_columns) { + if (column.name == kRowColumn) { + auto it = std::ranges::find_if(_projected_columns, [](const ColumnDefinition& field) { + return field.name == kRowColumn; + }); + if (it == _projected_columns.end()) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + columns->push_back(*it); + columns->back().type = column.type; + set_iceberg_delete_field_id(&columns->back()); + continue; + } + auto field = build_delete_file_column(column.name, column.type); + set_iceberg_delete_field_id(&field); + columns->push_back(std::move(field)); + } + return Status::OK(); +} + +const std::string& IcebergPositionDeleteSysTableV2Reader::_delete_file_output_path() const { + if (_delete_file_desc->__isset.original_path && !_delete_file_desc->original_path.empty()) { + return _delete_file_desc->original_path; + } + return _delete_file_desc->path; +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h new file mode 100644 index 00000000000000..09756daaabe4da --- /dev/null +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "roaring/roaring64map.hh" + +namespace doris { +class SlotDescriptor; +} // namespace doris + +namespace doris::format::iceberg { + +class IcebergPositionDeleteSysTableV2Reader final : public format::TableReader { +public: + ~IcebergPositionDeleteSysTableV2Reader() override; + + Status prepare_split(const format::SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + Status close() override; + std::string debug_string() const override; + +private: + enum class DeleteFileKind { + POSITION_DELETE, + DELETION_VECTOR, + }; + + struct ReadColumn { + std::string name; + DataTypePtr type; + }; + + Status _init_split(); + Status _init_position_delete_reader(); + Status _init_deletion_vector_reader(); + Status _append_position_delete_block(Block* output_block, const Block& delete_block, + size_t delete_rows, size_t* appended_rows); + Status _append_deletion_vector_block(Block* block, size_t* read_rows, bool* eof); + Status _append_sys_column(MutableColumnPtr& column, const SlotDescriptor& slot, + const Block* delete_block, size_t source_row, uint64_t dv_pos); + Status _append_partition_column(MutableColumnPtr& column, const SlotDescriptor& slot); + Block _create_delete_block() const; + bool _output_column_requested(const std::string& name) const; + void _init_read_columns(bool read_row); + Status _build_delete_file_projected_columns(std::vector* columns) const; + const std::string& _delete_file_output_path() const; + + TFileRangeDesc _current_range; + const TIcebergFileDesc* _iceberg_file_desc = nullptr; + const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; + DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; + std::unique_ptr _position_reader; + RuntimeProfile* _position_reader_profile = nullptr; + std::vector _read_columns; + ColumnPtr _partition_value; + roaring::Roaring64Map _dv_positions; + std::optional _next_dv_position; + bool _has_split = false; +}; + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index cc2be83f39f8e5..407d5924e9f64a 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -36,7 +36,7 @@ #include "core/field.h" #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" -#include "format_v2/deletion_vector_reader.h" +#include "format/table/deletion_vector_reader.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/equality_delete_predicate.h" #include "format_v2/orc/orc_reader.h" @@ -44,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/debug_points.h" #include "util/url_coding.h" namespace doris::format::iceberg { @@ -91,7 +92,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - Field initial_default; + VExprSPtr literal; if (table_field.initial_default_value_is_base64 || table_field.type->get_primitive_type() == TYPE_VARBINARY) { // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its @@ -104,19 +105,26 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit table_field.name); } if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - initial_default = Field::create_field(StringView(decoded_default)); + const auto initial_default = + Field::create_field(StringView(decoded_default)); + // VLiteral must copy the borrowed StringView while decoded_default is alive; UUID and + // long FIXED defaults otherwise retain a pointer into freed decode storage. + literal = VLiteral::create_shared(table_field.type, initial_default); } else { DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - initial_default = Field::create_field(decoded_default); + literal = VLiteral::create_shared(table_field.type, + Field::create_field(decoded_default)); } } else { // An added field's initial default is its logical value in every older data file that lacks // the physical column. FE normalizes the string for the current Doris table type. + Field initial_default; RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( *table_field.initial_default_value, initial_default)); + literal = VLiteral::create_shared(table_field.type, initial_default); } - auto literal = VLiteral::create_shared(table_field.type, initial_default); + DORIS_CHECK(literal != nullptr); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); return Status::OK(); @@ -188,7 +196,7 @@ static std::string iceberg_params_debug_string(const std::optional( - *remove_nullable((block.get_by_position(ICEBERG_FILE_PATH_BLOCK_POSITION).column))); - const auto& pos_column = assert_cast( - *remove_nullable(block.get_by_position(ICEBERG_ROW_POS_BLOCK_POSITION).column)); + const auto& file_path_column_ptr = + block.get_by_position(ICEBERG_FILE_PATH_BLOCK_POSITION).column; + const auto& pos_column_ptr = block.get_by_position(ICEBERG_ROW_POS_BLOCK_POSITION).column; + if (const auto* nullable_column = check_and_get_column(*file_path_column_ptr); + nullable_column != nullptr && nullable_column->has_null(0, read_rows)) { + return Status::Corruption("Iceberg position delete column file_path contains null values"); + } + if (const auto* nullable_column = check_and_get_column(*pos_column_ptr); + nullable_column != nullptr && nullable_column->has_null(0, read_rows)) { + return Status::Corruption("Iceberg position delete column pos contains null values"); + } + const auto& file_path_column = + assert_cast(*remove_nullable(file_path_column_ptr)); + const auto& pos_column = assert_cast(*remove_nullable(pos_column_ptr)); for (size_t row = 0; row < read_rows; ++row) { const auto file_path = file_path_column.get_data_at(row).to_string(); (*_rows_by_data_file)[file_path].push_back(pos_column.get_element(row)); @@ -215,25 +233,31 @@ Status IcebergTableReader::PositionDeleteRowsCollector::collect(const Block& blo } Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options) { - _row_lineage_columns = {}; - _iceberg_params.reset(); - _delete_predicates_initialized = false; - _position_delete_rows_storage.clear(); - _equality_delete_filters.clear(); - _split_cache = options.cache; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.iceberg_params) { - const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; - _iceberg_params = iceberg_params; - if (iceberg_params.__isset.first_row_id) { - _row_lineage_columns.first_row_id = iceberg_params.first_row_id; - } - if (iceberg_params.__isset.last_updated_sequence_number) { - _row_lineage_columns.last_updated_sequence_number = - iceberg_params.last_updated_sequence_number; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _row_lineage_columns = {}; + _iceberg_params.reset(); + _delete_predicates_initialized = false; + _position_delete_rows_storage.clear(); + _equality_delete_filters.clear(); + _split_cache = options.cache; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.iceberg_params) { + const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; + _iceberg_params = iceberg_params; + if (iceberg_params.__isset.first_row_id) { + _row_lineage_columns.first_row_id = iceberg_params.first_row_id; + } + if (iceberg_params.__isset.last_updated_sequence_number) { + _row_lineage_columns.last_updated_sequence_number = + iceberg_params.last_updated_sequence_number; + } } } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -245,6 +269,8 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options if (_is_table_level_count_active()) { return Status::OK(); } + DBUG_EXECUTE_IF("IcebergTableReader.prepare_split.before_delete_file_scan", + DBUG_RUN_CALLBACK()); RETURN_IF_ERROR(_init_delete_predicates(options.current_range.table_format_params)); return Status::OK(); } @@ -377,10 +403,8 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes if (deletion_vector == nullptr) { return Status::OK(); } - if (!deletion_vector->__isset.content_offset || - !deletion_vector->__isset.content_size_in_bytes) { - return Status::InternalError("Deletion vector is missing content offset or length"); - } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(*deletion_vector, bytes_read)); const std::string data_file_path = iceberg_params.__isset.original_file_path ? iceberg_params.original_file_path @@ -388,7 +412,7 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes desc->key = build_iceberg_deletion_vector_cache_key(data_file_path, *deletion_vector); desc->path = deletion_vector->path; desc->start_offset = deletion_vector->content_offset; - desc->size = deletion_vector->content_size_in_bytes; + desc->size = static_cast(bytes_read); desc->file_size = -1; desc->format = DeleteFileDesc::Format::ICEBERG; *has_delete_file = true; @@ -558,14 +582,13 @@ void IcebergTableReader::_append_equality_delete_row_count_carrier( DORIS_CHECK(request != nullptr); // Columnar readers establish a filter batch's row count from predicate columns. If all // equality keys are missing, the predicate consists only of NULL literals and the filter block - // would otherwise have zero rows. Read one physical column eagerly as a row-count carrier; - // normal final materialization ignores this hidden dependency. - const auto carrier_it = std::ranges::find_if( - _data_reader.file_schema, [](const format::ColumnDefinition& field) { - return field.column_type == format::ColumnType::DATA_COLUMN; - }); - DORIS_CHECK(carrier_it != _data_reader.file_schema.end()); - _append_file_scan_column(request, format::LocalColumnId(carrier_it->file_local_id()), + // would otherwise have zero rows. Use the virtual row-position column as the carrier instead + // of an arbitrary physical column. For example, a data file may start with an unsupported + // TIME_MILLIS leaf while the query projects only a supported `id`; selecting that TIME leaf as + // a hidden carrier would make Parquet reject a column the query never requested. Row position + // has one value per input row in both Parquet and ORC, is already used by delete predicates, + // and is explicitly excluded from physical logical-type validation. + _append_file_scan_column(request, format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), &request->predicate_columns); } @@ -635,10 +658,13 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && scan_params.enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = + scan_params.__isset.enable_mapping_varbinary && scan_params.enable_mapping_varbinary; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { + // Delete and data files must parse raw binary fields with the same scan-level mapping. *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + enable_mapping_timestamp_tz, enable_mapping_varbinary); } else { *reader = std::make_unique(system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 79e0174a3d7eaa..d28be3d7f98f0b 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -27,20 +27,19 @@ #include "core/block/block.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format_v2/file_reader.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "format_v2/table_reader.h" #include "gen_cpp/PlanNodes_types.h" namespace doris { class Block; +struct DeleteFileDesc; namespace io { struct FileDescription; struct FileSystemProperties; } // namespace io } // namespace doris -namespace doris::format { -struct DeleteFileDesc; -} namespace doris::format::iceberg { // Iceberg table-level reader. @@ -59,12 +58,22 @@ class IcebergTableReader : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && _has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) + ? schema_has_any_field_id(_data_reader.file_schema) + : schema_has_all_field_ids(_data_reader.file_schema); + if (!_data_reader.file_schema.empty() && has_field_ids) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; } protected: + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->enable_row_lineage_virtual_columns = true; + options->allow_idless_complex_wrapper_projection = + supports_iceberg_scan_semantics_v1(_scan_params) && _format == FileFormat::PARQUET; + } + Status materialize_virtual_columns(Block* table_block) override; Status customize_file_scan_request(format::FileScanRequest* file_request) override; @@ -78,26 +87,6 @@ class IcebergTableReader : public format::TableReader { private: struct EqualityDeleteFilter; - - bool _has_field_id(const std::vector& schema) const { - for (const auto& field : schema) { - // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the - // first-phase scan. That virtual column is not an Iceberg data field and therefore has - // no Iceberg field id. Do not let it downgrade schema-evolution reads to BY_NAME, - // otherwise old data files whose physical names predate a rename (for example, - // table column `new_new_id` stored as file column `id`) are materialized as defaults. - if (field.column_type != format::ColumnType::DATA_COLUMN) { - continue; - } - if (!field.has_identifier_field_id()) { - return false; - } - if (!_has_field_id(field.children)) { - return false; - } - } - return true; - } static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; static constexpr int POSITION_DELETE = 1; static constexpr int EQUALITY_DELETE = 2; diff --git a/be/src/format_v2/table/iceberg_schema_utils.h b/be/src/format_v2/table/iceberg_schema_utils.h new file mode 100644 index 00000000000000..516c982194bbe4 --- /dev/null +++ b/be/src/format_v2/table/iceberg_schema_utils.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format/table/iceberg_scan_semantics.h" +#include "format_v2/column_data.h" + +namespace doris::format::iceberg { + +inline bool schema_has_any_field_id(const std::vector& schema) { + for (const auto& field : schema) { + // Iceberg's hasIds contract is existential for the whole file. Ignore synthesized columns + // and retain ID projection when any real field, including a nested field, carries an ID. + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (field.has_identifier_field_id() || schema_has_any_field_id(field.children)) { + return true; + } + } + return false; +} + +inline bool schema_has_all_field_ids(const std::vector& schema) { + for (const auto& field : schema) { + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (!field.has_identifier_field_id() || !schema_has_all_field_ids(field.children)) { + return false; + } + } + return true; +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index de4562b04088a6..d815ef81c9b424 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -23,9 +23,9 @@ #include #include "exprs/vexpr_context.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" -#include "format_v2/deletion_vector_reader.h" #include "format_v2/jni/paimon_jni_reader.h" #include "format_v2/table/schema_history_util.h" #include "gen_cpp/PlanNodes_types.h" @@ -33,12 +33,19 @@ namespace doris::format::paimon { Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - const auto& paimon_params = options.current_range.table_format_params.paimon_params; - if (paimon_params.__isset.schema_id) { - _split_schema_id = paimon_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + const auto& paimon_params = options.current_range.table_format_params.paimon_params; + if (paimon_params.__isset.schema_id) { + _split_schema_id = paimon_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -75,11 +82,13 @@ Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_d return Status::OK(); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); desc->key = build_paimon_deletion_vector_cache_key(deletion_file); desc->path = deletion_file.path; desc->start_offset = deletion_file.offset; - desc->size = deletion_file.length + 4; + desc->size = static_cast(bytes_read); desc->file_size = -1; desc->format = DeleteFileDesc::Format::PAIMON; *has_delete_file = true; @@ -91,6 +100,8 @@ Status PaimonHybridReader::init(format::TableReadOptions&& options) { } Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { + // Child initialization uses the scanner profile too; hybrid dispatch must not nest the same + // timer around the first native or JNI child and double-count that initialization. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -106,6 +117,11 @@ bool PaimonHybridReader::current_split_pruned() const { return _current_split_reader->current_split_pruned(); } +bool PaimonHybridReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_uses_metadata_count(); +} + Status PaimonHybridReader::abort_split() { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->abort_split(); @@ -136,11 +152,47 @@ void PaimonHybridReader::set_batch_size(size_t batch_size) { } } +Status PaimonHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& PaimonHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + +int64_t PaimonHybridReader::condition_cache_hit_count() const { + // Both children survive split switches, so the wrapper must publish their cumulative totals; + // returning only the active child would make FileScannerV2's monotonic delta go backwards. + return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) + + (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()); +} + Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -151,7 +203,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO DCHECK(file_format == format::FileFormat::PARQUET || file_format == format::FileFormat::ORC); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::paimon::PaimonReader::create_unique(); + } +#else _native_reader = format::paimon::PaimonReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); @@ -167,12 +227,14 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, + .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index efaf29d5c76059..b4f076fc6e4878 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -17,11 +17,12 @@ #pragma once +#include #include #include "format_v2/table_reader.h" -namespace doris::format { +namespace doris { struct DeleteFileDesc; } namespace doris::format::paimon { @@ -66,9 +67,13 @@ class PaimonHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; + int64_t condition_cache_hit_count() const override; #ifdef BE_TEST static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); } @@ -83,6 +88,16 @@ class PaimonHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) { + _native_reader->TEST_set_condition_cache_hit_count(native_hits); + _jni_reader->TEST_set_condition_cache_hit_count(jni_hits); + } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -95,6 +110,10 @@ class PaimonHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle parquet/orc native splits std::unique_ptr _jni_reader; // handle serialized JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::paimon diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index c67cece6e05b8c..bd2c08b5f2310f 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -20,8 +20,13 @@ #include #include +#include +#include +#include #include +#include #include +#include #include #include @@ -37,7 +42,10 @@ #include "format/arrow/arrow_utils.h" #include "format_v2/materialized_reader_util.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/query_context.h" #include "runtime/runtime_state.h" +#include "runtime/thread_context.h" #include "util/timezone_utils.h" namespace doris::format::remote_doris { @@ -63,7 +71,12 @@ Status validate_remote_doris_range(const TFileRangeDesc& range) { class FlightRemoteDorisStream final : public RemoteDorisStream { public: - explicit FlightRemoteDorisStream(const TFileRangeDesc& range) : _range(range) {} + FlightRemoteDorisStream(const TFileRangeDesc& range, std::shared_ptr io_ctx, + RuntimeState* runtime_state, int timeout_seconds) + : _range(range), + _io_ctx(std::move(io_ctx)), + _runtime_state(runtime_state), + _timeout_seconds(std::max(1, timeout_seconds)) {} Status open() { RETURN_IF_ERROR(validate_remote_doris_range(_range)); @@ -74,14 +87,103 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { arrow::flight::Ticket ticket; RETURN_DORIS_STATUS_IF_ERROR( arrow::flight::Ticket::Deserialize(params.ticket).Value(&ticket)); + struct PendingOpen { + std::mutex mutex; + std::condition_variable cv; + bool done = false; + bool abandoned = false; + arrow::Status status = arrow::Status::OK(); + std::unique_ptr client; + std::unique_ptr stream; + }; + auto pending = std::make_shared(); + std::unique_ptr flight_client; RETURN_DORIS_STATUS_IF_ERROR( - arrow::flight::FlightClient::Connect(location).Value(&_flight_client)); - RETURN_DORIS_STATUS_IF_ERROR(_flight_client->DoGet(ticket).Value(&_stream)); + arrow::flight::FlightClient::Connect(location).Value(&flight_client)); + arrow::flight::FlightCallOptions options; + // A Flight deadline covers streaming reads as well as DoGet setup, so a stalled Next() + // cannot outlive the query execution timeout indefinitely. + options.timeout = std::chrono::seconds(_timeout_seconds); + // Start before DoGet because endpoint setup is itself a blocking RPC covered by the same + // query/scanner cancellation contract as streaming Next(). + _cancellation_watcher = std::jthread( + [this](std::stop_token stop_token) { _watch_cancellation(stop_token); }); + + std::shared_ptr resource_ctx; + if (_runtime_state != nullptr && _runtime_state->get_query_ctx() != nullptr) { + resource_ctx = _runtime_state->get_query_ctx()->resource_ctx(); + } + std::thread do_get_thread([pending, options, ticket, resource_ctx, + client = std::move(flight_client)]() mutable { + const auto do_get = [&] { + std::unique_ptr stream; + auto status = client->DoGet(options, ticket).Value(&stream); + { + std::lock_guard lock(pending->mutex); + if (!pending->abandoned) { + pending->status = std::move(status); + pending->client = std::move(client); + pending->stream = std::move(stream); + } else { + // A detached worker must release its query-owned Flight client + // before leaving the task attachment that accounts for it. + client.reset(); + } + pending->done = true; + } + pending->cv.notify_all(); + }; + if (resource_ctx != nullptr) { + SCOPED_ATTACH_TASK(resource_ctx); + do_get(); + } else { + SCOPED_INIT_THREAD_CONTEXT(); + do_get(); + } + }); + bool cancelled_during_open = false; + { + std::unique_lock lock(pending->mutex); + while (!pending->done && !_is_cancelled()) { + pending->cv.wait_for(lock, std::chrono::milliseconds(25)); + } + if (!pending->done) { + pending->abandoned = true; + cancelled_during_open = true; + } + } + if (cancelled_during_open) { + // Arrow 17 exposes no cancellable handle until DoGet returns. Detaching the bounded RPC + // keeps query/scanner shutdown prompt while the call is still capped by its deadline; + // the shared state owns all Arrow objects until that worker exits. + do_get_thread.detach(); + _stop_cancellation_watcher(); + return Status::Cancelled("Remote Doris Flight open was cancelled"); + } + do_get_thread.join(); + if (!pending->status.ok()) { + _stop_cancellation_watcher(); + RETURN_DORIS_STATUS_IF_ERROR(pending->status); + } + { + std::lock_guard lock(_flight_mutex); + _flight_client = std::move(pending->client); + _stream = std::move(pending->stream); + } + if (_is_cancelled()) { + _cancel_flight_call(); + _stop_cancellation_watcher(); + return Status::Cancelled("Remote Doris Flight open was cancelled"); + } return Status::OK(); } Status next(std::shared_ptr* batch) override { DORIS_CHECK(batch != nullptr); + if (_io_ctx != nullptr && _io_ctx->should_stop) { + _cancel_flight_call(); + return Status::Cancelled("Remote Doris Flight read was cancelled"); + } arrow::flight::FlightStreamChunk chunk; RETURN_DORIS_STATUS_IF_ERROR(_stream->Next().Value(&chunk)); *batch = chunk.data; @@ -89,6 +191,13 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { } Status close() override { + _stop_cancellation_watcher(); + { + std::lock_guard lock(_flight_mutex); + if (_stream != nullptr) { + _stream->Cancel(); + } + } _stream.reset(); if (_flight_client != nullptr) { RETURN_DORIS_STATUS_IF_ERROR(_flight_client->Close()); @@ -98,14 +207,65 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { } private: + bool _is_cancelled() const { + return (_runtime_state != nullptr && _runtime_state->is_cancelled()) || + (_io_ctx != nullptr && _io_ctx->should_stop); + } + + void _cancel_flight_call() { + std::lock_guard lock(_flight_mutex); + if (_stream != nullptr) { + _stream->Cancel(); + } + } + + void _watch_cancellation_loop(std::stop_token watcher_stop_token) { + while (!watcher_stop_token.stop_requested()) { + if (_is_cancelled()) { + _cancel_flight_call(); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + } + + void _watch_cancellation(std::stop_token watcher_stop_token) { + if (_runtime_state != nullptr && _runtime_state->get_query_ctx() != nullptr && + _runtime_state->get_query_ctx()->resource_ctx() != nullptr) { + // The watcher is query-owned and may allocate in Arrow while signalling cancellation. + SCOPED_ATTACH_TASK(_runtime_state); + _watch_cancellation_loop(watcher_stop_token); + return; + } + // Metadata/tests can construct a RuntimeState without a QueryContext; initialize TLS there + // instead of violating AttachTask's non-null resource-context invariant. + SCOPED_INIT_THREAD_CONTEXT(); + _watch_cancellation_loop(watcher_stop_token); + } + + void _stop_cancellation_watcher() { + if (_cancellation_watcher.joinable()) { + _cancellation_watcher.request_stop(); + _cancellation_watcher.join(); + } + } + const TFileRangeDesc _range; + std::shared_ptr _io_ctx; + RuntimeState* _runtime_state; + int _timeout_seconds; + std::jthread _cancellation_watcher; + std::mutex _flight_mutex; std::unique_ptr _flight_client; std::unique_ptr _stream; }; -Status create_flight_stream(const TFileRangeDesc& range, std::unique_ptr* out) { +Status create_flight_stream(const TFileRangeDesc& range, std::shared_ptr io_ctx, + RuntimeState* runtime_state, int timeout_seconds, + std::unique_ptr* out) { DORIS_CHECK(out != nullptr); - auto stream = std::make_unique(range); + auto stream = std::make_unique(range, std::move(io_ctx), runtime_state, + timeout_seconds); RETURN_IF_ERROR(stream->open()); *out = std::move(stream); return Status::OK(); @@ -177,8 +337,31 @@ RemoteDorisFileReader::~RemoteDorisFileReader() { static_cast(close()); } +void RemoteDorisFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + const auto hierarchy = file_scan_profile::ensure_hierarchy(_profile); + _io_time = hierarchy.io; + static const char* remote_profile = "RemoteDorisFileReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, remote_profile, file_scan_profile::FILE_READER, 1); + _open_stream_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisOpenStreamTime", remote_profile, 1); + _next_batch_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisNextBatchTime", remote_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisMaterializeTime", remote_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisFilterTime", remote_profile, 1); +} + Status RemoteDorisFileReader::init(RuntimeState* state) { - (void)state; + _init_profile(); + SCOPED_TIMER(_total_time); + if (state != nullptr) { + _flight_timeout_seconds = std::max(1, state->execution_timeout()); + } + _runtime_state = state; RETURN_IF_ERROR(validate_remote_doris_range(_range)); RETURN_IF_ERROR(_build_col_name_to_file_id()); _eof = false; @@ -186,6 +369,7 @@ Status RemoteDorisFileReader::init(RuntimeState* state) { } Status RemoteDorisFileReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_schema != nullptr); file_schema->clear(); file_schema->reserve(_file_slot_descs.size()); @@ -206,6 +390,8 @@ Status RemoteDorisFileReader::get_schema(std::vector* file_sch } Status RemoteDorisFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_stream_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); RETURN_IF_ERROR(_open_stream()); _eof = false; @@ -213,31 +399,51 @@ Status RemoteDorisFileReader::open(std::shared_ptr request) { } Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); if (_stream == nullptr) { return Status::InternalError("Remote Doris v2 reader is not open"); } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + // Observe cancellation before entering a potentially blocking Flight read; the production + // stream also carries a query-bounded RPC deadline for cancellation arriving mid-read. + RETURN_IF_ERROR(close()); + *rows = 0; + *eof = true; + return Status::OK(); + } *rows = 0; *eof = false; std::shared_ptr batch; - RETURN_IF_ERROR(_stream->next(&batch)); + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_next_batch_time); + RETURN_IF_ERROR(_stream->next(&batch)); + } if (batch == nullptr) { *eof = true; _eof = true; return Status::OK(); } - RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + } _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR( - apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + } return Status::OK(); } Status RemoteDorisFileReader::close() { + SCOPED_TIMER(_total_time); if (_stream != nullptr) { RETURN_IF_ERROR(_stream->close()); _stream.reset(); @@ -252,7 +458,8 @@ Status RemoteDorisFileReader::_open_stream() { if (_stream_factory) { RETURN_IF_ERROR(_stream_factory(_range, &_stream)); } else { - RETURN_IF_ERROR(create_flight_stream(_range, &_stream)); + RETURN_IF_ERROR(create_flight_stream(_range, _io_ctx, _runtime_state, + _flight_timeout_seconds, &_stream)); } DORIS_CHECK(_stream != nullptr); return Status::OK(); @@ -349,7 +556,12 @@ Status RemoteDorisReader::init(TableReadOptions&& options) { } Status RemoteDorisReader::prepare_split(const SplitReadOptions& options) { - RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + { + // Keep protocol validation visible while avoiding overlap with TableReader's own scopes. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + } return TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h index b4dd2a505a95ad..3b4a03f4569396 100644 --- a/be/src/format_v2/table/remote_doris_reader.h +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -71,6 +71,7 @@ class RemoteDorisFileReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _open_stream(); Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, size_t* rows) const; @@ -83,6 +84,14 @@ class RemoteDorisFileReader final : public FileReader { const std::vector _file_slot_descs; RemoteDorisStreamFactory _stream_factory; cctz::time_zone _ctz; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_stream_time = nullptr; + RuntimeProfile::Counter* _next_batch_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + RuntimeState* _runtime_state = nullptr; + int _flight_timeout_seconds = 300; std::unique_ptr _stream; std::unordered_map _col_name_to_file_id; }; diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index 10109839e6987d..440211607d633d 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -77,6 +77,8 @@ void annotate_column_from_field(ColumnDefinition* column, const schema::external } column->name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}; + column->has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; if (!field.__isset.nestedField) { return; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 5112a547df9ea9..a073f01f237506 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -39,16 +39,18 @@ #include "core/data_type/primitive_type.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" -#include "format_v2/deletion_vector_reader.h" #include "format_v2/delimited_text/csv_reader.h" #include "format_v2/delimited_text/text_reader.h" #include "format_v2/json/json_reader.h" #include "format_v2/native/native_reader.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" +#include "runtime/file_scan_profile.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" #include "util/string_util.h" @@ -143,6 +145,93 @@ const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& return field_ptr.field_ptr.get(); } +ColumnDefinition build_schema_identity_from_external_field(const schema::external::TField& field) { + ColumnDefinition identity; + if (field.__isset.id) { + identity.identifier = Field::create_field(field.id); + } + identity.name = field.__isset.name ? field.name : ""; + identity.name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}; + identity.has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; + if (!field.__isset.nestedField) { + return identity; + } + if (field.nestedField.__isset.struct_field && field.nestedField.struct_field.__isset.fields) { + for (const auto& child_ptr : field.nestedField.struct_field.fields) { + if (const auto* child = get_field_ptr(child_ptr); child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + } + } + } else if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field) { + if (const auto* child = get_field_ptr(field.nestedField.array_field.item_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "element"; + } + } else if (field.nestedField.__isset.map_field) { + if (field.nestedField.map_field.__isset.key_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.key_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "key"; + } + } + if (field.nestedField.map_field.__isset.value_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.value_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "value"; + } + } + } + return identity; +} + +const ColumnDefinition* find_identity_child(const ColumnDefinition& projected_child, + const ColumnDefinition& identity_parent) { + const auto child_it = std::ranges::find_if( + identity_parent.children, [&](const ColumnDefinition& identity_child) { + if (projected_child.has_identifier_field_id() && + identity_child.has_identifier_field_id()) { + return projected_child.get_identifier_field_id() == + identity_child.get_identifier_field_id(); + } + if (to_lower(projected_child.name) == to_lower(identity_child.name)) { + return true; + } + return std::ranges::any_of( + identity_child.name_mapping, [&](const std::string& alias) { + return to_lower(projected_child.name) == to_lower(alias); + }); + }); + return child_it == identity_parent.children.end() ? nullptr : &*child_it; +} + +void attach_full_schema_identity(ColumnDefinition* projected, const ColumnDefinition& identity) { + DORIS_CHECK(projected != nullptr); + // Access-path children control materialization, but wrapper discovery needs sibling IDs that + // were pruned from that projection. Keep the complete identity tree on a separate channel. + projected->identity_children = identity.children; + for (auto& projected_child : projected->children) { + if (const auto* identity_child = find_identity_child(projected_child, identity); + identity_child != nullptr) { + attach_full_schema_identity(&projected_child, *identity_child); + } + } +} + +void clear_initial_default_metadata(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + column->initial_default_value.reset(); + column->initial_default_value_is_base64 = false; + for (auto& child : column->children) { + clear_initial_default_metadata(&child); + } +} + bool external_field_matches_name(const schema::external::TField& field, const std::string& name) { if (field.__isset.name && to_lower(field.name) == to_lower(name)) { return true; @@ -189,6 +278,8 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .name = field.__isset.name ? field.name : "", .name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}, + .has_name_mapping = field.__isset.name_mapping_is_authoritative && + field.name_mapping_is_authoritative, .type = std::move(type), .children = {}, .default_expr = nullptr, @@ -298,12 +389,32 @@ const schema::external::TField* find_external_root_field(const TFileScanRangePar if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { return nullptr; } + if (!supports_iceberg_scan_semantics_v1(params)) { + // Old BEs used one ordered current-name/alias pass. Preserve that result for old-FE plans + // until the explicit scan-semantics marker makes exact-name precedence cluster-wide. + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && external_field_matches_name(*field, column.name)) { + return field; + } + } + return nullptr; + } + // A reused name identifies the newly added field, not an older sibling that retained that + // spelling as an alias. Exhaust exact current names before consulting historical aliases. for (const auto& field_ptr : schema->root_field.fields) { const auto* field = get_field_ptr(field_ptr); - if (field == nullptr) { - continue; + if (field != nullptr && field->__isset.name && + to_lower(field->name) == to_lower(column.name)) { + return field; } - if (external_field_matches_name(*field, column.name)) { + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.name_mapping && + std::ranges::any_of(field->name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(column.name); + })) { return field; } } @@ -486,8 +597,20 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); + const bool use_current_semantics = supports_iceberg_scan_semantics_v1(context->scan_params); + if (!use_current_semantics) { + // IDs and encoded defaults predate the result-changing semantics. Strip only the new + // default channel so an old-FE plan keeps the same generic root/nested values on every BE. + clear_initial_default_metadata(&*context->schema_column); + } column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; + column->has_name_mapping = context->schema_column->has_name_mapping; + // Projected roots already carry a generic FE default expression, but Iceberg binary defaults + // need the raw Base64 marker so missing-file materialization can decode rather than copy text. + column->initial_default_value = context->schema_column->initial_default_value; + column->initial_default_value_is_base64 = + context->schema_column->initial_default_value_is_base64; return Status::OK(); } @@ -520,23 +643,15 @@ std::optional TableReader::_find_current_table_column_by_field } Status TableReader::init(TableReadOptions&& options) { - _scan_params = options.scan_params; - _format = options.format; - _io_ctx = options.io_ctx; - _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; - _file_slot_descs = options.file_slot_descs; - _push_down_agg_type = options.push_down_agg_type; - _initial_condition_cache_digest = options.condition_cache_digest; - _condition_cache_digest = _initial_condition_cache_digest; - _projected_columns = std::move(options.projected_columns); - _system_properties = create_system_properties(_scan_params); - _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); - if (_scanner_profile != nullptr) { - static const char* table_profile = "TableReader"; - ADD_TIMER_WITH_LEVEL(_scanner_profile, table_profile, 1); + const auto hierarchy = file_scan_profile::ensure_hierarchy(_scanner_profile); + static const char* table_profile = file_scan_profile::TABLE_READER; + static const char* file_reader_profile = file_scan_profile::FILE_READER; + _profile.total_timer = hierarchy.table_reader; + _profile.file_reader_total_timer = hierarchy.file_reader; + _profile.init_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "InitTime", table_profile, 1); _profile.num_delete_files = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteFiles", TUnit::UNIT, table_profile, 1); _profile.num_delete_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteRows", @@ -563,16 +678,124 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PrepareSplitTime", table_profile, 1); _profile.finalize_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "FinalizeBlockTime", table_profile, 1); + _profile.residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "ResidualFilterTime", table_profile, 1); _profile.create_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CreateReaderTime", table_profile, 1); _profile.pushdown_agg_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PushDownAggTime", table_profile, 1); _profile.open_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "OpenReaderTime", table_profile, 1); - _profile.runtime_filter_partition_prune_timer = ADD_TIMER_WITH_LEVEL( - _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", 1); - _profile.runtime_filter_partition_pruned_range_counter = ADD_COUNTER_WITH_LEVEL( - _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, 1); + _profile.runtime_filter_partition_prune_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", table_profile, 1); + _profile.runtime_filter_partition_pruned_range_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, + table_profile, 1); + _profile.close_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CloseTime", table_profile, 1); + // Lifecycle timer names remain globally unique because RuntimeProfile's visual hierarchy + // does not namespace counters that share the same display parent. + _profile.file_reader_init_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderInitTime", file_reader_profile, 1); + _profile.file_reader_schema_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetSchemaTime", file_reader_profile, 1); + _profile.file_reader_mapper_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCreateColumnMapperTime", file_reader_profile, 1); + _profile.file_reader_open_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderOpenTime", file_reader_profile, 1); + _profile.file_reader_get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetBlockTime", file_reader_profile, 1); + _profile.file_reader_aggregate_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderAggregatePushDownTime", file_reader_profile, 1); + _profile.file_reader_close_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCloseTime", file_reader_profile, 1); + } + // Establish lifecycle timers before consuming options or constructing filesystem properties; + // placing these scopes at the tail records only scope teardown and hides expensive init work. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _scan_params = options.scan_params; + _format = options.format; + _io_ctx = options.io_ctx; + _runtime_state = options.runtime_state; + _file_slot_descs = options.file_slot_descs; + _push_down_agg_type = options.push_down_agg_type; + _push_down_count_columns = options.push_down_count_columns; + _initial_condition_cache_digest = options.condition_cache_digest; + _condition_cache_digest = _initial_condition_cache_digest; + _table_reader_owned_conjunct_count = + options.table_reader_owned_conjunct_count.value_or(options.conjuncts.size()); + DORIS_CHECK_LE(_table_reader_owned_conjunct_count, options.conjuncts.size()); + _projected_columns = std::move(options.projected_columns); + if (supports_iceberg_scan_semantics_v1(_scan_params)) { + for (auto& projected_column : _projected_columns) { + const auto* schema_field = find_external_root_field(_scan_params, projected_column); + if (schema_field != nullptr) { + attach_full_schema_identity( + &projected_column, + build_schema_identity_from_external_field(*schema_field)); + } + } + } + _system_properties = create_system_properties(_scan_params); + _mapper_options.mode = TableColumnMappingMode::BY_NAME; + return _replace_conjuncts(options.conjuncts); +} + +Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared) { + DORIS_CHECK(source != nullptr); + DORIS_CHECK(source->root() != nullptr); + DORIS_CHECK(prepared != nullptr); + VExprSPtr root; + RETURN_IF_ERROR(clone_table_expr_tree(source->root(), &root)); + auto conjunct = VExprContext::create_shared(std::move(root)); + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + *prepared = std::move(conjunct); + return Status::OK(); +} + +Status TableReader::_replace_conjuncts(const VExprContextSPtrs& conjuncts) { + VExprContextSPtrs prepared; + prepared.reserve(conjuncts.size()); + for (const auto& source : conjuncts) { + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + prepared.push_back(std::move(conjunct)); + } + _conjuncts = std::move(prepared); + return Status::OK(); +} + +Status TableReader::append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count) { + DORIS_CHECK(!_appended_table_reader_owned_conjunct_count.has_value()); + _appended_table_reader_owned_conjunct_count = table_reader_owned_conjunct_count; + auto status = append_conjuncts(conjuncts); + _appended_table_reader_owned_conjunct_count.reset(); + return status; +} + +Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + DORIS_CHECK_LE(owned_count, conjuncts.size()); + // Once Scanner owns a suffix, later predicates cannot be inserted into the TableReader-owned + // prefix without reordering them ahead of that stateful/error-preserving barrier. + DORIS_CHECK(owned_count == 0 || _table_reader_owned_conjunct_count == _conjuncts.size()); + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + const auto& source = conjuncts[conjunct_index]; + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + _conjuncts.push_back(conjunct); + if (conjunct_index < owned_count) { + ++_table_reader_owned_conjunct_count; + } + if (_current_task != nullptr && conjunct_index < owned_count) { + // The active reader has already fixed its localized predicate set. Appended runtime + // filters must remain residual until the next split rebuilds its FileScanRequest. + _remaining_conjuncts.push_back(std::move(conjunct)); + } } return Status::OK(); } @@ -581,22 +804,83 @@ Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); _constant_pruning_safe_filter_count = 0; bool in_safe_prefix = true; - for (const auto& conjunct : _conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { + const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); // `_table_filters` omits expressions without slot references, but such an expression still // occupies a position in the row-level conjunct order. Record how many localized filters // precede the first unsafe original conjunct so constant pruning cannot jump over a - // slotless non-deterministic/error-preserving barrier. - if (in_safe_prefix && !_is_safe_to_pre_execute(conjunct)) { + // slotless non-deterministic/error-preserving barrier. An unsafe predicate is either kept + // on TableReader's post-materialization path by a standalone caller or carried only for + // analysis when FileScannerV2 owns the ordered suffix. + if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { in_safe_prefix = false; } - if (!in_safe_prefix) { - continue; - } + const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); - _constant_pruning_safe_filter_count = _table_filters.size(); + for (size_t filter_index = filters_before; filter_index < _table_filters.size(); + ++filter_index) { + _table_filters[filter_index].source_conjunct_index = conjunct_index; + _table_filters[filter_index].can_localize = in_safe_prefix; + } + if (in_safe_prefix) { + _constant_pruning_safe_filter_count = _table_filters.size(); + } + } + return Status::OK(); +} + +Status TableReader::_prepare_all_conjuncts_as_remaining() { + // Expression contexts carry mutable state (for example sequence/stateful functions). Select + // from the TableReader-owned contexts instead of reopening clones for every split. + _remaining_conjuncts.assign( + _conjuncts.begin(), + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count)); + return Status::OK(); +} + +Status TableReader::_prepare_remaining_conjuncts( + const FilterLocalizationResult& localization_result) { + DORIS_CHECK(localization_result.localized_filters.size() == _table_filters.size()); + std::vector localized_conjuncts(_conjuncts.size(), false); + for (size_t filter_index = 0; filter_index < _table_filters.size(); ++filter_index) { + if (!localization_result.localized_filters[filter_index]) { + continue; + } + const size_t source_index = _table_filters[filter_index].source_conjunct_index; + DORIS_CHECK(source_index < localized_conjuncts.size()); + localized_conjuncts[source_index] = true; + } + + _remaining_conjuncts.clear(); + for (size_t conjunct_index = 0; conjunct_index < _table_reader_owned_conjunct_count; + ++conjunct_index) { + if (localized_conjuncts[conjunct_index]) { + continue; + } + _remaining_conjuncts.push_back(_conjuncts[conjunct_index]); + } + return Status::OK(); +} + +Status TableReader::_filter_remaining_conjuncts(Block* block, size_t* rows) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + if (*rows == 0 || _remaining_conjuncts.empty()) { + return Status::OK(); + } + SCOPED_TIMER(_profile.residual_filter_timer); + const size_t rows_before_filter = *rows; + auto status = VExprContext::filter_block(_remaining_conjuncts, block, block->columns()); + if (!status.ok() && _format == FileFormat::ORC) { + status.prepend("Orc row reader nextBatch failed. reason = "); + } + RETURN_IF_ERROR(status); + *rows = block->columns() == 0 ? rows_before_filter : block->rows(); + if (_io_ctx != nullptr) { + _io_ctx->predicate_filtered_rows += rows_before_filter - *rows; } return Status::OK(); } @@ -722,7 +1006,12 @@ Status TableReader::create_next_reader(bool* eos) { if (_batch_size > 0) { _data_reader.reader->set_batch_size(_batch_size); } - Status st = _data_reader.reader->init(_runtime_state); + Status st; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_init_timer); + st = _data_reader.reader->init(_runtime_state); + } if (!st.ok()) { if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { *eos = true; @@ -753,10 +1042,16 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { const bool enable_mapping_timestamp_tz = _scan_params != nullptr && _scan_params->__isset.enable_mapping_timestamp_tz && _scan_params->enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_varbinary && + _scan_params->enable_mapping_varbinary; if (_format == FileFormat::PARQUET) { + // V2 must honor the scan contract directly; otherwise Hive STRING columns backed by an + // unannotated BYTE_ARRAY are silently exposed as VARBINARY and predicate bytes no longer + // match the table type. *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); return Status::OK(); } if (_format == FileFormat::ORC) { @@ -827,8 +1122,12 @@ std::unique_ptr create_file_description(const TFileRangeDes } Status TableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; + // Predicate localization belongs to the physical schema of one split. Clear the previous + // ownership before any early return so a pruned or failed split cannot leak it to the next one. + _remaining_conjuncts.clear(); _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; _condition_cache_digest_covers_current_split = options.condition_cache_digest.has_value(); if (options.condition_cache_digest.has_value()) { @@ -842,7 +1141,7 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _condition_cache_digest = _initial_condition_cache_digest; } if (options.conjuncts.has_value()) { - _conjuncts = *options.conjuncts; + RETURN_IF_ERROR(_replace_conjuncts(*options.conjuncts)); } // Update to current split format to handle ORC/PARQUET files in one table. _format = options.current_split_format; @@ -861,6 +1160,8 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _deletion_vector = nullptr; _aggregate_pushdown_tried = false; _remaining_table_level_count = -1; + _remaining_file_level_count = -1; + _current_split_uses_metadata_count = false; _current_reader_reached_eof = false; RETURN_IF_ERROR(_evaluate_partition_prune_conjuncts(options.partition_prune_conjuncts, &_current_split_pruned)); @@ -875,12 +1176,18 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { // active and no predicate can arrive later. The metadata path can return several batches for // one split; after its first synthetic batch there is no way to recover the real rows if a // runtime filter arrives before the next scheduler turn. - if (_push_down_agg_type == TPushAggOp::type::COUNT && options.all_runtime_filters_applied && + // Table-level metadata only contains the number of rows; it cannot evaluate an expression or + // the NULL state of a COUNT argument. Require the new FE's explicit empty argument list, which + // means COUNT(*)/COUNT(1). A non-empty list means COUNT(col), while nullopt comes from an old FE + // whose COUNT semantics are unknown during a BE-first rolling upgrade. + if (_push_down_agg_type == TPushAggOp::type::COUNT && _push_down_count_columns.has_value() && + _push_down_count_columns->empty() && options.all_runtime_filters_applied && _conjuncts.empty() && options.current_range.__isset.table_format_params && options.current_range.table_format_params.__isset.table_level_row_count) { DORIS_CHECK(options.current_range.table_format_params.table_level_row_count >= -1); _remaining_table_level_count = options.current_range.table_format_params.table_level_row_count; + _current_split_uses_metadata_count = _is_table_level_count_active(); } if (_is_table_level_count_active()) { return Status::OK(); @@ -904,7 +1211,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& // Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is // skipped, a later predicate could prune the split before the unsafe one reaches its // normal row-level evaluation point. - if (!_is_safe_to_pre_execute(conjunct)) { + if (!is_safe_to_pre_execute(conjunct)) { break; } std::set global_indices; @@ -940,7 +1247,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& can_filter_all); } -bool TableReader::_is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { +bool TableReader::is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); const auto root = conjunct->root(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index f62426e5aa5d9f..48e3d996ce22fd 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -54,9 +54,9 @@ #include "exprs/vexpr_context.h" #include "exprs/vexpr_fwd.h" #include "exprs/vslot_ref.h" +#include "format/table/deletion_vector.h" #include "format_v2/column_data.h" #include "format_v2/column_mapper.h" -#include "format_v2/deletion_vector.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" @@ -69,19 +69,23 @@ namespace doris { class Block; +struct DeleteFileDesc; class RuntimeState; } // namespace doris namespace doris::format { -struct DeleteFileDesc; using DeleteRows = std::vector; // Row-level predicates on table/global schema. They are rewritten to file-local expressions when -// possible, and remain the source of row-level filtering after localization. +// possible; otherwise TableReader evaluates them after final table-schema materialization. struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; + size_t source_conjunct_index = 0; + // False after the first unsafe source conjunct so file-local execution cannot reorder a later + // predicate ahead of stateful or error-preserving table semantics. + bool can_localize = true; }; struct ScanTask { @@ -99,6 +103,8 @@ struct ProjectedColumnBuildContext { }; struct ReadProfile { + RuntimeProfile::Counter* total_timer = nullptr; + RuntimeProfile::Counter* init_timer = nullptr; RuntimeProfile::Counter* num_delete_files = nullptr; RuntimeProfile::Counter* num_delete_rows = nullptr; RuntimeProfile::Counter* parse_delete_file_time = nullptr; @@ -110,11 +116,28 @@ struct ReadProfile { RuntimeProfile::Counter* exec_timer = nullptr; RuntimeProfile::Counter* prepare_split_timer = nullptr; RuntimeProfile::Counter* finalize_timer = nullptr; + RuntimeProfile::Counter* residual_filter_timer = nullptr; RuntimeProfile::Counter* create_reader_timer = nullptr; RuntimeProfile::Counter* pushdown_agg_timer = nullptr; RuntimeProfile::Counter* open_reader_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_prune_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_pruned_range_counter = nullptr; + RuntimeProfile::Counter* close_timer = nullptr; + RuntimeProfile::Counter* file_reader_total_timer = nullptr; + RuntimeProfile::Counter* file_reader_init_timer = nullptr; + RuntimeProfile::Counter* file_reader_schema_timer = nullptr; + RuntimeProfile::Counter* file_reader_mapper_timer = nullptr; + RuntimeProfile::Counter* file_reader_open_timer = nullptr; + RuntimeProfile::Counter* file_reader_get_block_timer = nullptr; + RuntimeProfile::Counter* file_reader_aggregate_timer = nullptr; + RuntimeProfile::Counter* file_reader_close_timer = nullptr; +}; + +struct MaterializedBlockStats { + bool has_materialized_input = false; + size_t rows = 0; + size_t bytes = 0; + size_t allocated_bytes = 0; }; struct TableReadOptions { @@ -123,6 +146,10 @@ struct TableReadOptions { const std::vector projected_columns; // All complex conjuncts from scan operator const VExprContextSPtrs conjuncts; + // Number of leading conjuncts whose row-level execution is owned by TableReader/FileReader. + // FileScannerV2 still passes the complete ordered list so mapping, pruning guards, aggregate + // eligibility, and condition-cache analysis see the exact query semantics. nullopt means all. + const std::optional table_reader_owned_conjunct_count = std::nullopt; // File format of the underlying data files, needed for reader initialization and reader-level // filter pushdown. const FileFormat format; @@ -135,6 +162,10 @@ struct TableReadOptions { const std::vector* file_slot_descs = nullptr; // Push-down aggregate type. const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; + // Table/global indices of explicit COUNT arguments. nullopt means an old FE did not send the + // semantic argument field, while an explicit empty vector means COUNT(*)/COUNT(1). Keeping + // those states separate prevents a rolling-upgrade plan from being reinterpreted by a new BE. + const std::optional> push_down_count_columns = std::nullopt; // Initial digest of predicates available during scanner open. Scanner-driven splits override it // with SplitReadOptions::condition_cache_digest after collecting late-arrival runtime filters. // A zero digest disables condition cache. @@ -147,7 +178,7 @@ struct SplitReadOptions { // Latest scanner conjuncts rewritten to table/global column indices. Runtime filters may // arrive after TableReader::init(), so scanner-driven splits replace the initial snapshot. // nullopt preserves the initial snapshot for standalone TableReader callers. - std::optional conjuncts; + std::optional conjuncts = std::nullopt; // Independent clones used for partition pruning because evaluation prepares and opens them // against a synthetic partition block before the file reader opens its row-level conjuncts. VExprContextSPtrs partition_prune_conjuncts; @@ -191,6 +222,11 @@ class TableReader { #ifdef BE_TEST size_t TEST_batch_size() const { return _batch_size; } + size_t TEST_conjunct_count() const { return _conjuncts.size(); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } + void TEST_set_condition_cache_hit_count(int64_t hits) { _condition_cache_hit_count = hits; } bool TEST_current_data_file_is_immutable() const { DORIS_CHECK(_current_task != nullptr); DORIS_CHECK(_current_task->data_file != nullptr); @@ -207,11 +243,37 @@ class TableReader { virtual Status prepare_split(const SplitReadOptions& options); virtual bool current_split_pruned() const { return _current_split_pruned; } + virtual bool current_split_uses_metadata_count() const { + return _current_split_uses_metadata_count; + } + + // Runtime filters that arrive after a split has opened cannot be pushed into that file reader. + // Keep their expression contexts in TableReader and evaluate them as residual predicates for + // the active reader; later splits can localize them normally. + virtual Status append_conjuncts(const VExprContextSPtrs& conjuncts); + + // Append a full ordered snapshot delta while marking only its leading prefix as owned by + // TableReader/FileReader. This non-virtual wrapper preserves the long-standing virtual API and + // carries the ownership boundary through hybrid readers to their children. + Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count); + + // Shared safety classification for deciding which ordered conjunct prefix may execute below + // Scanner without changing stateful or error-preserving semantics. + static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); + + virtual const MaterializedBlockStats& last_materialized_block_stats() const { + return _last_materialized_block_stats; + } // Discard the active split after the caller decides an error is ignorable, for example a // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. virtual Status abort_split() { + // Ignored open failures still spend time closing partially initialized readers. Include + // that recovery path in the common lifecycle profile so NOT_FOUND cannot become invisible. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader != nullptr) { RETURN_IF_ERROR(close_current_reader()); } else { @@ -220,7 +282,10 @@ class TableReader { } _delete_rows = nullptr; _remaining_table_level_count = -1; + _remaining_file_level_count = -1; + _current_split_uses_metadata_count = false; _current_split_pruned = false; + _remaining_conjuncts.clear(); return Status::OK(); } @@ -228,7 +293,9 @@ class TableReader { // advances across EOF, and closes exhausted readers. Subclasses provide protected hooks for // table-format-specific behavior. virtual Status get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _last_materialized_block_stats = {}; DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -245,6 +312,10 @@ class TableReader { RETURN_IF_ERROR(_read_table_level_count(block, eos)); return Status::OK(); } + if (_is_file_level_count_active()) { + RETURN_IF_ERROR(_read_file_level_count(block, eos)); + return Status::OK(); + } RETURN_IF_ERROR(create_next_reader(eos)); if (!_data_reader.reader) { DCHECK(*eos); @@ -277,8 +348,12 @@ class TableReader { _data_reader.block_template.clear_column_data( cast_set(_data_reader.file_block_layout.size())); size_t current_rows = 0; - RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, - ¤t_rows, ¤t_eof)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_get_block_timer); + RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, + ¤t_rows, ¤t_eof)); + } const bool stopped_during_read = _io_ctx != nullptr && _io_ctx->should_stop; if (current_rows == 0) { if (current_eof) { @@ -293,7 +368,7 @@ class TableReader { RETURN_IF_ERROR(_check_file_block_columns("after file reader get_block", current_rows)); #endif DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); - RETURN_IF_ERROR(finalize_chunk(block, current_rows)); + RETURN_IF_ERROR(finalize_chunk(block, ¤t_rows)); #ifndef NDEBUG RETURN_IF_ERROR( _check_table_block_columns("after finalize_chunk", block, current_rows)); @@ -302,6 +377,13 @@ class TableReader { _current_reader_reached_eof = !stopped_during_read; RETURN_IF_ERROR(close_current_reader()); } + if (current_rows == 0) { + // One materialized batch is one Scanner progress unit even when residual + // predicates reject every row. Returning here preserves row-budget and + // cancellation checks in Scanner::get_block(). + block->clear_column_data(_projected_columns.size()); + return Status::OK(); + } return Status::OK(); } } @@ -309,16 +391,21 @@ class TableReader { // Close the table reader and the currently active file reader. Subclasses that hold additional // table-format resources should override this and call TableReader::close() first. virtual Status close() { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader) { RETURN_IF_ERROR(close_current_reader()); } _current_task.reset(); _current_file_description.reset(); _remaining_table_level_count = -1; + _remaining_file_level_count = -1; + _current_split_uses_metadata_count = false; + _remaining_conjuncts.clear(); return Status::OK(); } - int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } + virtual int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } virtual std::string debug_string() const; @@ -360,6 +447,7 @@ class TableReader { Status create_next_reader(bool* eos); virtual Status create_file_reader(std::unique_ptr* reader); virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } + virtual void configure_mapper_options(TableColumnMapperOptions*) const {} virtual Status annotate_file_schema(std::vector* file_schema) { DORIS_CHECK(file_schema != nullptr); return Status::OK(); @@ -370,14 +458,23 @@ class TableReader { SCOPED_TIMER(_profile.open_reader_timer); // 1. Get file schema and create column mapping. std::vector file_schema; - RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_schema_timer); + RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + } // For Paimon/Hudi, FE can provide field ids through `history_schema_info`. Annotate the // file schema before column mapping when the table format maps columns by field id. RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); + configure_mapper_options(&_mapper_options); - _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_mapper_timer); + _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + } DORIS_CHECK(_data_reader.column_mapper != nullptr); RETURN_IF_ERROR(_data_reader.column_mapper->create_mapping(_projected_columns, _partition_values, file_schema)); @@ -390,14 +487,31 @@ class TableReader { // reader with the request. File scan request carries row-level expression filters and // file-level pruning hints. Only expression filters decide returned rows. auto file_request = std::make_shared(); + FilterLocalizationResult localization_result; RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( - _table_filters, _projected_columns, file_request.get(), _runtime_state)); + _table_filters, _projected_columns, file_request.get(), _runtime_state, + &localization_result)); bool constant_filter_pruned_split = false; RETURN_IF_ERROR(_evaluate_constant_filters(&constant_filter_pruned_split)); if (constant_filter_pruned_split) { RETURN_IF_ERROR(close_current_reader()); return Status::OK(); } + RETURN_IF_ERROR(_prepare_remaining_conjuncts(localization_result)); + // COUNT(*) has no semantic column argument, but Nereids retains a minimum-width scan slot + // so the scan node still has an output tuple. Record only the current non-predicate file + // columns before table-format hooks add row-position or equality-delete dependencies. This + // marker is independent of aggregate eligibility: with position deletes, for example, + // metadata COUNT must fall back to reading rows, but an arbitrary unsupported TIME_MILLIS + // placeholder still must not be validated or decoded merely to carry the surviving count. + if (_push_down_agg_type == TPushAggOp::type::COUNT && + _push_down_count_columns.has_value() && _push_down_count_columns->empty()) { + file_request->count_star_placeholder_columns.reserve( + file_request->non_predicate_columns.size()); + for (const auto& column : file_request->non_predicate_columns) { + file_request->count_star_placeholder_columns.push_back(column.column_id()); + } + } RETURN_IF_ERROR(customize_file_scan_request(file_request.get())); RETURN_IF_ERROR(_open_local_filter_exprs(*file_request)); _data_reader.file_block_layout.clear(); @@ -449,15 +563,23 @@ class TableReader { VLOG_DEBUG << "TableReader debug: " << debug_string(); } RETURN_IF_ERROR(_open_mapping_exprs()); - RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_open_timer); + RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + } RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); return Status::OK(); } Status _build_table_filters_from_conjuncts(); + Status _replace_conjuncts(const VExprContextSPtrs& conjuncts); + Status _prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared); + Status _prepare_remaining_conjuncts(const FilterLocalizationResult& localization_result); + Status _prepare_all_conjuncts_as_remaining(); + Status _filter_remaining_conjuncts(Block* block, size_t* rows); Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, bool* can_filter_all); - static bool _is_safe_to_pre_execute(const VExprContextSPtr& conjunct); Status _build_partition_prune_block(Block* block) const; Status _open_local_filter_exprs(const FileScanRequest& file_request); Status _init_reader_condition_cache(const FileScanRequest& file_request); @@ -476,7 +598,7 @@ class TableReader { if (table_filter.conjunct == nullptr) { continue; } - DORIS_CHECK(_is_safe_to_pre_execute(table_filter.conjunct)); + DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); // RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by the // row-level filter path through execute_filter(). Constant split pruning uses // VExprContext::execute() on a one-row synthetic block, so runtime filters must not be @@ -572,39 +694,64 @@ class TableReader { bool _is_table_level_count_active() const { return _remaining_table_level_count >= 0; } + bool _is_file_level_count_active() const { return _remaining_file_level_count >= 0; } + Status _materialize_count_rows(size_t rows, Block* block) const { DORIS_CHECK(block != nullptr); DORIS_CHECK(block->columns() > 0 || rows == 0); for (size_t column_idx = 0; column_idx < block->columns(); ++column_idx) { auto column = block->get_by_position(column_idx).type->create_column(); - column->resize(rows); + if (auto* nullable = check_and_get_column(*column)) { + // Metadata COUNT emits synthetic input rows for the unchanged upper aggregate. + // They must be non-NULL for COUNT(nullable_col), and constructing them explicitly + // also keeps every nullable null map boolean-valid in debug/ASAN block checks. + nullable->get_nested_column().insert_many_defaults(rows); + nullable->get_null_map_data().resize_fill(rows, 0); + } else { + column->insert_many_defaults(rows); + } block->replace_by_position(column_idx, std::move(column)); } return Status::OK(); } - Status _read_table_level_count(Block* block, bool* eos) { + Status _materialize_next_count_batch(int64_t* remaining_rows, Block* block) const { + DORIS_CHECK(remaining_rows != nullptr); + DORIS_CHECK(*remaining_rows > 0); + const int64_t batch_size = _runtime_state == nullptr + ? *remaining_rows + : static_cast(_runtime_state->batch_size()); + const auto rows = std::min(*remaining_rows, batch_size); + RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); + *remaining_rows -= rows; + return Status::OK(); + } + + Status _read_count_batch(int64_t* remaining_rows, Block* block, bool* eos) { DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(_push_down_agg_type == TPushAggOp::type::COUNT); - DORIS_CHECK(_remaining_table_level_count >= 0); - if (_remaining_table_level_count == 0) { - _remaining_table_level_count = -1; + DORIS_CHECK(remaining_rows != nullptr); + DORIS_CHECK(*remaining_rows >= 0); + if (*remaining_rows == 0) { + *remaining_rows = -1; _current_task.reset(); *eos = true; return Status::OK(); } - - const int64_t batch_size = _runtime_state == nullptr - ? _remaining_table_level_count - : static_cast(_runtime_state->batch_size()); - const auto rows = std::min(_remaining_table_level_count, batch_size); - RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); - _remaining_table_level_count -= rows; + RETURN_IF_ERROR(_materialize_next_count_batch(remaining_rows, block)); *eos = false; return Status::OK(); } + Status _read_table_level_count(Block* block, bool* eos) { + return _read_count_batch(&_remaining_table_level_count, block, eos); + } + + Status _read_file_level_count(Block* block, bool* eos) { + return _read_count_batch(&_remaining_file_level_count, block, eos); + } + void _append_file_scan_column(FileScanRequest* request, LocalColumnId column_id, std::vector* scan_columns) { DORIS_CHECK(request != nullptr); @@ -656,7 +803,11 @@ class TableReader { // close(), so it should remain idempotent. virtual Status close_current_reader() { _finalize_reader_condition_cache(); - RETURN_IF_ERROR(_data_reader.reader->close()); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_close_timer); + RETURN_IF_ERROR(_data_reader.reader->close()); + } _data_reader.reader.reset(); if (_data_reader.column_mapper != nullptr) { _data_reader.column_mapper->clear(); @@ -664,6 +815,7 @@ class TableReader { } _table_filters.clear(); _constant_pruning_safe_filter_count = 0; + _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); _data_reader.block_template.clear(); @@ -679,26 +831,41 @@ class TableReader { } } + void _reset_materialized_block_stats() { _last_materialized_block_stats = {}; } + + void _record_materialized_block_stats(const Block& block, size_t rows) { + _last_materialized_block_stats = { + .has_materialized_input = true, + .rows = rows, + .bytes = block.bytes(), + .allocated_bytes = block.allocated_bytes(), + }; + } + // Finalize file-local block to table/global schema block. - Status finalize_chunk(Block* block, const size_t rows) { + Status finalize_chunk(Block* block, size_t* rows) { + DORIS_CHECK(rows != nullptr); SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; - for (const auto& mapping : _data_reader.column_mapper->mappings()) { + const auto& mappings = _data_reader.column_mapper->mappings(); + for (const auto& mapping : mappings) { ColumnPtr column; - RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column)); - // Keep projection columns shared with the file block. Before the next read, - // clear_column_data() replaces shared columns with clone_empty(), so forcing a mutable - // deep copy here only duplicates the just-read batch (which can be several GB for - // nested string/map columns). - block->replace_by_position(idx, std::move(column)); + RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, + *rows, &column, + idx + 1 == mappings.size())); + block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } RETURN_IF_ERROR(materialize_virtual_columns(block)); // Enforce CHAR/VARCHAR length declared by the table schema after all file-to-table // materialization has finished. RETURN_IF_ERROR(_truncate_char_or_varchar_columns(block)); - return Status::OK(); + // Preserve the cost of materialization before residual predicates shrink the block. The + // scanner uses this snapshot for bounded progress and adaptive batch sizing. + _record_materialized_block_stats(*block, *rows); + // Predicate ownership is split-local: only predicates not acknowledged as exact by this + // split's FileScanRequest run here, after virtual/default/schema-evolution values exist. + return _filter_remaining_conjuncts(block, rows); } // Materialize virtual columns in the table block, such as Iceberg _row_id and @@ -862,31 +1029,7 @@ class TableReader { // - table VARCHAR(10), file STRING: truncate to 10 because STRING has no declared bound; // - table STRING, any file type: no truncation because the target has no bound. static bool _should_truncate_char_or_varchar_column(const ColumnMapping& mapping) { - if (mapping.table_type == nullptr) { - return false; - } - const auto table_type = remove_nullable(mapping.table_type); - const auto primitive_type = table_type->get_primitive_type(); - if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { - return false; - } - const auto target_len = assert_cast(table_type.get())->len(); - if (target_len <= 0) { - return false; - } - if (mapping.file_type == nullptr) { - return true; - } - const auto file_type = remove_nullable(mapping.file_type); - DORIS_CHECK(file_type != nullptr); - int file_len = -1; - if (file_type->get_primitive_type() == TYPE_VARCHAR || - file_type->get_primitive_type() == TYPE_CHAR || - file_type->get_primitive_type() == TYPE_STRING) { - file_len = assert_cast(file_type.get())->len(); - } - - return file_len < 0 || target_len < file_len; + return requires_char_or_varchar_truncation(mapping); } // Truncate a materialized CHAR/VARCHAR column in place by reusing the vectorized substring @@ -941,13 +1084,30 @@ class TableReader { FileAggregateRequest file_request; RETURN_IF_ERROR(_build_file_aggregate_request(_push_down_agg_type, &file_request)); FileAggregateResult file_result; - const auto status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + Status status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_aggregate_timer); + status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + } if (status.is()) { return Status::OK(); } RETURN_IF_ERROR(status); - RETURN_IF_ERROR( - _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); + if (_push_down_agg_type == TPushAggOp::type::COUNT) { + DORIS_CHECK(file_result.count >= 0); + // The upper aggregate consumes synthetic input rows, but emitting the whole metadata + // count in one block bypasses the runtime batch contract and can allocate by file size. + // Keep the remaining cardinality as split state and expose at most one batch per call. + _remaining_file_level_count = file_result.count; + _current_split_uses_metadata_count = true; + if (_remaining_file_level_count > 0) { + RETURN_IF_ERROR(_materialize_next_count_batch(&_remaining_file_level_count, block)); + } + } else { + RETURN_IF_ERROR( + _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); + } *pushed_down = true; RETURN_IF_ERROR(close_current_reader()); return Status::OK(); @@ -966,9 +1126,8 @@ class TableReader { if (!_all_runtime_filters_applied_for_split) { return false; } - // Scanner owns the original conjunct list and evaluates it after TableReader finalizes - // rows. Even a slotless conjunct that cannot become a TableFilter must see every source - // row before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. + // Even a slotless conjunct that cannot become a TableFilter must see every source row + // before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. if (!_conjuncts.empty()) { return false; } @@ -983,7 +1142,31 @@ class TableReader { return false; } if (agg_type == TPushAggOp::type::COUNT) { - return true; + // Old FEs do not serialize push_down_count_slot_ids. During the supported BE-first + // rolling upgrade, nullopt therefore means "COUNT semantics are unknown", not + // COUNT(*). Fall back to reading rows until the FE explicitly sends either an empty + // list for COUNT(*) or one slot for COUNT(col). + if (!_push_down_count_columns.has_value()) { + return false; + } + // COUNT(*) needs no column metadata. COUNT(col) currently supports one direct file + // column; multiple COUNT arguments fall back to the normal scan so every upper + // aggregate receives the original rows. + if (_push_down_count_columns->empty()) { + return true; + } + if (_push_down_count_columns->size() != 1) { + return false; + } + const auto& mapping = _push_down_count_mapping(); + // Metadata COUNT skips TableReader's normal materialization path. Only a trivial + // mapping is safe: for example, a nullable Parquet INT mapped to a NOT NULL table + // BIGINT normally needs both an INT->BIGINT cast and nullability validation. Counting + // footer values directly would bypass both operations and could hide invalid data. + return mapping.file_local_id.has_value() && mapping.file_type != nullptr && + mapping.table_type != nullptr && mapping.is_trivial && + mapping.virtual_column_type == TableVirtualColumnType::INVALID && + mapping.default_expr == nullptr; } // For MIN/MAX, only support direct file-to-table column mappings. The two emitted rows // must be enough for the upper MIN/MAX aggregate without evaluating default expressions or @@ -1002,6 +1185,22 @@ class TableReader { return true; } + static ColumnPtr _detach_column(ColumnPtr column) { + DORIS_CHECK(column.get() != nullptr); + return IColumn::mutate(std::move(column)); + } + + static ColumnPtr _take_and_detach_block_column(Block* block, int position) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(position >= 0 && position < static_cast(block->columns())); + auto& source = block->get_by_position(position); + ColumnPtr column = source.column; + // The final mapping no longer needs the file block. Release its COW owner before mutate(), + // otherwise nested MAP/STRING columns are deep-copied and a multi-GB payload can OOM. + block->replace_by_position(position, source.type->create_column()); + return _detach_column(std::move(column)); + } + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); @@ -1145,7 +1344,8 @@ class TableReader { } Status _materialize_mapping_column(const ColumnMapping& mapping, Block* current_block, - const size_t rows, ColumnPtr* column) { + const size_t rows, ColumnPtr* column, + bool take_projection_result = false) { if (!mapping.is_trivial && mapping.file_local_id.has_value() && !mapping.child_mappings.empty()) { DCHECK(mapping.projection != nullptr); @@ -1158,7 +1358,9 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), *mapping.file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; + ColumnPtr result_column = take_projection_result + ? _take_and_detach_block_column(current_block, res_id) + : current_block->get_by_position(res_id).column; RETURN_IF_ERROR( _materialize_complex_mapping_column(mapping, result_column, rows, column)); return Status::OK(); @@ -1177,8 +1379,12 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; - *column = std::move(result_column); + if (take_projection_result) { + *column = _take_and_detach_block_column(current_block, res_id); + } else { + ColumnPtr result_column = current_block->get_by_position(res_id).column; + *column = _detach_column(std::move(result_column)); + } return Status::OK(); } if (mapping.default_expr != nullptr) { @@ -1188,7 +1394,7 @@ class TableReader { mapping.default_expr, current_block, &result)); ColumnPtr result_column = result.column; RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); - *column = std::move(result_column); + *column = _detach_column(std::move(result_column)); } else { DORIS_CHECK(mapping.constant_index.has_value()); Block eval_block; @@ -1199,12 +1405,12 @@ class TableReader { mapping.default_expr, &eval_block, &result)); ColumnPtr result_column = result.column; RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); - *column = std::move(result_column); + *column = _detach_column(std::move(result_column)); } return Status::OK(); } ColumnPtr result_column = mapping.table_type->create_column_const_with_default_value(rows); - *column = std::move(result_column); + *column = _detach_column(std::move(result_column)); return Status::OK(); } @@ -1225,7 +1431,7 @@ class TableReader { RETURN_IF_ERROR(_materialize_map_mapping_column(mapping, file_column, rows, column)); break; default: - *column = file_column; + *column = _detach_column(file_column); break; } return Status::OK(); @@ -1318,7 +1524,10 @@ class TableReader { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { child_columns.push_back( - child_mapping->table_type->create_column_const_with_default_value(rows) + (child_mapping->initial_default_column + ? child_mapping->initial_default_column->clone_resized(rows) + : child_mapping->table_type + ->create_column_const_with_default_value(rows)) ->convert_to_full_column_if_const()); continue; } @@ -1461,24 +1670,19 @@ class TableReader { request->agg_type = agg_type; request->columns.clear(); if (agg_type == TPushAggOp::type::COUNT) { - // COUNT pushdown historically meant COUNT(*) and therefore carried no columns. For - // complex COUNT(col), materializing the full MAP/LIST/STRUCT value only to test the - // top-level NULL bit can be extremely expensive. When the scan projects exactly one - // directly-mapped complex column, pass that file column to the reader so formats such - // as Parquet can count the column shape from metadata/levels without decoding payload - // values like MAP value strings. Other COUNT cases stay on the existing row-count path - // to avoid changing count(*) semantics. - if (_data_reader.column_mapper->mappings().size() == 1) { - const auto& mapping = _data_reader.column_mapper->mappings()[0]; - if (mapping.file_local_id.has_value() && mapping.file_type != nullptr && - is_complex_type(remove_nullable(mapping.file_type)->get_primitive_type()) && - mapping.virtual_column_type == TableVirtualColumnType::INVALID && - mapping.default_expr == nullptr) { - FileAggregateRequest::Column column; - column.projection = - LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); - request->columns.push_back(std::move(column)); - } + DORIS_CHECK(_push_down_count_columns.has_value()); + // An empty explicit list is the semantic signal for COUNT(*). Do not inspect the + // mapping count: `SELECT COUNT(*) FROM t` may still project one nullable column because + // the planner keeps a placeholder slot. In a 10,000-row file where that arbitrary slot + // has 9,015 non-null values, passing the slot would ask Parquet/ORC metadata for + // COUNT(slot)=9,015 instead of the required row count 10,000. + if (!_push_down_count_columns->empty()) { + const auto& mapping = _push_down_count_mapping(); + DORIS_CHECK(mapping.file_local_id.has_value()); + FileAggregateRequest::Column column; + column.projection = + LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); + request->columns.push_back(std::move(column)); } return Status::OK(); } @@ -1495,16 +1699,22 @@ class TableReader { return Status::OK(); } + const ColumnMapping& _push_down_count_mapping() const { + DORIS_CHECK(_push_down_count_columns.has_value()); + DORIS_CHECK(_push_down_count_columns->size() == 1); + const auto mapping_it = + std::ranges::find(_data_reader.column_mapper->mappings(), + _push_down_count_columns->front(), &ColumnMapping::global_index); + // FileScannerV2 translates FE SlotIds through the same projected-column list used to build + // the mapper, so a missing mapping is an FE/BE contract violation rather than a fallback. + DORIS_CHECK(mapping_it != _data_reader.column_mapper->mappings().end()); + return *mapping_it; + } + Status _materialize_aggregate_pushdown_rows(TPushAggOp::type agg_type, const FileAggregateResult& file_result, Block* block) { - if (agg_type == TPushAggOp::type::COUNT) { - // COUNT pushdown is not a final count value. It emits `count` default rows so the - // upper COUNT(*) aggregate can count them and produce the final result, including - // zero rows when count is 0. - DORIS_CHECK(file_result.count >= 0); - return _materialize_count_rows(cast_set(file_result.count), block); - } + DORIS_CHECK(agg_type == TPushAggOp::type::MINMAX); // MIN/MAX pushdown emits two rows, min first and max second, for each projected column. // The upper MIN/MAX aggregate consumes those two rows to produce the final aggregate value. DORIS_CHECK(file_result.columns.size() == _data_reader.column_mapper->mappings().size()); @@ -1544,9 +1754,10 @@ class TableReader { for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); ++column_idx) { ColumnPtr table_column; - RETURN_IF_ERROR( - _materialize_mapping_column(_data_reader.column_mapper->mappings()[column_idx], - &file_block, 2, &table_column)); + RETURN_IF_ERROR(_materialize_mapping_column( + _data_reader.column_mapper->mappings()[column_idx], &file_block, 2, + &table_column, + column_idx + 1 == _data_reader.column_mapper->mappings().size())); block->replace_by_position(column_idx, std::move(table_column)); } return Status::OK(); @@ -1588,6 +1799,10 @@ class TableReader { // intentionally absent from that vector but must still act as ordering barriers. size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; + size_t _table_reader_owned_conjunct_count = 0; + std::optional _appended_table_reader_owned_conjunct_count; + VExprContextSPtrs _remaining_conjuncts; + MaterializedBlockStats _last_materialized_block_stats; ReadProfile _profile; // Parsed from row-position based delete files, including position delete and deletion vector. DeleteRows* _delete_rows = nullptr; @@ -1599,6 +1814,7 @@ class TableReader { const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; + std::optional> _push_down_count_columns; size_t _batch_size = 0; uint64_t _initial_condition_cache_digest = 0; uint64_t _condition_cache_digest = 0; @@ -1612,6 +1828,11 @@ class TableReader { int64_t _condition_cache_hit_count = 0; bool _current_reader_reached_eof = false; int64_t _remaining_table_level_count = -1; + int64_t _remaining_file_level_count = -1; + // True only after the active split selects a table-level row-count shortcut or successfully + // materializes COUNT rows from file metadata. FileScannerV2 uses this result, rather than the + // raw aggregate opcode, to keep adaptive batching enabled for normal row-scan fallbacks. + bool _current_split_uses_metadata_count = false; // Snapshot supplied by FileScannerV2 for the active split. It gates every shortcut that emits // irreversible aggregate rows, not only the table-level row-count shortcut in prepare_split(). bool _all_runtime_filters_applied_for_split = true; diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 30e1336e2a8bd7..94eab329687f08 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -124,9 +124,11 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren return diff; } -FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) { +FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, + const std::string& parent_counter) + : _profile(profile) { static const char* cache_profile = "FileCache"; - ADD_TIMER_WITH_LEVEL(profile, cache_profile, 2); + total_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, cache_profile, parent_counter.c_str(), 2); num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal", TUnit::UNIT, cache_profile, 1); num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT, @@ -200,6 +202,12 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) { } void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) const { + // These are the outer cache-path phases. Their sum keeps the group timer actionable instead of + // displaying zero while individual cache IO and coordination timers are non-zero. + COUNTER_UPDATE(total_time, statistics->local_io_timer + statistics->remote_io_timer + + statistics->peer_io_timer + statistics->remote_wait_timer + + statistics->write_cache_io_timer + + statistics->cache_get_or_set_timer); COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); COUNTER_UPDATE(num_peer_io_total, statistics->num_peer_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 297b2532c026b8..7015f4b04fb64f 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -70,6 +70,8 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren const FileCacheStatistics& previous); struct FileCacheProfileReporter { + RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* num_local_io_total = nullptr; RuntimeProfile::Counter* num_remote_io_total = nullptr; RuntimeProfile::Counter* num_peer_io_total = nullptr; @@ -110,7 +112,9 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* segment_footer_index_remote_io_timer = nullptr; RuntimeProfile::Counter* segment_footer_index_peer_io_timer = nullptr; - FileCacheProfileReporter(RuntimeProfile* profile); + explicit FileCacheProfileReporter( + RuntimeProfile* profile, + const std::string& parent_counter = RuntimeProfile::ROOT_COUNTER); void update(const FileCacheStatistics* statistics) const; }; diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index e8005ef821ba3a..91eafdba4319fe 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -32,6 +32,7 @@ #include "common/status.h" #include "core/custom_allocator.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -657,7 +658,9 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File std::function sync_buffer = nullptr; if (profile != nullptr) { const char* prefetch_buffered_reader = "PrefetchBufferedReader"; - ADD_TIMER(profile, prefetch_buffered_reader); + auto* total_time = + ADD_CHILD_TIMER(profile, prefetch_buffered_reader, + file_scan_profile::parent_or_root(profile, file_scan_profile::IO)); auto copy_time = ADD_CHILD_TIMER(profile, "CopyTime", prefetch_buffered_reader); auto read_time = ADD_CHILD_TIMER(profile, "ReadTime", prefetch_buffered_reader); auto prefetch_request_io = @@ -669,6 +672,7 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File auto request_bytes = ADD_CHILD_COUNTER(profile, "RequestBytes", TUnit::BYTES, prefetch_buffered_reader); sync_buffer = [=](PrefetchBuffer& buf) { + COUNTER_UPDATE(total_time, buf._statis.copy_time + buf._statis.read_time); COUNTER_UPDATE(copy_time, buf._statis.copy_time); COUNTER_UPDATE(read_time, buf._statis.read_time); COUNTER_UPDATE(prefetch_request_io, buf._statis.prefetch_request_io); @@ -926,7 +930,9 @@ RangeCacheFileReader::RangeCacheFileReader(RuntimeProfile* profile, io::FileRead if (_profile != nullptr) { const char* random_profile = "RangeCacheFileReader"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, random_profile, 1); _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, @@ -987,6 +993,7 @@ Status RangeCacheFileReader::read_at_impl(size_t offset, Slice result, size_t* b void RangeCacheFileReader::_collect_profile_before_close() { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _cache_statistics.request_time); COUNTER_UPDATE(_request_io, _cache_statistics.request_io); COUNTER_UPDATE(_request_bytes, _cache_statistics.request_bytes); COUNTER_UPDATE(_request_time, _cache_statistics.request_time); diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index f88fb2a575fce6..04566bea0951da 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -38,6 +38,7 @@ #include "io/fs/path.h" #include "io/fs/s3_file_reader.h" #include "io/io_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -181,6 +182,7 @@ class RangeCacheFileReader : public io::FileReader { bool _closed = false; RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _request_bytes = nullptr; RuntimeProfile::Counter* _request_time = nullptr; RuntimeProfile::Counter* _read_to_cache_time = nullptr; @@ -303,7 +305,9 @@ class MergeRangeFileReader : public io::FileReader { if (_profile != nullptr) { const char* random_profile = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", random_profile, 1); _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", random_profile, 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, @@ -352,6 +356,7 @@ class MergeRangeFileReader : public io::FileReader { void _collect_profile_before_close() override { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -365,6 +370,7 @@ class MergeRangeFileReader : public io::FileReader { } private: + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _copy_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index 7fd8b5df36e6f5..505a2da1976343 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -32,6 +32,7 @@ #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/hdfs_util.h" +#include "runtime/file_scan_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" #include "service/backend_options.h" @@ -84,7 +85,9 @@ HdfsFileReader::HdfsFileReader(Path path, std::string fs_name, FileHandleCache:: if (_profile != nullptr && is_hdfs(_fs_name)) { #ifdef USE_HADOOP_HDFS const char* hdfs_profile_name = "HdfsIO"; - ADD_TIMER(_profile, hdfs_profile_name); + _total_read_time = + ADD_CHILD_TIMER(_profile, hdfs_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); _hdfs_profile.total_bytes_read = ADD_CHILD_COUNTER(_profile, "TotalBytesRead", TUnit::BYTES, hdfs_profile_name); _hdfs_profile.total_local_bytes_read = @@ -118,6 +121,7 @@ Status HdfsFileReader::close() { Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, const IOContext* io_ctx) { + SCOPED_TIMER(_total_read_time); auto st = do_read_at_impl(offset, result, bytes_read, io_ctx); if (!st.ok()) { _handle = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index 08f98bca29af0c..7bb73f30909452 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -88,6 +88,7 @@ class HdfsFileReader final : public FileReader { CachedHdfsFileHandle* _handle = nullptr; // owned by _cached_file_handle std::atomic _closed = false; RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* _total_read_time = nullptr; int64_t _mtime; #ifdef USE_HADOOP_HDFS HDFSProfile _hdfs_profile; diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 4eaa10f3311e06..af8dde36d2df50 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -37,6 +37,7 @@ #include "io/fs/err_utils.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -214,7 +215,9 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea void S3FileReader::_collect_profile_before_close() { if (_profile != nullptr) { const char* s3_profile_name = "S3Profile"; - ADD_TIMER(_profile, s3_profile_name); + auto* total_time = + ADD_CHILD_TIMER(_profile, s3_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); RuntimeProfile::Counter* total_get_request_counter = ADD_CHILD_COUNTER(_profile, "TotalGetRequest", TUnit::UNIT, s3_profile_name); RuntimeProfile::Counter* too_many_request_err_counter = @@ -231,6 +234,7 @@ void S3FileReader::_collect_profile_before_close() { COUNTER_UPDATE(too_many_request_sleep_time, _s3_stats.too_many_request_sleep_time_ms); COUNTER_UPDATE(total_bytes_read, _s3_stats.total_bytes_read); COUNTER_UPDATE(total_get_request_time_ns, _s3_stats.total_get_request_time_ns); + COUNTER_UPDATE(total_time, _s3_stats.total_get_request_time_ns); } } diff --git a/be/src/runtime/file_scan_profile.h b/be/src/runtime/file_scan_profile.h new file mode 100644 index 00000000000000..4ec71c4f4820d4 --- /dev/null +++ b/be/src/runtime/file_scan_profile.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/runtime_profile.h" + +namespace doris::file_scan_profile { + +inline constexpr const char* SCANNER = "FileScannerV2"; +inline constexpr const char* TABLE_READER = "TableReader"; +inline constexpr const char* FILE_READER = "FileReader"; +inline constexpr const char* IO = "IO"; + +struct Hierarchy { + RuntimeProfile::Counter* scanner = nullptr; + RuntimeProfile::Counter* table_reader = nullptr; + RuntimeProfile::Counter* file_reader = nullptr; + RuntimeProfile::Counter* io = nullptr; +}; + +inline Hierarchy ensure_hierarchy(RuntimeProfile* profile) { + if (profile == nullptr) { + return {}; + } + // RuntimeProfile stores one flat counter namespace and a separate parent map. Create every + // layer in order so later format and IO profiles cannot accidentally become scanner siblings. + auto* scanner = ADD_TIMER_WITH_LEVEL(profile, SCANNER, 1); + auto* table_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, TABLE_READER, SCANNER, 1); + auto* file_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, FILE_READER, TABLE_READER, 1); + auto* io = ADD_CHILD_TIMER_WITH_LEVEL(profile, IO, FILE_READER, 1); + return {scanner, table_reader, file_reader, io}; +} + +inline const char* parent_or_root(RuntimeProfile* profile, const char* preferred_parent) { + return profile != nullptr && profile->get_counter(preferred_parent) != nullptr + ? preferred_parent + : RuntimeProfile::ROOT_COUNTER.c_str(); +} + +} // namespace doris::file_scan_profile diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 75f05269853039..a12c8743ee4cfe 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -155,6 +155,18 @@ RuntimeState::RuntimeState(const TUniqueId& query_id, int32_t fragment_id, DCHECK(_query_mem_tracker != nullptr); } +RuntimeState::RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals) + : _profile(""), + _load_channel_profile(""), + _obj_pool(new ObjectPool()), + _unreported_error_idx(0), + _per_fragment_instance_idx(0) { + Status status = init(TUniqueId(), query_options, query_globals, nullptr); + _exec_env = ExecEnv::GetInstance(); + init_mem_trackers(""); + DCHECK(status.ok()); +} + RuntimeState::RuntimeState(const TQueryGlobals& query_globals) : _profile(""), _load_channel_profile(""), diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 4fdce6c07c2d0d..40ba041ffab388 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -100,20 +100,16 @@ class RuntimeState { const TQueryGlobals& query_globals, ExecEnv* exec_env, const std::shared_ptr& query_mem_tracker); + // Keep this production-visible because master format_v2 constructs a lightweight state for + // page-cache reads; a BE_TEST-only overload would hide release-build incompatibilities. + RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals); + // RuntimeState for executing expr in fe-support. RuntimeState(const TQueryGlobals& query_globals); // for job task only RuntimeState(); -#ifdef BE_TEST - // Compatibility constructor for format_v2 tests backported to branch-4.1. - RuntimeState(const TQueryOptions& query_options, const TQueryGlobals& query_globals) - : RuntimeState(query_globals) { - set_query_options(query_options); - } -#endif - // Empty d'tor to avoid issues with unique_ptr. MOCK_DEFINE(virtual) ~RuntimeState(); diff --git a/be/src/storage/segment/adaptive_block_size_predictor.cpp b/be/src/storage/segment/adaptive_block_size_predictor.cpp index d8cc700f579853..7a5ad573a2ea6c 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.cpp +++ b/be/src/storage/segment/adaptive_block_size_predictor.cpp @@ -32,11 +32,14 @@ AdaptiveBlockSizePredictor::AdaptiveBlockSizePredictor(size_t preferred_block_si _metadata_hint_bytes_per_row(metadata_hint_bytes_per_row) {} void AdaptiveBlockSizePredictor::update(const Block& block) { - size_t rows = block.rows(); + update(block.rows(), block.bytes()); +} + +void AdaptiveBlockSizePredictor::update(size_t rows, size_t bytes) { if (rows == 0) { return; } - double cur = static_cast(block.bytes()) / static_cast(rows); + double cur = static_cast(bytes) / static_cast(rows); if (!_has_history) { _bytes_per_row = cur; diff --git a/be/src/storage/segment/adaptive_block_size_predictor.h b/be/src/storage/segment/adaptive_block_size_predictor.h index e03f18c2a536d2..f327fec8517f63 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.h +++ b/be/src/storage/segment/adaptive_block_size_predictor.h @@ -60,6 +60,7 @@ class AdaptiveBlockSizePredictor { // Update EWMA estimates from a completed batch. Must be called only when block.rows() > 0 // and the batch returned Status::OK(). void update(const Block& block); + void update(size_t rows, size_t bytes); // Predict how many rows the next batch should read. // Never exceeds |block_size_rows|; never returns less than 1. diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index f800f72ebbbdcb..5229527d1122f4 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -770,11 +770,19 @@ class SnappyBlockCompression : public BlockCompressionCodec { } Status decompress(const Slice& input, Slice* output) override { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(input.data, input.size, &uncompressed_size)) { + return Status::InvalidArgument("Fail to get Snappy uncompressed length"); + } + // RawUncompress has no capacity argument, so reject an undersized destination first. + if (uncompressed_size > output->size) { + return Status::InvalidArgument("Snappy output size {} exceeds buffer capacity {}", + uncompressed_size, output->size); + } if (!snappy::RawUncompress(input.data, input.size, output->data)) { return Status::InvalidArgument("Fail to do Snappy decompress"); } - // NOTE: GetUncompressedLength only takes O(1) time - snappy::GetUncompressedLength(input.data, input.size, &output->size); + output->size = uncompressed_size; return Status::OK(); } diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 2987d1dc099ce8..3c1af89e228f3f 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -376,6 +376,17 @@ void CpuInfo::_get_cache_info(long cache_sizes[NUM_CACHE_LEVELS], #endif } +long CpuInfo::get_l2_cache_size() { + static const long l2_cache_size = [] { + long cache_sizes[NUM_CACHE_LEVELS] = {}; + long cache_line_sizes[NUM_CACHE_LEVELS] = {}; + _get_cache_info(cache_sizes, cache_line_sizes); + constexpr long DEFAULT_L2_CACHE_SIZE = 256 * 1024; + return cache_sizes[L2_CACHE] > 0 ? cache_sizes[L2_CACHE] : DEFAULT_L2_CACHE_SIZE; + }(); + return l2_cache_size; +} + std::string CpuInfo::debug_string() { DCHECK(initialized_); std::stringstream stream; diff --git a/be/src/util/cpu_info.h b/be/src/util/cpu_info.h index 19548ecc964659..c75b814721cc57 100644 --- a/be/src/util/cpu_info.h +++ b/be/src/util/cpu_info.h @@ -146,6 +146,10 @@ class CpuInfo { return model_name_; } + // Some virtualized hosts do not expose cache topology. Always return a positive value so hot + // decode paths can use this threshold without repeating platform-specific guards. + static long get_l2_cache_size(); + static std::string debug_string(); /// A utility class for temporarily disabling CPU features. Usage: diff --git a/be/src/util/jdbc_utils.cpp b/be/src/util/jdbc_utils.cpp new file mode 100644 index 00000000000000..cf091f6c9ff425 --- /dev/null +++ b/be/src/util/jdbc_utils.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/jdbc_utils.h" + +#include + +#include "cloud/config.h" +#include "common/config.h" +#include "runtime/plugin/cloud_plugin_downloader.h" + +namespace doris { + +Status JdbcUtils::resolve_driver_url(const std::string& url, std::string* result_url) { + // Already a full URL (e.g. "file:///path/to/driver.jar" or "hdfs://...") + if (url.find(":/") != std::string::npos) { + *result_url = url; + return Status::OK(); + } + + const char* doris_home = std::getenv("DORIS_HOME"); + if (doris_home == nullptr) { + return Status::InternalError("DORIS_HOME environment variable is not set"); + } + + std::string default_url = std::string(doris_home) + "/plugins/jdbc_drivers"; + std::string default_old_url = std::string(doris_home) + "/jdbc_drivers"; + + if (config::jdbc_drivers_dir == default_url) { + std::string target_path = default_url + "/" + url; + std::string old_target_path = default_old_url + "/" + url; + if (std::filesystem::exists(target_path)) { + *result_url = "file://" + target_path; + } else if (std::filesystem::exists(old_target_path)) { + *result_url = "file://" + old_target_path; + } else if (config::is_cloud_mode()) { + // In cloud/elastic deployments, BEs are ephemeral and driver JARs + // may not exist locally. Try downloading from cloud storage. + std::string downloaded_path; + RETURN_IF_ERROR(CloudPluginDownloader::download_from_cloud( + CloudPluginDownloader::PluginType::JDBC_DRIVERS, url, target_path, + &downloaded_path)); + *result_url = "file://" + downloaded_path; + } else { + return Status::InternalError("JDBC driver file does not exist: " + url); + } + } else { + *result_url = "file://" + config::jdbc_drivers_dir + "/" + url; + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/util/jdbc_utils.h b/be/src/util/jdbc_utils.h new file mode 100644 index 00000000000000..8c6004c4c24882 --- /dev/null +++ b/be/src/util/jdbc_utils.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" + +namespace doris { + +/** + * Utility functions for JDBC driver management. + */ +class JdbcUtils { +public: + /** + * Resolve a JDBC driver URL to an absolute file:// URL. + * + * FE sends just the JAR filename (e.g. "mysql-connector-java-8.0.25.jar"). + * This method resolves it to a full file:// URL by searching in the + * configured jdbc_drivers_dir (or the default DORIS_HOME/plugins/jdbc_drivers). + * + * If the URL already contains ":/", it is assumed to be a full URL and + * returned as-is. + * + * @param url The driver URL from FE (may be just a filename) + * @param result_url Output: the resolved file:// URL + * @return Status::OK on success, or InternalError if the file is not found + */ + static Status resolve_driver_url(const std::string& url, std::string* result_url); +}; + +} // namespace doris diff --git a/be/src/util/timezone_utils.cpp b/be/src/util/timezone_utils.cpp index 48227963cd3341..c18e3cbb7a156f 100644 --- a/be/src/util/timezone_utils.cpp +++ b/be/src/util/timezone_utils.cpp @@ -154,7 +154,14 @@ bool TimezoneUtils::find_cctz_time_zone(const std::string& timezone, cctz::time_ ctz = it->second; return true; } - return false; + // V2 readers and Iceberg writers may resolve UTC/fixed offsets before ExecEnv preloads the + // timezone cache, so retain the cache fast path but handle those self-contained zones here. + const auto normalized = to_lower_copy(timezone); + if (normalized == "utc" || normalized == "etc/utc" || normalized == "zulu") { + ctz = cctz::utc_time_zone(); + return true; + } + return parse_tz_offset_string(timezone, ctz); } bool TimezoneUtils::parse_tz_offset_string(const std::string& timezone, cctz::time_zone& ctz) { diff --git a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp index 69cf458e2fdc5f..18e4aa8a1cf848 100644 --- a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp @@ -365,7 +365,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadIntegersFromUnsignedSources) { TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalValues) { { - std::vector values = {0, 127, 255, 32767, 65535, -1}; + std::vector values = {0, 127, 255}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); auto result = read_column(std::make_shared(), view); @@ -375,12 +375,9 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal EXPECT_EQ(0, column.get_element(0)); EXPECT_EQ(127, column.get_element(1)); EXPECT_EQ(255, column.get_element(2)); - EXPECT_EQ(255, column.get_element(3)); - EXPECT_EQ(255, column.get_element(4)); - EXPECT_EQ(255, column.get_element(5)); } { - std::vector values = {32767, 65535, -1}; + std::vector values = {32767, 65535}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 16, false); auto result = read_column(std::make_shared(), view); @@ -389,7 +386,6 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal ASSERT_EQ(values.size(), column.size()); EXPECT_EQ(32767, column.get_element(0)); EXPECT_EQ(65535, column.get_element(1)); - EXPECT_EQ(65535, column.get_element(2)); } { std::vector values = {-1}; @@ -414,17 +410,42 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal } } -TEST(DataTypeSerDeDecodedValuesTest, ReadSignedLogicalIntegersCastsPhysicalValues) { +TEST(DataTypeSerDeDecodedValuesTest, StrictSignedLogicalIntegersRejectOutOfRangePhysicalCarrier) { std::vector values = {127, 128, 255, -1}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, true); + // Carrier bounds are metadata validation: strict scans reject malformed values, while + // permissive scans intentionally preserve the declared bit-width conversion. + view.enable_strict_mode = true; auto result = read_column(std::make_shared(), view); + expect_data_quality_error(result.status); + EXPECT_EQ(result.column->size(), 0); +} + +TEST(DataTypeSerDeDecodedValuesTest, StrictUnsignedLogicalIntegersRejectOutOfRangePhysicalCarrier) { + std::vector values = {255, 256}; + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); + view.enable_strict_mode = true; + auto result = read_column(std::make_shared(), view); + expect_data_quality_error(result.status); + EXPECT_EQ(result.column->size(), 0); +} + +TEST(DataTypeSerDeDecodedValuesTest, + NonStrictNullableLogicalIntegerCarrierPreservesBitWidthCompatibility) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {127, 1000, -128}; + std::vector null_map(values.size(), 0); + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values, &null_map), 8, + true); + auto result = read_column(type, view); + ASSERT_TRUE(result.status.ok()) << result.status; - const auto& column = assert_cast(*result.column); - ASSERT_EQ(values.size(), column.size()); - EXPECT_EQ(static_cast(127), column.get_element(0)); - EXPECT_EQ(static_cast(-128), column.get_element(1)); - EXPECT_EQ(static_cast(-1), column.get_element(2)); - EXPECT_EQ(static_cast(-1), column.get_element(3)); + const auto& nullable = assert_cast(*result.column); + const auto& nested = assert_cast(nullable.get_nested_column()); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_FALSE(nullable.is_null_at(1)); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_EQ(nested.get_data(), (ColumnInt8::Container {127, -24, -128})); } TEST(DataTypeSerDeDecodedValuesTest, ReadFloatAndDouble) { @@ -1633,7 +1654,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadFieldPrimitiveValues) { TEST(DataTypeSerDeDecodedValuesTest, ReadFieldLogicalIntegerCastsPhysicalValue) { { - std::vector values = {32767}; + std::vector values = {255}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); auto field = read_field(std::make_shared(), view); diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp new file mode 100644 index 00000000000000..b7a4329f0d261b --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -0,0 +1,891 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { +namespace { + +#pragma pack(1) +struct TestParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(TestParquetInt96Timestamp) == 12); + +constexpr int32_t JULIAN_UNIX_EPOCH = 2440588; +constexpr int64_t NANOS_PER_DAY = 86400000000000LL; + +class TestParquetDecodeSource final : public ParquetDecodeSource { +public: + template + void set_fixed_values(const std::vector& values) { + _value_width = sizeof(T); + _fixed_values.resize(values.size() * sizeof(T)); + memcpy(_fixed_values.data(), values.data(), _fixed_values.size()); + } + + void set_fixed_bytes(std::vector values, size_t value_width) { + _fixed_values = std::move(values); + _value_width = value_width; + } + + void set_dictionary(std::vector values, size_t value_width, + std::vector indices) { + _dictionary = std::move(values); + _dictionary_width = value_width; + _indices = std::move(indices); + _index_offset = 0; + ++_dictionary_generation; + } + + template + void set_fixed_dictionary(const std::vector& values, std::vector indices) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + set_dictionary(std::move(encoded), sizeof(T), std::move(indices)); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_LE((_fixed_offset + num_values) * _value_width, _fixed_values.size()); + const uint8_t* begin = _fixed_values.data() + _fixed_offset * _value_width; + _fixed_offset += num_values; + return consumer.consume(begin, num_values, _value_width); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_LE(_binary_offset + num_values, _binary_refs.size()); + const StringRef* begin = _binary_refs.data() + _binary_offset; + _binary_offset += num_values; + return consumer.consume(begin, num_values); + } + + Status skip_values(size_t num_values) override { + _fixed_offset += num_values; + _binary_offset += num_values; + _index_offset += num_values; + return Status::OK(); + } + + bool has_dictionary() const override { return !_dictionary.empty(); } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + size_t dictionary_size() const override { + return _dictionary_width == 0 ? 0 : _dictionary.size() / _dictionary_width; + } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + ++_dictionary_decode_calls; + return fixed_consumer.consume(_dictionary.data(), dictionary_size(), _dictionary_width); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + DORIS_CHECK_LE(_index_offset + num_values, _indices.size()); + indices->assign(_indices.begin() + _index_offset, + _indices.begin() + _index_offset + num_values); + _index_offset += num_values; + return Status::OK(); + } + + size_t dictionary_decode_calls() const { return _dictionary_decode_calls; } + +private: + std::vector _fixed_values; + std::vector _binary_refs; + std::vector _dictionary; + std::vector _indices; + size_t _value_width = 0; + size_t _dictionary_width = 0; + size_t _fixed_offset = 0; + size_t _binary_offset = 0; + size_t _index_offset = 0; + uint64_t _dictionary_generation = 0; + size_t _dictionary_decode_calls = 0; +}; + +TEST(DataTypeSerDeParquetTest, MaterializesLogicalUnsignedIntegersDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 0, 7}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 32, + .logical_integer_is_signed = false}; + ParquetMaterializationState state; + DataTypeInt64 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 4294967295LL); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 7); +} + +TEST(DataTypeSerDeParquetTest, NonStrictNarrowUnsignedIntegersPreserveBitWidthCompatibility) { + TestParquetDecodeSource source; + source.set_fixed_values( + {0, 255, 32767, 65535, std::numeric_limits::max(), -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = false}; + IColumn::Filter null_map(6, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt16 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 6, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data, (ColumnInt16::Container {0, 255, 255, 255, 255, 255})); + EXPECT_EQ(null_map, IColumn::Filter(6, 0)); +} + +TEST(DataTypeSerDeParquetTest, LogicalIntegerRejectsOutOfRangePhysicalCarrier) { + TestParquetDecodeSource source; + source.set_fixed_values({1000}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = true}; + ParquetMaterializationState state; + state.enable_strict_mode = true; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .is()); + EXPECT_EQ(column->size(), 0); +} + +TEST(DataTypeSerDeParquetTest, NonStrictLogicalIntegerDictionaryPreservesBitWidthCompatibility) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {-24, -24})); + EXPECT_EQ(null_map, IColumn::Filter({0, 0})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFloat16Directly) { + TestParquetDecodeSource source; + source.set_fixed_values({0x3C00, 0xC000, 0x7C00}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::FLOAT16, + .type_length = 2, + .logical_float16 = true}; + ParquetMaterializationState state; + DataTypeFloat32 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_FLOAT_EQ(data[0], 1.0F); + EXPECT_FLOAT_EQ(data[1], -2.0F); + EXPECT_TRUE(std::isinf(data[2])); +} + +TEST(DataTypeSerDeParquetTest, RescalesFixedBinaryDecimalDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x04, 0xD2, 0xFB, 0x2E}, 2); // 12.34 and -12.34 at scale 2. + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 2, + .decimal_precision = 4, + .decimal_scale = 2}; + ParquetMaterializationState state; + DataTypeDecimal64 type(18, 3); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 12340); + EXPECT_EQ(data[1].value, -12340); +} + +TEST(DataTypeSerDeParquetTest, RescalesExactIntegerDecimalsWithoutRowFailures) { + TestParquetDecodeSource source; + source.set_fixed_values({12300, -98700, 2147483600}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 10, + .decimal_scale = 4}; + ParquetMaterializationState state; + DataTypeDecimal64 type(9, 2); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + // Exact scale reduction can stay on the fast integer path without per-row failure bookkeeping. + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, -987); + EXPECT_EQ(data[2].value, 21474836); +} + +TEST(DataTypeSerDeParquetTest, DecimalScaleDownRejectsLossyPlainAndDictionaryValues) { + ParquetDecodeContext plain_context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 6, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + { + TestParquetDecodeSource source; + source.set_fixed_values({1230, 1234}); + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, 0); + EXPECT_EQ(null_map, IColumn::Filter({0, 1})); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({1234}); + ParquetMaterializationState state; + state.enable_strict_mode = true; + auto column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_dictionary({1230, 1234}, {1, 0, 1}); + auto dictionary_context = plain_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde() + ->read_column_from_parquet(*column, source, dictionary_context, 3, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[1].value, 123); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalUsesWideIntermediateBeforeNarrowing) { + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 19, + .decimal_scale = 0}; + DataTypeDecimal32 type(9, 0); + constexpr int64_t WRAPS_TO_ONE_IN_INT32 = 4294967297LL; + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_fixed_dictionary({WRAPS_TO_ONE_IN_INT32}, {0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_values({WRAPS_TO_ONE_IN_INT32}); + context.encoding = ParquetValueEncoding::PLAIN; + } + IColumn::Filter null_map(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1})); + EXPECT_EQ(assert_cast(*column).get_data()[0].value, 0); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalAcceptsSignExtendedBinaryAfterWideExactScaling) { + const std::vector values {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xCE, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFB, 0x32}; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 8, + .decimal_precision = 19, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_dictionary(values, 8, {1, 0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_bytes(values, 8); + context.encoding = ParquetValueEncoding::PLAIN; + } + ParquetMaterializationState state; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, dictionary ? -123 : 123); + EXPECT_EQ(data[1].value, dictionary ? 123 : -123); + } +} + +TEST(DataTypeSerDeParquetTest, ReusesTypedDictionary) { + TestParquetDecodeSource source; + std::vector dictionary {'a', 'a', 'b', 'b'}; + source.set_dictionary(std::move(dictionary), 2, {1, 0, 1, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2}; + ParquetMaterializationState state; + DataTypeString type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + ASSERT_EQ(state.typed_dictionary->size(), 2); + EXPECT_EQ(column->get_data_at(0).to_string(), "bb"); + EXPECT_EQ(column->get_data_at(1).to_string(), "aa"); + EXPECT_EQ(column->get_data_at(2).to_string(), "bb"); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 1); + EXPECT_EQ(column->get_data_at(3).to_string(), "aa"); + + source.set_dictionary({'c', 'c'}, 2, {0}); + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 2); + EXPECT_EQ(column->get_data_at(4).to_string(), "cc"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDictionaryIndicesWithoutValues) { + TestParquetDecodeSource source; + source.set_dictionary({'a', 'a', 'b', 'b'}, 2, {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2, + .dictionary_index_only = true}; + ParquetMaterializationState state; + DataTypeString type; + auto column = ColumnInt32::create(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = column->get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 1); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 1); + EXPECT_FALSE(state.typed_dictionary); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDateDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, 1, -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ParquetMaterializationState state; + DataTypeDateV2 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 2), "1969-12-31"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampUnitsAndNegativeEpochDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, -1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1969-12-31 23:59:59.999999"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000001"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimeUnitsDirectly) { + struct TimeUnitCase { + ParquetTimeUnit unit; + int64_t units_per_day; + int64_t expected_last_micros; + }; + const std::vector cases { + {ParquetTimeUnit::MILLIS, 86400000, 86399999000}, + {ParquetTimeUnit::MICROS, 86400000000, 86399999999}, + {ParquetTimeUnit::NANOS, 86400000000000, 86399999999}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(static_cast(test_case.unit)); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = test_case.unit}; + DataTypeTimeV2 type(6); + + TestParquetDecodeSource source; + source.set_fixed_values({0, test_case.units_per_day - 1}); + ParquetMaterializationState state; + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0], 0); + EXPECT_EQ(data[1], test_case.expected_last_micros); + + TestParquetDecodeSource nullable_source; + nullable_source.set_fixed_values({-1, test_case.units_per_day}); + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState nullable_state; + nullable_state.conversion_failure_null_map = &null_map; + auto nullable_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*nullable_column, nullable_source, context, + 2, nullable_state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + + TestParquetDecodeSource strict_source; + strict_source.set_fixed_values({-1}); + ParquetMaterializationState strict_state; + auto strict_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*strict_column, strict_source, context, 1, + strict_state) + .ok()); + EXPECT_EQ(strict_column->size(), 0); + + TestParquetDecodeSource unused_invalid_dictionary; + unused_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day - 1, test_case.units_per_day}, {1, 2}); + auto dictionary_context = context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState dictionary_state; + auto dictionary_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, + unused_invalid_dictionary, + dictionary_context, 2, dictionary_state) + .ok()); + EXPECT_EQ(dictionary_column->size(), 2); + + TestParquetDecodeSource referenced_invalid_dictionary; + referenced_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day}, {0}); + ParquetMaterializationState invalid_dictionary_state; + auto invalid_dictionary_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet( + *invalid_dictionary_column, referenced_invalid_dictionary, + dictionary_context, 1, invalid_dictionary_state) + .ok()); + EXPECT_EQ(invalid_dictionary_column->size(), 0); + } +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1969); + EXPECT_EQ(data[0].microsecond(), 999999); + EXPECT_EQ(data[1].second(), 1); + EXPECT_EQ(data[1].microsecond(), 1); +} + +TEST(DataTypeSerDeParquetTest, ValidatesInt96BeforeDateTimeMaterialization) { + { + TestParquetDecodeSource source; + source.set_fixed_values( + {{0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY - 1, JULIAN_UNIX_EPOCH}}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 23:59:59.999999"); + } + + const std::vector invalid_values { + {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}, + {-1, JULIAN_UNIX_EPOCH}, + {0, std::numeric_limits::max()}}; + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, Int96DictionaryFailuresFollowDecodedIds) { + const std::vector dictionary_values { + {0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + std::vector dictionary(sizeof(TestParquetInt96Timestamp) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(TestParquetInt96Timestamp), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 00:00:00.000000"); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { + constexpr int64_t MIN_TIMESTAMP_MICROS = -62135596800000000LL; + constexpr int64_t MAX_TIMESTAMP_MICROS = 253402300799999999LL; + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + + { + TestParquetDecodeSource source; + source.set_fixed_values({MIN_TIMESTAMP_MICROS, MAX_TIMESTAMP_MICROS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1); + EXPECT_EQ(data[1].year(), 9999); + EXPECT_EQ(data[1].microsecond(), 999999); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max(), YEAR_10000_MILLIS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDecodedValuesUseTheSameBounds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector values {std::numeric_limits::max(), YEAR_10000_MILLIS, 0}; + NullMap conversion_failures(3, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT64, + .time_unit = DecodedTimeUnit::MILLIS, + .row_count = 3, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(conversion_failures, NullMap({1, 1, 0})); + EXPECT_EQ(assert_cast(*column).get_data()[2].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDictionaryFailuresFollowDecodedIds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector dictionary_values {0, YEAR_10000_MILLIS}; + std::vector dictionary(sizeof(int64_t) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(int64_t), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(assert_cast(*column).get_data()[1].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, Int96DecodedValuesRejectInvalidNanos) { + const std::vector values {{NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + NullMap conversion_failures(1, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT96, + .row_count = 1, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(conversion_failures, NullMap({1})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F}, + 16); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::UUID, + .type_length = 16, + .logical_uuid = true}; + ParquetMaterializationState state; + DataTypeVarbinary type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->get_data_at(0).to_string(), std::string("\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", + 16)); +} + +TEST(DataTypeSerDeParquetTest, NullableConversionFailuresBecomeNullOutsideStrictMode) { + auto expect_null = [](DataTypeSerDeSPtr serde, MutableColumnPtr column, + TestParquetDecodeSource* source, const ParquetDecodeContext& context) { + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + EXPECT_TRUE(serde->read_column_from_parquet(*column, *source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + }; + + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::min()}); + DataTypeDateV2 type; + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeDateTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({10}); + DataTypeDecimal32 type(1, 0); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 2, + .decimal_scale = 0}); + } +} + +TEST(DataTypeSerDeParquetTest, DictionaryConversionFailuresFollowDecodedIds) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map[0], 1); + EXPECT_EQ(null_map[1], 1); + EXPECT_EQ(source.dictionary_decode_calls(), 1); +} + +TEST(DataTypeSerDeParquetTest, NonNullableDictionaryConversionFailureRemainsAnError) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ParquetMaterializationState state; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_FALSE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 0); +} + +} // namespace +} // namespace doris diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index d4bd6ab6c06360..02cf3013139cd3 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -59,11 +59,13 @@ TColumnAccessPath meta_access_path() { format::ColumnDefinition field(int32_t id, std::string name, DataTypePtr type, std::vector children = {}, - std::vector aliases = {}) { + std::vector aliases = {}, + bool has_name_mapping = false) { return { .identifier = Field::create_field(id), .name = std::move(name), .name_mapping = std::move(aliases), + .has_name_mapping = has_name_mapping, .type = std::move(type), .children = std::move(children), }; @@ -167,7 +169,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { .type = struct_type, .children = { - field(101, "a", int_type), + field(101, "a", int_type, {}, {}, true), field(205, "b", int_type, {}, {"old_b"}), }, }; @@ -199,6 +201,15 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { expect_child(column.children[0], 205, "b"); EXPECT_EQ(column.children[0].name_mapping, std::vector({"old_b"})); } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "a"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); + } { auto column = root_column(100, "s", struct_type); auto status = AccessPathParser::build_nested_children( @@ -206,6 +217,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); expect_child(column.children[1], 205, "b"); } @@ -218,6 +230,52 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { } } +TEST(AccessPathParserTest, CurrentStructNameTakesPriorityOverHistoricalAlias) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type, int_type}, + Strings {"renamed_b", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(1, "renamed_b", int_type, {}, {"b"}, true), + field(2, "b", int_type, {}, {}, true), + }, + }; + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "b"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 2, "b"); +} + +TEST(AccessPathParserTest, LegacyPlanRetainsOrderedStructNameAndAliasLookup) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type, int_type}, + Strings {"renamed_b", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(1, "renamed_b", int_type, {}, {"b"}, true), + field(2, "b", int_type, {}, {}, true), + }, + }; + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "b"})}, &schema, false); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 1, "renamed_b"); +} + // Scenario: array access paths must pass through the "*" element token, then reuse struct child // parsing under the element wrapper; invalid array tokens are rejected. TEST(AccessPathParserTest, ArrayAccessPathMatrix) { @@ -235,7 +293,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { field(201, "element", element_type, { field(202, "item", string_type, {}, {"old_item"}), - field(203, "quantity", int_type), + field(203, "quantity", int_type, {}, {}, true), }), }, }; @@ -254,6 +312,18 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { EXPECT_EQ(column.children[0].children[0].name_mapping, std::vector({"old_item"})); } + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector {data_access_path({"items", "*", "quantity"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + ASSERT_EQ(column.children[0].children.size(), 1); + expect_child(column.children[0].children[0], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[0].has_name_mapping); + } { auto column = root_column(200, "items", array_type); auto status = AccessPathParser::build_nested_children( @@ -264,6 +334,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { ASSERT_EQ(column.children[0].children.size(), 2); expect_child(column.children[0].children[0], 202, "item"); expect_child(column.children[0].children[1], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[1].has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -293,7 +364,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { field(302, "value", value_type, { field(303, "full_name", string_type, {}, {"name"}), - field(304, "age", int_type), + field(304, "age", int_type, {}, {}, true), field(305, "gender", string_type), }), }, @@ -329,6 +400,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { expect_child(column.children[1], 302, "value"); ASSERT_EQ(column.children[1].children.size(), 1); expect_child(column.children[1].children[0], 304, "age"); + EXPECT_TRUE(column.children[1].children[0].has_name_mapping); } { auto column = root_column(300, "m", map_type); @@ -357,6 +429,9 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); ASSERT_EQ(column.children[1].children.size(), 3); + const auto* age = find_child_by_name(column.children[1], "age"); + ASSERT_NE(age, nullptr); + EXPECT_TRUE(age->has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -368,4 +443,21 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { } } +TEST(AccessPathParserTest, PreservesNestedInitialDefaultMetadata) { + auto binary_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {binary_type}, Strings {"data"}); + auto defaulted_child = field(101, "data", binary_type); + defaulted_child.initial_default_value = "AAEC/w=="; + defaulted_child.initial_default_value_is_base64 = true; + auto schema = field(100, "s", struct_type, {defaulted_child}); + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "data"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + EXPECT_EQ(column.children[0].initial_default_value, std::optional("AAEC/w==")); + EXPECT_TRUE(column.children[0].initial_default_value_is_base64); +} + } // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 586d307fe32edb..3f3d13c98e9bac 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -38,6 +38,7 @@ #include "exec/scan/file_scanner.h" #include "exec/scan/split_source_connector.h" #include "exprs/create_predicate_function.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vbloom_predicate.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vliteral.h" @@ -48,6 +49,9 @@ namespace doris { namespace { +constexpr int kIcebergPositionDeleteContent = 1; +constexpr int kIcebergDeletionVectorContent = 3; + TFileRangeDesc range_with_format(std::string table_format, TFileFormatType::type format_type) { TFileRangeDesc range; range.__set_format_type(format_type); @@ -59,6 +63,14 @@ TFileRangeDesc range_with_format(std::string table_format, TFileFormatType::type return range; } +TFileRangeDesc iceberg_position_deletes_range(TFileFormatType::type format_type, int content) { + auto range = range_with_format("iceberg", format_type); + TIcebergFileDesc iceberg_params; + iceberg_params.__set_content(content); + range.table_format_params.__set_iceberg_params(std::move(iceberg_params)); + return range; +} + TFileRangeDesc hudi_range_with_delta_logs() { auto range = range_with_format("hudi", TFileFormatType::FORMAT_PARQUET); THudiFileDesc hudi_params; @@ -84,6 +96,29 @@ TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { return range; } +TEST(FileScannerTest, V1CountPushdownRequiresExplicitCountStarArguments) { + EXPECT_EQ(TPushAggOp::type::COUNT, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::vector {})); + + // A missing field is an old FE plan with unknown COUNT semantics, not COUNT(*). + EXPECT_EQ(TPushAggOp::type::NONE, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::nullopt)); + // V1 cannot evaluate COUNT(col) NULL/CAST semantics before replacing the reader with + // CountReader, so an explicit argument must use the normal scan path. + EXPECT_EQ(TPushAggOp::type::NONE, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::vector {7})); + + // The COUNT argument field must not affect other storage-layer aggregate operations. + EXPECT_EQ(TPushAggOp::type::MINMAX, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::MINMAX, std::nullopt)); +} + +TEST(FileScannerV2Test, AdaptiveBatchSizeRunsForCountFallbackOnly) { + EXPECT_TRUE(FileScannerV2::TEST_should_run_adaptive_batch_size(true, false)); + EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(true, true)); + EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(false, false)); +} + struct RetryableCloseState { int close_calls = 0; }; @@ -315,6 +350,27 @@ TEST(FileScannerV2Test, ConditionCacheDigestIncludesRuntimeFilterPayload) { 0); } +// Scenario: Iceberg position-delete system table splits use FileScannerV2 for both native delete +// formats and V3 deletion vectors. Avro remains unsupported and is rejected by FE before routing. +TEST(FileScannerV2Test, IcebergPositionDeletesSupportNativeFormats) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + const auto parquet_position_delete = iceberg_position_deletes_range( + TFileFormatType::FORMAT_PARQUET, kIcebergPositionDeleteContent); + const auto parquet_deletion_vector = iceberg_position_deletes_range( + TFileFormatType::FORMAT_PARQUET, kIcebergDeletionVectorContent); + const auto orc_position_delete = iceberg_position_deletes_range(TFileFormatType::FORMAT_ORC, + kIcebergPositionDeleteContent); + const auto avro_position_delete = iceberg_position_deletes_range(TFileFormatType::FORMAT_AVRO, + kIcebergPositionDeleteContent); + + EXPECT_TRUE(FileScannerV2::is_supported(params, parquet_position_delete)); + EXPECT_TRUE(FileScannerV2::is_supported(params, parquet_deletion_vector)); + EXPECT_TRUE(FileScannerV2::is_supported(params, orc_position_delete)); + EXPECT_FALSE(FileScannerV2::is_supported(params, avro_position_delete)); +} + TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesOnly) { TQueryOptions query_options; TFileScanRangeParams params; @@ -574,6 +630,14 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { EXPECT_EQ(profile.get_counter("BytesScannedFromRemote")->value(), 13); EXPECT_EQ(profile.get_counter("BytesScannedFromPeer")->value(), 17); EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + ASSERT_TRUE(children.contains("IO")); + EXPECT_TRUE(children.at("IO").contains("FileCache")); } TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { @@ -585,6 +649,16 @@ TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { EXPECT_FALSE(FileScannerV2::TEST_should_skip_not_found(Status::OK(), true)); } +TEST(FileScannerV2Test, EndOfFileIsSkippedAsEmptySplit) { + EXPECT_TRUE(FileScannerV2::TEST_should_skip_empty(Status::EndOfFile("empty file"), false)); + // Deletion-vector and Parquet readers also use EOF to unwind an interrupted read. Once either + // scanner stop flag is visible, the same status is no longer evidence of an empty file. + EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::EndOfFile("stop read."), true)); + EXPECT_FALSE( + FileScannerV2::TEST_should_skip_empty(Status::InternalError("read failed"), false)); + EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::OK(), false)); +} + // Scenario: partition slots are identified from the explicit FE category when present, otherwise // from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as // partition columns even if FE marks them as non-file slots. @@ -674,6 +748,23 @@ TEST(FileScannerV2Test, RewriteSlotRefsToGlobalIndexMatrix) { } } +TEST(FileScannerV2Test, AdaptsBranchRuntimeFilterForMasterTableReader) { + const auto int_type = std::make_shared(); + const auto node = bool_in_pred_node(); + auto impl = VDirectInPredicate::create_shared(node, nullptr); + impl->add_child(slot_ref(11, 2, int_type, "rf_value")); + VExprSPtr expr = VRuntimeFilterWrapper::create_shared(node, std::move(impl), 0.4, false, 7); + + ASSERT_TRUE(FileScannerV2::TEST_adapt_runtime_filter_for_table_reader(&expr).ok()); + const auto* runtime_filter = dynamic_cast(expr.get()); + ASSERT_NE(runtime_filter, nullptr); + ASSERT_NE(runtime_filter->get_impl(), nullptr); + ASSERT_EQ(runtime_filter->get_impl()->get_num_children(), 1); + const auto* child = + assert_cast(runtime_filter->get_impl()->children()[0].get()); + EXPECT_EQ(child->column_id(), 2); +} + TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { const auto bool_type = std::make_shared(); auto unsafe_predicate = std::make_shared(); @@ -694,4 +785,27 @@ TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { EXPECT_EQ(partition_conjuncts[0], conjuncts[0]); } +TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { + const auto bool_type = std::make_shared(); + auto unsafe_predicate = std::make_shared(); + unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); + VExprContextSPtrs conjuncts { + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 1), + runtime_filter_context(std::move(unsafe_predicate), 2), + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 3), + }; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("file_scanner_v2"); + FileScannerV2 scanner(&state, &profile, nullptr); + scanner.TEST_set_scanner_conjuncts(std::move(conjuncts)); + + EXPECT_EQ(scanner.TEST_table_reader_owned_conjunct_count(), 1); + EXPECT_EQ(scanner.TEST_scanner_residual_conjunct_count(), 2); + const auto* residual_predicates = profile.get_info_string("ScannerResidualPredicates"); + ASSERT_NE(residual_predicates, nullptr); + EXPECT_FALSE(residual_predicates->empty()); + EXPECT_NE(residual_predicates->find("SlotRef"), std::string::npos) << *residual_predicates; +} + } // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 0d31b69495187e..c926f3dc58d595 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -29,6 +29,9 @@ #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" #include "exec/scan/scanner.h" +// branch-4.1 creates VRuntimeFilterWrapper before the RuntimeFilterExpr rename, so assert the +// branch-native wrapper type while preserving the late-arrival batching invariant. +#include "exprs/vruntimefilter_wrapper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "testutil/column_helper.h" @@ -65,6 +68,23 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class HighCostPredicate final : public VExpr { +public: + HighCostPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + double execute_cost() const override { return 100.0; } + +private: + const std::string _expr_name = "high_cost_stateful_predicate"; +}; + class ScannerLateArrivalRfTest : public RuntimeFilterTest { public: void SetUp() override { @@ -83,6 +103,7 @@ class ScannerLateArrivalRfTest : public RuntimeFilterTest { // the counter advances after RFs arrive and that the second call short-circuits // via the fast path at the top of the function. TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { + _runtime_states[0]->_query_options.__set_enable_adjust_conjunct_order_by_cost(true); std::vector rf_descs = { TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build(), TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build()}; @@ -104,26 +125,55 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { auto local_state = std::make_shared(_runtime_states[0].get(), op.get()); + auto initial_conjunct = VExprContext::create_shared(std::make_shared()); + ASSERT_TRUE(initial_conjunct->prepare(_runtime_states[0].get(), row_desc).ok()); + ASSERT_TRUE(initial_conjunct->open(_runtime_states[0].get()).ok()); + local_state->_conjuncts.push_back(initial_conjunct); + std::vector> rf_dependencies; ASSERT_TRUE(local_state->_helper.init(_runtime_states[0].get(), true, 0, 0, rf_dependencies, "") .ok()); + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + local_state->_helper._consumers[0]->signal(producer.get()); + ASSERT_TRUE(local_state->_helper + .acquire_runtime_filter(_runtime_states[0].get(), local_state->_conjuncts, + row_desc) + .ok()); + ASSERT_EQ(local_state->_conjuncts.size(), 2); + auto scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), -1 /*limit*/, &_profile); - ASSERT_TRUE(scanner->init(_runtime_states[0].get(), {}).ok()); + ASSERT_TRUE(scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); + auto second_scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), + -1 /*limit*/, &_profile); + ASSERT_TRUE(second_scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); ASSERT_EQ(scanner->_total_rf_num, 2); ASSERT_EQ(scanner->_applied_rf_num, 0); - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - local_state->_helper._consumers[0]->signal(producer.get()); local_state->_helper._consumers[1]->signal(producer.get()); // First call after both RFs arrived: counter must advance to total. Before // the fix this stayed at 0 because the assignment was missing. ASSERT_TRUE(scanner->try_append_late_arrival_runtime_filter().ok()); ASSERT_EQ(scanner->_applied_rf_num, 2); + ASSERT_EQ(scanner->_late_arrival_rf_conjuncts.size(), 1); + for (const auto& conjunct : scanner->_late_arrival_rf_conjuncts) { + EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); + } + ASSERT_EQ(scanner->_conjuncts.size(), 3); + EXPECT_EQ(scanner->_conjuncts.back()->expr_name(), "high_cost_stateful_predicate"); + + // The first scanner consumes the shared helper's expression, so another scanner can only get + // the exact delta from the local state's append-only RF batch history. + ASSERT_TRUE(second_scanner->try_append_late_arrival_runtime_filter().ok()); + ASSERT_EQ(second_scanner->_applied_rf_num, 2); + ASSERT_EQ(second_scanner->_late_arrival_rf_conjuncts.size(), 1); + EXPECT_NE(dynamic_cast( + second_scanner->_late_arrival_rf_conjuncts[0]->root().get()), + nullptr); // Second call: must hit the fast-path early return without re-cloning. // We clear `_conjuncts` and verify the function does NOT repopulate them; diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp index d9fc5086503d90..7faa77ed702c68 100644 --- a/be/test/exec/sink/viceberg_delete_sink_test.cpp +++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp @@ -33,6 +33,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" +#include "format/table/deletion_vector.h" #include "gen_cpp/DataSinks_types.h" #include "gen_cpp/Types_types.h" #include "runtime/runtime_state.h" @@ -507,6 +508,19 @@ TEST_F(VIcebergDeleteSinkTest, TestUnsupportedDeleteType) { ASSERT_FALSE(status.ok()); } +TEST_F(VIcebergDeleteSinkTest, TestValidateDeletionVectorContentSize) { + constexpr size_t max_bitmap_size = static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES) - + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES; + int64_t content_size = 0; + ASSERT_TRUE( + calculate_iceberg_deletion_vector_content_size(max_bitmap_size, &content_size).ok()); + ASSERT_EQ(MAX_ICEBERG_DELETION_VECTOR_BYTES, content_size); + + const auto unsupported_status = + calculate_iceberg_deletion_vector_content_size(max_bitmap_size + 1, &content_size); + ASSERT_TRUE(unsupported_status.is()) << unsupported_status; +} + TEST_F(VIcebergDeleteSinkTest, TestWriteDeletionVectorsToSingleSharedPuffin) { std::filesystem::path temp_dir = std::filesystem::temp_directory_path() / ("iceberg_delete_sink_test_" + generate_uuid_string()); diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp new file mode 100644 index 00000000000000..d453177cf25044 --- /dev/null +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "exec/sink/writer/iceberg/viceberg_partition_writer.h" + +namespace doris { + +namespace { + +class FakeFileFormatTransformer final : public VFileFormatTransformer { +public: + explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs) + : VFileFormatTransformer(nullptr, output_exprs, false) {} + + Status open() override { return Status::OK(); } + Status write(const Block&) override { return Status::OK(); } + Status close() override { return Status::OK(); } + int64_t written_len() override { return 64; } +}; + +TDataSink make_table_sink(std::optional collect_column_stats) { + TIcebergTableSink iceberg_sink; + if (collect_column_stats.has_value()) { + iceberg_sink.__set_collect_column_stats(*collect_column_stats); + } + TDataSink sink; + sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); + sink.__set_iceberg_table_sink(iceberg_sink); + return sink; +} + +} // namespace + +class VIcebergPartitionWriterTest : public testing::Test { +protected: + static std::unique_ptr make_writer( + const TDataSink& sink, const VExprContextSPtrs& output_exprs, + const iceberg::Schema& schema, const std::string* schema_json, + const std::map& hadoop_conf) { + IPartitionWriterBase::WriteInfo write_info; + write_info.file_type = TFileType::FILE_LOCAL; + return std::make_unique( + sink, std::vector {}, output_exprs, schema, schema_json, + std::vector {}, std::move(write_info), "data", 0, + TFileFormatType::FORMAT_ORC, TFileCompressType::ZLIB, hadoop_conf); + } + + static void install_fake_transformer(VIcebergPartitionWriter* writer, + const VExprContextSPtrs& output_exprs) { + writer->_file_format_transformer = + std::make_unique(output_exprs); + } + + static Status build_commit_data(VIcebergPartitionWriter* writer, + TIcebergCommitData* commit_data) { + return writer->_build_iceberg_commit_data(commit_data); + } + + static bool collect_column_stats(const VIcebergPartitionWriter& writer) { + return writer._collect_column_stats; + } +}; + +TEST_F(VIcebergPartitionWriterTest, OrcSkipsFooterCollectionWhenMetricsAreDisabled) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf); + install_fake_transformer(writer.get(), output_exprs); + + TIcebergCommitData commit_data; + ASSERT_TRUE(build_commit_data(writer.get(), &commit_data).ok()); + EXPECT_FALSE(commit_data.__isset.column_stats); +} + +TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollingUpgrade) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = make_writer(make_table_sink(std::nullopt), output_exprs, schema, &schema_json, + hadoop_conf); + + EXPECT_TRUE(collect_column_stats(*writer)); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp index 69758b794dbd5a..94b8aec8c77ea6 100644 --- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp +++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp @@ -19,6 +19,8 @@ #include +#include + #include "core/data_type/data_type_date_or_datetime_v2.h" namespace doris { @@ -115,6 +117,28 @@ TEST_F(PartitionTransformersTest, test_string_truncate_transform) { } } +TEST_F(PartitionTransformersTest, test_floating_point_special_partition_value) { + auto float_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_FLOAT, false); + auto double_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DOUBLE, false); + IdentityPartitionColumnTransform float_transform(float_type); + IdentityPartitionColumnTransform double_transform(double_type); + + EXPECT_EQ("NaN", float_transform.get_partition_value( + float_type, std::numeric_limits::quiet_NaN())); + EXPECT_EQ("Infinity", float_transform.get_partition_value( + float_type, std::numeric_limits::infinity())); + EXPECT_EQ("-Infinity", float_transform.get_partition_value( + float_type, -std::numeric_limits::infinity())); + EXPECT_EQ("NaN", double_transform.get_partition_value( + double_type, std::numeric_limits::quiet_NaN())); + EXPECT_EQ("Infinity", double_transform.get_partition_value( + double_type, std::numeric_limits::infinity())); + EXPECT_EQ("-Infinity", double_transform.get_partition_value( + double_type, -std::numeric_limits::infinity())); +} + TEST_F(PartitionTransformersTest, test_integer_bucket_transform) { const std::vector values({34, -123}); // 2017239379, -471378254 auto column = ColumnInt32::create(); diff --git a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp index e149d69328810d..0b0ac1f6404e24 100644 --- a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp +++ b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp @@ -17,15 +17,32 @@ #include "format/table/iceberg_delete_file_reader_helper.h" +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include +#include "exec/common/endian.h" +#include "format/table/deletion_vector_reader.h" #include "io/fs/file_meta_cache.h" +#include "roaring/roaring64map.hh" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/hash_util.hpp" namespace doris { @@ -37,6 +54,13 @@ constexpr const char* kMixedPositionDeleteFile = constexpr const char* kTargetDataFilePath = "s3://warehouse/wh/test_db/000_target_data_file.parquet"; +TUniqueId make_query_id() { + TUniqueId query_id; + query_id.hi = 300; + query_id.lo = 400; + return query_id; +} + class CollectPositionDeleteVisitor final : public IcebergPositionDeleteVisitor { public: Status visit(const std::string& file_path, int64_t pos) override { @@ -49,6 +73,119 @@ class CollectPositionDeleteVisitor final : public IcebergPositionDeleteVisitor { size_t total_rows = 0; }; +TFileScanRangeParams make_local_parquet_scan_params() { + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + return scan_params; +} + +TIcebergDeleteFileDesc make_iceberg_deletion_vector(const std::string& path, int64_t offset, + int64_t size) { + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(3); + delete_file.__set_path(path); + delete_file.__set_content_offset(offset); + delete_file.__set_content_size_in_bytes(size); + return delete_file; +} + +std::vector build_iceberg_deletion_vector_blob( + const std::vector& deleted_positions) { + roaring::Roaring64Map rows; + for (const auto position : deleted_positions) { + rows.add(position); + } + + const size_t bitmap_size = rows.getSizeInBytes(); + std::vector blob(4 + 4 + bitmap_size + 4); + rows.write(blob.data() + 8); + + const uint32_t total_length = static_cast(4 + bitmap_size); + BigEndian::Store32(blob.data(), total_length); + constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; + memcpy(blob.data() + 4, DV_MAGIC, 4); + const uint32_t crc = HashUtil::zlib_crc_hash(blob.data() + 4, total_length, 0); + BigEndian::Store32(blob.data() + 8 + bitmap_size, crc); + return blob; +} + +int64_t write_iceberg_deletion_vector_file(const std::string& file_path, + const std::vector& deleted_positions) { + const auto blob = build_iceberg_deletion_vector_blob(deleted_positions); + + std::ofstream output(file_path, std::ios::binary); + EXPECT_TRUE(output.is_open()); + output.write(blob.data(), static_cast(blob.size())); + EXPECT_TRUE(output.good()); + return static_cast(blob.size()); +} + +std::string temp_parquet_path(const std::string& filename) { + return (std::filesystem::temp_directory_path() / filename).string(); +} + +void write_position_delete_parquet(const std::string& path, + const std::vector>& file_paths, + const std::vector>& positions) { + ASSERT_EQ(file_paths.size(), positions.size()); + + arrow::StringBuilder path_builder; + arrow::Int64Builder pos_builder; + for (size_t i = 0; i < file_paths.size(); ++i) { + if (file_paths[i].has_value()) { + ASSERT_TRUE(path_builder.Append(*file_paths[i]).ok()); + } else { + ASSERT_TRUE(path_builder.AppendNull().ok()); + } + if (positions[i].has_value()) { + ASSERT_TRUE(pos_builder.Append(*positions[i]).ok()); + } else { + ASSERT_TRUE(pos_builder.AppendNull().ok()); + } + } + + std::shared_ptr path_array; + std::shared_ptr pos_array; + ASSERT_TRUE(path_builder.Finish(&path_array).ok()); + ASSERT_TRUE(pos_builder.Finish(&pos_array).ok()); + + auto schema = arrow::schema({arrow::field("file_path", arrow::utf8(), true), + arrow::field("pos", arrow::int64(), true)}); + auto table = arrow::Table::Make(schema, {path_array, pos_array}); + + std::shared_ptr output; + ASSERT_TRUE(arrow::io::FileOutputStream::Open(path).Value(&output).ok()); + PARQUET_THROW_NOT_OK( + ::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1024)); +} + +IcebergDeleteFileReaderOptions make_delete_file_reader_options( + RuntimeState* state, RuntimeProfile* profile, const TFileScanRangeParams* scan_params, + io::IOContext* io_ctx) { + return { + .state = state, + .profile = profile, + .scan_params = scan_params, + .io_ctx = io_ctx, + }; +} + +IcebergDeleteFileReaderOptions delete_reader_options(RuntimeState* runtime_state, + RuntimeProfile* profile, + TFileScanRangeParams* scan_params, + IcebergDeleteFileIOContext* io_context, + FileMetaCache* meta_cache) { + IcebergDeleteFileReaderOptions options; + options.state = runtime_state; + options.profile = profile; + options.scan_params = scan_params; + options.io_ctx = &io_context->io_ctx; + options.meta_cache = meta_cache; + options.batch_size = 1024; + return options; +} + } // namespace TEST(IcebergDeleteFileReaderHelperTest, BuildDeleteFileRange) { @@ -71,9 +208,359 @@ TEST(IcebergDeleteFileReaderHelperTest, IsNotDeletionVectorWhenContentMissing) { EXPECT_FALSE(is_iceberg_deletion_vector(delete_file)); } +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorCacheKeyIncludesLocationAndRange) { + // Scenario: one Puffin file can hold several DV blobs. The cache key must isolate both the + // data file and the blob range so a scanner never reuses another file's DV. + TIcebergDeleteFileDesc first_delete_file; + first_delete_file.__set_path("s3://bucket/shared/delete.puffin"); + first_delete_file.__set_content_offset(128); + first_delete_file.__set_content_size_in_bytes(64); + + TIcebergDeleteFileDesc different_offset = first_delete_file; + different_offset.__set_content_offset(256); + + TIcebergDeleteFileDesc different_length = first_delete_file; + different_length.__set_content_size_in_bytes(96); + + TIcebergDeleteFileDesc different_delete_file = first_delete_file; + different_delete_file.__set_path("s3://bucket/shared/other-delete.puffin"); + + const std::string data_file_path = "s3://bucket/table/data-00001.parquet"; + const auto first_key = + build_iceberg_deletion_vector_cache_key(data_file_path, first_delete_file); + + EXPECT_NE(first_key, build_iceberg_deletion_vector_cache_key(data_file_path, different_offset)); + EXPECT_NE(first_key, build_iceberg_deletion_vector_cache_key(data_file_path, different_length)); + EXPECT_NE(first_key, + build_iceberg_deletion_vector_cache_key(data_file_path, different_delete_file)); + EXPECT_NE(first_key, + build_iceberg_deletion_vector_cache_key( + "s3://bucket/table/snapshot-branch/data-00001.parquet", first_delete_file)); +} + +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorCacheKeyEscapesPathBoundaries) { + TIcebergDeleteFileDesc first_delete_file; + first_delete_file.__set_path("middle#right#tail.puffin"); + first_delete_file.__set_content_offset(1); + first_delete_file.__set_content_size_in_bytes(2); + + TIcebergDeleteFileDesc second_delete_file = first_delete_file; + second_delete_file.__set_path("right#tail.puffin"); + + const std::string first_data_file_path = "s3://bucket/table/data#left"; + const std::string second_data_file_path = "s3://bucket/table/data#left#middle"; + ASSERT_EQ(first_data_file_path + "#" + first_delete_file.path, + second_data_file_path + "#" + second_delete_file.path); + + EXPECT_NE(build_iceberg_deletion_vector_cache_key(first_data_file_path, first_delete_file), + build_iceberg_deletion_vector_cache_key(second_data_file_path, second_delete_file)); +} + +TEST(IcebergDeleteFileReaderHelperTest, ValidateDeletionVectorDescriptor) { + size_t bytes_read = 0; + + TIcebergDeleteFileDesc missing_path; + missing_path.__set_content_offset(0); + missing_path.__set_content_size_in_bytes(12); + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor(missing_path, bytes_read).ok()); + + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", -1, 12), bytes_read) + .ok()); + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, 11), bytes_read) + .ok()); + EXPECT_TRUE( + validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, MAX_ICEBERG_DELETION_VECTOR_BYTES), + bytes_read) + .ok()); + EXPECT_EQ(static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES), bytes_read); + const auto unsupported_status = validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, MAX_ICEBERG_DELETION_VECTOR_BYTES + 1), + bytes_read); + EXPECT_TRUE(unsupported_status.is()) << unsupported_status; + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", + std::numeric_limits::max() - 10, 12), + bytes_read) + .ok()); + + EXPECT_TRUE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 7, 12), bytes_read) + .ok()); + EXPECT_EQ(bytes_read, 12); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReportsMissingFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_missing_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto missing_path = (test_dir / "missing-delete-vector.bin").string(); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + auto status = read_iceberg_deletion_vector(make_iceberg_deletion_vector(missing_path, 0, 16), + options, &rows_to_delete); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(missing_path), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorRejectsRangePastFile) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_deletion_vector_range_past_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {1, 3}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + auto status = read_iceberg_deletion_vector( + make_iceberg_deletion_vector(dv_path, 0, dv_size + 1), options, &rows_to_delete); + + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("range exceeds file size"), std::string::npos); + EXPECT_NE(status.to_string().find(dv_path), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorReaderValidatesOpenedFileRange) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_deletion_vector_opened_file_range_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {1, 3}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + + { + TFileRangeDesc exact_range = build_iceberg_delete_file_range(dv_path); + exact_range.start_offset = 4; + exact_range.size = dv_size - exact_range.start_offset; + DeletionVectorReader exact_reader(&state, &profile, scan_params, exact_range, + &io_context.io_ctx); + const auto exact_status = exact_reader.open(); + EXPECT_TRUE(exact_status.ok()) << exact_status; + + TFileRangeDesc oversized_range = exact_range; + oversized_range.size = MAX_ICEBERG_DELETION_VECTOR_BYTES; + DeletionVectorReader oversized_reader(&state, &profile, scan_params, oversized_range, + &io_context.io_ctx); + const auto oversized_status = oversized_reader.open(); + EXPECT_TRUE(oversized_status.is()) << oversized_status; + EXPECT_NE(oversized_status.to_string().find("range exceeds file size"), std::string::npos); + EXPECT_NE(oversized_status.to_string().find(dv_path), std::string::npos); + } + + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorStopsWhenIoContextStops) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_stop_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {0}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + io_context.io_ctx.should_stop = true; + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + auto status = read_iceberg_deletion_vector(make_iceberg_deletion_vector(dv_path, 0, dv_size), + options, &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("stop read"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, DecodeDeletionVectorRejectsCorruptPayload) { + std::vector corrupted(12, 0); + BigEndian::Store32(corrupted.data(), 4); + memcpy(corrupted.data() + 4, "BAD!", 4); + + roaring::Roaring64Map rows_to_delete; + auto status = decode_iceberg_deletion_vector_buffer(corrupted.data(), corrupted.size(), + &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("magic number mismatch"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); +} + +TEST(IcebergDeleteFileReaderHelperTest, DecodeDeletionVectorRejectsCrcMismatch) { + auto corrupted = build_iceberg_deletion_vector_blob({1, 3}); + corrupted.back() ^= 1; + + roaring::Roaring64Map rows_to_delete; + auto status = decode_iceberg_deletion_vector_buffer(corrupted.data(), corrupted.size(), + &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("CRC32 mismatch"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReadsMillionDeletePositions) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_large_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + constexpr uint64_t delete_position_count = 1'000'000; + std::vector deleted_positions; + deleted_positions.reserve(delete_position_count); + for (uint64_t position = 0; position < delete_position_count; ++position) { + deleted_positions.push_back(position * 2); + } + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, deleted_positions); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + + ASSERT_TRUE(read_iceberg_deletion_vector(make_iceberg_deletion_vector(dv_path, 0, dv_size), + options, &rows_to_delete) + .ok()); + + EXPECT_EQ(rows_to_delete.cardinality(), delete_position_count); + EXPECT_TRUE(rows_to_delete.contains(static_cast(0))); + EXPECT_TRUE(rows_to_delete.contains(static_cast(999'998))); + EXPECT_TRUE(rows_to_delete.contains(static_cast(1'999'998))); + EXPECT_FALSE(rows_to_delete.contains(static_cast(1'999'999))); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorConcurrentReadsAreStable) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_deletion_vector_concurrent_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_dv_path = (test_dir / "delete-vector-first.bin").string(); + const auto second_dv_path = (test_dir / "delete-vector-second.bin").string(); + const auto first_dv_size = write_iceberg_deletion_vector_file(first_dv_path, {0, 2, 4}); + const auto second_dv_size = write_iceberg_deletion_vector_file(second_dv_path, {1, 3, 5}); + + auto read_and_verify = [](const std::string& path, int64_t size, + const std::vector& expected_positions) -> Status { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + roaring::Roaring64Map rows_to_delete; + auto options = + make_delete_file_reader_options(&state, &profile, &scan_params, &io_context.io_ctx); + RETURN_IF_ERROR(read_iceberg_deletion_vector(make_iceberg_deletion_vector(path, 0, size), + options, &rows_to_delete)); + if (rows_to_delete.cardinality() != expected_positions.size()) { + return Status::InternalError("unexpected deletion vector cardinality"); + } + for (const auto position : expected_positions) { + if (!rows_to_delete.contains(position)) { + return Status::InternalError("missing deletion vector position {}", position); + } + } + return Status::OK(); + }; + + std::vector workers; + std::vector statuses(16); + for (size_t idx = 0; idx < statuses.size(); ++idx) { + workers.emplace_back([&, idx]() { + SCOPED_INIT_THREAD_CONTEXT(); + if (idx % 2 == 0) { + statuses[idx] = read_and_verify(first_dv_path, first_dv_size, {0, 2, 4}); + } else { + statuses[idx] = read_and_verify(second_dv_path, second_dv_size, {1, 3, 5}); + } + }); + } + for (auto& worker : workers) { + worker.join(); + } + for (const auto& status : statuses) { + EXPECT_TRUE(status.ok()) << status; + } + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergDeleteFileReaderHelperTest, IoContextPropagatesQueryIdentityForSelect) { + TQueryOptions query_options; + query_options.__set_query_type(TQueryType::SELECT); + + const auto query_id = make_query_id(); + MockRuntimeState state; + state._query_id = query_id; + state.set_query_options(query_options); + + IcebergDeleteFileIOContext io_context(&state); + + EXPECT_EQ(io_context.io_ctx.query_id, &state.query_id()); + EXPECT_EQ(io_context.io_ctx.reader_type, ReaderType::READER_QUERY); + EXPECT_EQ(io_context.io_ctx.file_cache_stats, &io_context.file_cache_stats); + EXPECT_EQ(io_context.io_ctx.file_reader_stats, &io_context.file_reader_stats); +} + +TEST(IcebergDeleteFileReaderHelperTest, IoContextDoesNotMarkLoadAsQueryReader) { + TQueryOptions query_options; + query_options.__set_query_type(TQueryType::LOAD); + + MockRuntimeState state; + state._query_id = make_query_id(); + state.set_query_options(query_options); + state._query_ctx = nullptr; + + IcebergDeleteFileIOContext io_context(&state); + + EXPECT_EQ(io_context.io_ctx.query_id, &state.query_id()); + EXPECT_EQ(io_context.io_ctx.reader_type, ReaderType::UNKNOWN); + EXPECT_EQ(io_context.io_ctx.file_cache_stats, &io_context.file_cache_stats); + EXPECT_EQ(io_context.io_ctx.file_reader_stats, &io_context.file_reader_stats); +} + TEST(IcebergDeleteFileReaderHelperTest, ReadMixedEncodingParquetPositionDeleteFile) { RuntimeProfile profile("test_profile"); - RuntimeState runtime_state((TQueryGlobals())); + RuntimeState runtime_state((TQueryOptions()), TQueryGlobals()); FileMetaCache meta_cache(1024); IcebergDeleteFileIOContext io_context(&runtime_state); @@ -107,4 +594,67 @@ TEST(IcebergDeleteFileReaderHelperTest, ReadMixedEncodingParquetPositionDeleteFi EXPECT_EQ(it->second, expected_positions); } +TEST(IcebergDeleteFileReaderHelperTest, ReadOptionalParquetPositionDeleteColumnsWithoutNulls) { + const auto delete_file_path = + temp_parquet_path("iceberg_optional_position_delete_without_nulls.parquet"); + write_position_delete_parquet(delete_file_path, + {std::string(kTargetDataFilePath), + std::string(kTargetDataFilePath), "s3://other/file.parquet"}, + {3, 9, 11}); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state((TQueryOptions()), TQueryGlobals()); + FileMetaCache meta_cache(1024); + IcebergDeleteFileIOContext io_context(&runtime_state); + + TFileScanRangeParams scan_params; + scan_params.file_type = TFileType::FILE_LOCAL; + scan_params.format_type = TFileFormatType::FORMAT_PARQUET; + + TIcebergDeleteFileDesc delete_file; + delete_file.path = delete_file_path; + delete_file.file_format = TFileFormatType::FORMAT_PARQUET; + delete_file.__isset.file_format = true; + + CollectPositionDeleteVisitor visitor; + auto options = + delete_reader_options(&runtime_state, &profile, &scan_params, &io_context, &meta_cache); + auto st = read_iceberg_position_delete_file(delete_file, options, &visitor); + + std::filesystem::remove(delete_file_path); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(visitor.total_rows, 3); + EXPECT_EQ(visitor.delete_rows[kTargetDataFilePath], std::vector({3, 9})); +} + +TEST(IcebergDeleteFileReaderHelperTest, RejectParquetPositionDeleteColumnsWithActualNulls) { + const auto delete_file_path = temp_parquet_path("iceberg_position_delete_with_nulls.parquet"); + write_position_delete_parquet(delete_file_path, + {std::string(kTargetDataFilePath), std::nullopt}, {3, 9}); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state((TQueryOptions()), TQueryGlobals()); + FileMetaCache meta_cache(1024); + IcebergDeleteFileIOContext io_context(&runtime_state); + + TFileScanRangeParams scan_params; + scan_params.file_type = TFileType::FILE_LOCAL; + scan_params.format_type = TFileFormatType::FORMAT_PARQUET; + + TIcebergDeleteFileDesc delete_file; + delete_file.path = delete_file_path; + delete_file.file_format = TFileFormatType::FORMAT_PARQUET; + delete_file.__isset.file_format = true; + + CollectPositionDeleteVisitor visitor; + auto options = + delete_reader_options(&runtime_state, &profile, &scan_params, &io_context, &meta_cache); + auto st = read_iceberg_position_delete_file(delete_file, options, &visitor); + + std::filesystem::remove(delete_file_path); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("file_path contains null values"), std::string::npos) + << st.to_string(); +} + } // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..f3c5ef1c386c8f --- /dev/null +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,720 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" +#include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris { + +namespace { + +class ProfileTrackingReader final : public GenericReader { +public: + int collect_calls = 0; + + Status get_next_block(Block* /*block*/, size_t* /*read_rows*/, bool* /*eof*/) override { + return Status::OK(); + } + +protected: + void _collect_profile_before_close() override { ++collect_calls; } +}; + +class FillColumnsTrackingReader final : public GenericReader { +public: + int set_fill_columns_calls = 0; + + Status get_next_block(Block* /*block*/, size_t* /*read_rows*/, bool* /*eof*/) override { + return Status::OK(); + } + + Status set_fill_columns( + const std::unordered_map>& + /*partition_columns*/, + const std::unordered_map& /*missing_columns*/, + const std::unordered_map& /*partition_value_is_null*/) override { + ++set_fill_columns_calls; + return Status::OK(); + } +}; + +class RejectAllRowsPredicate final : public VExpr { +public: + RejectAllRowsPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 0); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(); + return Status::OK(); + } + +private: + const std::string _name = "RejectAllRowsPredicate"; +}; + +SlotDescriptor* make_slot(ObjectPool* pool, int id, std::string name, DataTypePtr type) { + TSlotDescriptor slot_desc; + slot_desc.__set_id(id); + slot_desc.__set_parent(0); + slot_desc.__set_slotType(type->to_thrift()); + slot_desc.__set_columnPos(id); + slot_desc.__set_byteOffset(0); + slot_desc.__set_nullIndicatorByte(id / 8); + slot_desc.__set_nullIndicatorBit(id % 8); + slot_desc.__set_slotIdx(id); + slot_desc.__set_isMaterialized(true); + slot_desc.__set_colName(std::move(name)); + return pool->add(new SlotDescriptor(slot_desc)); +} + +Block make_output_block(const std::vector& slots) { + Block block; + for (const auto* slot : slots) { + auto type = slot->get_data_type_ptr(); + block.insert(ColumnWithTypeAndName(type->create_column(), type, slot->col_name())); + } + return block; +} + +const IColumn& nested_column(const Block& block, const std::string& name) { + const auto position = block.get_position_by_name(name); + DORIS_CHECK(position >= 0); + const auto& column = *block.get_by_position(position).column; + if (const auto* nullable = check_and_get_column(&column)) { + return nullable->get_nested_column(); + } + return column; +} + +bool is_null_at(const Block& block, const std::string& name, size_t row) { + const auto position = block.get_position_by_name(name); + DORIS_CHECK(position >= 0); + const auto* nullable = + check_and_get_column(block.get_by_position(position).column.get()); + DORIS_CHECK(nullable != nullptr); + return nullable->is_null_at(row); +} + +std::string string_at(const Block& block, const std::string& name, size_t row) { + const auto* column = check_and_get_column(&nested_column(block, name)); + DORIS_CHECK(column != nullptr); + return column->get_data_at(row).to_string(); +} + +Int64 int_at(const Block& block, const std::string& name, size_t row) { + return nested_column(block, name).get_int(row); +} + +Int64 struct_int_at(const Block& block, const std::string& name, size_t child_index, size_t row) { + const auto& struct_column = assert_cast(nested_column(block, name)); + const auto& child = assert_cast(struct_column.get_column(child_index)); + DORIS_CHECK(!child.is_null_at(row)); + return child.get_nested_column().get_int(row); +} + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +} // namespace + +TEST(IcebergPositionDeleteSysTableReaderTest, UsesScannerIOContext) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + io::FileReaderStats file_reader_stats; + scanner_io_ctx->file_reader_stats = &file_reader_stats; + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + + EXPECT_EQ(scanner_io_ctx.get(), reader._io_ctx.get()); + EXPECT_EQ(&file_reader_stats, reader._io_ctx->file_reader_stats); + scanner_io_ctx->should_stop = true; + EXPECT_TRUE(reader._io_ctx->should_stop); + + EXPECT_TRUE(reader.count_read_rows()); + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::DELETION_VECTOR; + EXPECT_FALSE(reader.count_read_rows()); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, ForwardsProfileCollectionToNestedReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + auto nested_reader = std::make_unique(); + auto* nested_reader_ptr = nested_reader.get(); + reader._position_reader = std::move(nested_reader); + + reader.collect_profile_before_close(); + reader.collect_profile_before_close(); + + EXPECT_EQ(1, nested_reader_ptr->collect_calls); + reader._dv_positions.add(uint64_t {1}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + reader._partition_value = std::make_shared()->create_column(); + ASSERT_TRUE(reader.close().ok()); + EXPECT_TRUE(reader._dv_positions.isEmpty()); + EXPECT_FALSE(reader._next_dv_position.has_value()); + EXPECT_EQ(nullptr, reader._partition_value.get()); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, StopsBeforeExpandingDeletionVector) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + scanner_io_ctx->should_stop = true; + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + Block block; + size_t read_rows = 1; + bool eof = false; + + ASSERT_TRUE(reader.get_next_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(0, read_rows); + EXPECT_TRUE(eof); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, FinalizesNestedReaderColumns) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + + IcebergPositionDeleteSysTableReader reader(file_slot_descs, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + auto nested_reader = std::make_unique(); + auto* nested_reader_ptr = nested_reader.get(); + reader._position_reader = std::move(nested_reader); + + std::unordered_map> + partition_columns; + std::unordered_map missing_columns; + ASSERT_TRUE(reader.set_fill_columns(partition_columns, missing_columns).ok()); + EXPECT_EQ(1, nested_reader_ptr->set_fill_columns_calls); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, ValidatesRangeAndDeleteFileMetadata) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + auto scanner_io_ctx = std::make_shared(); + + TFileRangeDesc empty_range; + IcebergPositionDeleteSysTableReader invalid_context_reader( + file_slot_descs, &state, &profile, empty_range, ¶ms, nullptr, nullptr); + auto status = invalid_context_reader.init_reader(); + EXPECT_TRUE(status.is()) << status; + + IcebergPositionDeleteSysTableReader missing_params_reader( + file_slot_descs, &state, &profile, empty_range, ¶ms, scanner_io_ctx, nullptr); + status = missing_params_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("range misses params")); + + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(TIcebergFileDesc()); + TFileRangeDesc no_delete_file_range; + no_delete_file_range.__set_table_format_params(std::move(table_format_desc)); + IcebergPositionDeleteSysTableReader no_delete_file_reader(file_slot_descs, &state, &profile, + no_delete_file_range, ¶ms, + scanner_io_ctx, nullptr); + status = no_delete_file_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("exactly one delete file")); + + TIcebergDeleteFileDesc missing_content; + auto missing_content_range = range_with_delete_file(missing_content); + IcebergPositionDeleteSysTableReader missing_content_reader(file_slot_descs, &state, &profile, + missing_content_range, ¶ms, + scanner_io_ctx, nullptr); + status = missing_content_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("misses content")); + + TIcebergDeleteFileDesc equality_delete; + equality_delete.__set_content(2); + auto equality_delete_range = range_with_delete_file(equality_delete); + IcebergPositionDeleteSysTableReader equality_delete_reader(file_slot_descs, &state, &profile, + equality_delete_range, ¶ms, + scanner_io_ctx, nullptr); + status = equality_delete_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("does not support delete file content 2")); + + TIcebergDeleteFileDesc missing_format; + missing_format.__set_content(1); + auto missing_format_range = range_with_delete_file(missing_format); + IcebergPositionDeleteSysTableReader missing_format_reader(file_slot_descs, &state, &profile, + missing_format_range, ¶ms, + scanner_io_ctx, nullptr); + status = missing_format_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("misses file format")); + + TIcebergDeleteFileDesc unsupported_format; + unsupported_format.__set_content(1); + unsupported_format.__set_file_format(TFileFormatType::FORMAT_CSV_PLAIN); + auto unsupported_format_range = range_with_delete_file(unsupported_format); + IcebergPositionDeleteSysTableReader unsupported_format_reader(file_slot_descs, &state, &profile, + unsupported_format_range, ¶ms, + scanner_io_ctx, nullptr); + status = unsupported_format_reader.init_reader(); + EXPECT_TRUE(status.is()) << status; + + TIcebergDeleteFileDesc invalid_dv; + invalid_dv.__set_content(3); + auto invalid_dv_range = range_with_delete_file(invalid_dv); + IcebergPositionDeleteSysTableReader invalid_dv_reader( + file_slot_descs, &state, &profile, invalid_dv_range, ¶ms, scanner_io_ctx, nullptr); + status = invalid_dv_reader.init_reader(); + EXPECT_NE(std::string::npos, status.to_string().find("misses referenced data file path")); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, AppendsDeletionVectorMetadataAndCachesPartition) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + ObjectPool pool; + const auto nullable_string = make_nullable(std::make_shared()); + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto nullable_int64 = make_nullable(std::make_shared()); + const auto partition_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"p"})); + std::vector slots { + make_slot(&pool, 0, "file_path", nullable_string), + make_slot(&pool, 1, "pos", nullable_int64), + make_slot(&pool, 2, "row", nullable_int32), + make_slot(&pool, 3, "partition", partition_type), + make_slot(&pool, 4, "spec_id", nullable_int32), + make_slot(&pool, 5, "delete_file_path", nullable_string), + make_slot(&pool, 6, "content_offset", nullable_int64), + make_slot(&pool, 7, "content_size_in_bytes", nullable_int64), + }; + auto scanner_io_ctx = std::make_shared(); + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_partition_spec_id(7); + iceberg_desc.__set_partition_data_json(R"({"p":42})"); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("/physical-delete.puffin"); + delete_file.__set_original_path("s3://bucket/delete.puffin"); + delete_file.__set_referenced_data_file_path("s3://bucket/data.parquet"); + delete_file.__set_content_offset(12); + delete_file.__set_content_size_in_bytes(34); + reader._iceberg_file_desc = &iceberg_desc; + reader._delete_file_desc = &delete_file; + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::DELETION_VECTOR; + reader._batch_size = 1; + reader._dv_positions.add(uint64_t {5}); + reader._dv_positions.add(uint64_t {9}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_FALSE(eof); + ASSERT_EQ(1, block.rows()); + ColumnPtr cached_partition = reader._partition_value; + ASSERT_NE(nullptr, cached_partition.get()); + EXPECT_EQ("s3://bucket/data.parquet", string_at(block, "file_path", 0)); + EXPECT_EQ(5, int_at(block, "pos", 0)); + EXPECT_TRUE(is_null_at(block, "row", 0)); + EXPECT_FALSE(is_null_at(block, "partition", 0)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 0)); + EXPECT_EQ(7, int_at(block, "spec_id", 0)); + EXPECT_EQ("s3://bucket/delete.puffin", string_at(block, "delete_file_path", 0)); + EXPECT_EQ(12, int_at(block, "content_offset", 0)); + EXPECT_EQ(34, int_at(block, "content_size_in_bytes", 0)); + + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); + ASSERT_EQ(2, block.rows()); + EXPECT_EQ(cached_partition.get(), reader._partition_value.get()); + EXPECT_EQ(9, int_at(block, "pos", 1)); + EXPECT_FALSE(is_null_at(block, "partition", 1)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 1)); + + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(0, read_rows); + EXPECT_TRUE(eof); + + reader._dv_positions = roaring::Roaring64Map(); + reader._dv_positions.add(uint64_t {13}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + std::vector partial_slots {slots[0], slots[1]}; + Block partial_block = make_output_block(partial_slots); + ASSERT_TRUE(reader._append_deletion_vector_block(&partial_block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); + EXPECT_EQ(13, int_at(partial_block, "pos", 0)); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, AppendsPositionDeleteRowsAndValidatesColumns) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + ObjectPool pool; + const auto nullable_string = make_nullable(std::make_shared()); + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto nullable_int64 = make_nullable(std::make_shared()); + std::vector slots { + make_slot(&pool, 0, "file_path", nullable_string), + make_slot(&pool, 1, "pos", nullable_int64), + make_slot(&pool, 2, "row", nullable_int32), + }; + auto scanner_io_ctx = std::make_shared(); + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + TIcebergFileDesc iceberg_desc; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("/delete.parquet"); + reader._iceberg_file_desc = &iceberg_desc; + reader._delete_file_desc = &delete_file; + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::POSITION_DELETE; + + reader.set_batch_size(17); + EXPECT_EQ(17, reader.get_batch_size()); + EXPECT_TRUE(reader._output_column_requested("row")); + EXPECT_FALSE(reader._output_column_requested("missing")); + reader._init_read_columns(true); + ASSERT_EQ(3, reader._read_columns.size()); + EXPECT_EQ("row", reader._read_columns.back().name); + + std::unordered_map name_to_type; + std::unordered_set missing_column_names; + ASSERT_TRUE(reader.get_columns(&name_to_type, &missing_column_names).ok()); + EXPECT_TRUE(missing_column_names.empty()); + EXPECT_EQ(3, name_to_type.size()); + + Block delete_block = reader._create_delete_block(); + { + auto columns_guard = delete_block.mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + parquet_utils::insert_string(columns[0], "s3://bucket/data.parquet"); + parquet_utils::insert_int64(columns[1], 19); + parquet_utils::insert_int32(columns[2], 23); + } + Block output_block = make_output_block(slots); + size_t appended_rows = 0; + ASSERT_TRUE(reader._append_position_delete_block(&output_block, delete_block, 1, &appended_rows) + .ok()); + EXPECT_EQ(1, appended_rows); + ASSERT_EQ(1, output_block.rows()); + EXPECT_EQ("s3://bucket/data.parquet", string_at(output_block, "file_path", 0)); + EXPECT_EQ(19, int_at(output_block, "pos", 0)); + EXPECT_EQ(23, int_at(output_block, "row", 0)); + + Block empty_delete_block = reader._create_delete_block(); + auto path_column = nullable_string->create_column(); + auto status = reader._append_sys_column(path_column, *slots[0], &empty_delete_block, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("file_path column is missing")); + auto pos_column = nullable_int64->create_column(); + status = reader._append_sys_column(pos_column, *slots[1], &empty_delete_block, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("pos column is missing")); + auto row_column = nullable_int32->create_column(); + ASSERT_TRUE(reader._append_sys_column(row_column, *slots[2], &empty_delete_block, 0, 0).ok()); + ASSERT_EQ(1, row_column->size()); + + Block missing_columns; + path_column = nullable_string->create_column(); + status = reader._append_sys_column(path_column, *slots[0], &missing_columns, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("file_path column is missing")); + pos_column = nullable_int64->create_column(); + status = reader._append_sys_column(pos_column, *slots[1], &missing_columns, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("pos column is missing")); + + std::vector partial_slots {slots[0], slots[1]}; + Block partial_output_block = make_output_block(partial_slots); + appended_rows = 0; + ASSERT_TRUE(reader._append_position_delete_block(&partial_output_block, delete_block, 1, + &appended_rows) + .ok()); + EXPECT_EQ(1, appended_rows); + EXPECT_EQ(1, partial_output_block.rows()); + + auto* unknown_slot = make_slot(&pool, 3, "unknown", nullable_string); + auto unknown_column = nullable_string->create_column(); + status = reader._append_sys_column(unknown_column, *unknown_slot, &delete_block, 0, 0); + EXPECT_NE(std::string::npos, status.to_string().find("Unknown Iceberg")); + + Block unused_block; + size_t read_rows = 0; + bool eof = false; + status = reader.get_next_block(&unused_block, &read_rows, &eof); + EXPECT_NE(std::string::npos, status.to_string().find("reader is not initialized")); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, AppendsNullMetadataAndUsesDeletePathFallback) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileRangeDesc range; + TFileScanRangeParams params; + ObjectPool pool; + const auto nullable_string = make_nullable(std::make_shared()); + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto nullable_int64 = make_nullable(std::make_shared()); + const auto partition_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"p"})); + std::vector slots { + make_slot(&pool, 0, "partition", partition_type), + make_slot(&pool, 1, "spec_id", nullable_int32), + make_slot(&pool, 2, "delete_file_path", nullable_string), + make_slot(&pool, 3, "content_offset", nullable_int64), + make_slot(&pool, 4, "content_size_in_bytes", nullable_int64), + }; + auto scanner_io_ctx = std::make_shared(); + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + TIcebergFileDesc iceberg_desc; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_path("/fallback-delete-file"); + reader._iceberg_file_desc = &iceberg_desc; + reader._delete_file_desc = &delete_file; + reader._delete_file_kind = IcebergPositionDeleteSysTableReader::DeleteFileKind::DELETION_VECTOR; + reader._dv_positions.add(uint64_t {1}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); + EXPECT_TRUE(is_null_at(block, "partition", 0)); + EXPECT_TRUE(is_null_at(block, "spec_id", 0)); + EXPECT_EQ("/fallback-delete-file", string_at(block, "delete_file_path", 0)); + EXPECT_TRUE(is_null_at(block, "content_offset", 0)); + EXPECT_TRUE(is_null_at(block, "content_size_in_bytes", 0)); + + std::vector empty_slots; + IcebergPositionDeleteSysTableReader empty_reader(empty_slots, &state, &profile, range, ¶ms, + scanner_io_ctx, nullptr); + empty_reader._dv_positions.add(uint64_t {2}); + empty_reader._next_dv_position.emplace(empty_reader._dv_positions.begin()); + Block empty_block; + ASSERT_TRUE(empty_reader._append_deletion_vector_block(&empty_block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_TRUE(eof); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, RecordsDeletionVectorRows) { + io::FileReaderStats file_reader_stats; + auto scanner_io_ctx = std::make_shared(); + scanner_io_ctx->file_reader_stats = &file_reader_stats; + std::vector file_slot_descs; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._io_ctx = scanner_io_ctx; + reader._file_slot_descs = &file_slot_descs; + reader._batch_size = 2; + reader._dv_positions.add(uint64_t {7}); + reader._dv_positions.add(uint64_t {9}); + reader._dv_positions.add(uint64_t {11}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block; + size_t read_rows = 0; + bool eof = true; + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(2, read_rows); + EXPECT_FALSE(eof); + EXPECT_EQ(2, file_reader_stats.read_rows); + + ASSERT_TRUE(reader._append_deletion_vector_block(&block, &read_rows, &eof).ok()); + EXPECT_EQ(1, read_rows); + EXPECT_FALSE(eof); + EXPECT_EQ(3, file_reader_stats.read_rows); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, CachesAndClearsPartitionValue) { + ObjectPool pool; + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto partition_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"p"})); + auto* partition_slot = make_slot(&pool, 0, "partition", partition_type); + + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_partition_data_json(R"({"p":42})"); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._iceberg_file_desc = &iceberg_desc; + auto partition_column = partition_type->create_column(); + ASSERT_TRUE(reader._append_partition_column(partition_column, *partition_slot).ok()); + ColumnPtr cached_partition = reader._partition_value; + ASSERT_NE(nullptr, cached_partition.get()); + + ASSERT_TRUE(reader._append_partition_column(partition_column, *partition_slot).ok()); + EXPECT_EQ(cached_partition.get(), reader._partition_value.get()); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(partition_column), partition_type, "partition")); + ASSERT_EQ(2, block.rows()); + EXPECT_FALSE(is_null_at(block, "partition", 0)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 0)); + EXPECT_FALSE(is_null_at(block, "partition", 1)); + EXPECT_EQ(42, struct_int_at(block, "partition", 0, 1)); + + reader._has_split = true; + reader._dv_positions.add(uint64_t {1}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + ASSERT_TRUE(reader.close().ok()); + EXPECT_EQ(nullptr, reader._iceberg_file_desc); + EXPECT_EQ(nullptr, reader._partition_value.get()); + EXPECT_TRUE(reader._dv_positions.isEmpty()); + EXPECT_FALSE(reader._next_dv_position.has_value()); + EXPECT_FALSE(reader._has_split); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ValidatesDeleteFileContentAfterBindingDescriptor) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc equality_delete; + equality_delete.__set_content(2); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(equality_delete); + + auto status = reader._init_split(); + EXPECT_NE(std::string::npos, status.to_string().find("does not support delete file content 2")); + ASSERT_NE(nullptr, reader._iceberg_file_desc); + ASSERT_NE(nullptr, reader._delete_file_desc); + EXPECT_EQ(reader._iceberg_file_desc->delete_files.data(), reader._delete_file_desc); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVector) { + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._io_ctx = std::make_shared(); + reader._io_ctx->should_stop = true; + + Block block; + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_TRUE(eof); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, + AllFilteredDeletionVectorYieldsBeforeObservingCancellation) { + ObjectPool pool; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + const auto nullable_int64 = make_nullable(std::make_shared()); + std::vector file_slot_descs { + make_slot(&pool, 0, "pos", nullable_int64), + }; + + auto conjunct = VExprContext::create_shared(std::make_shared()); + RowDescriptor row_desc; + ASSERT_TRUE(conjunct->prepare(&state, row_desc).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._io_ctx = std::make_shared(); + reader._file_slot_descs = &file_slot_descs; + reader._projected_columns.resize(file_slot_descs.size()); + reader._remaining_conjuncts = {conjunct}; + reader._has_split = true; + reader._delete_file_kind = + format::iceberg::IcebergPositionDeleteSysTableV2Reader::DeleteFileKind::DELETION_VECTOR; + reader._batch_size = 1; + reader._dv_positions.add(uint64_t {7}); + reader._dv_positions.add(uint64_t {9}); + reader._dv_positions.add(uint64_t {11}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(file_slot_descs); + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_FALSE(eof); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); + + reader._io_ctx->should_stop = true; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); +} + +} // namespace doris diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp new file mode 100644 index 00000000000000..4ea14766356d15 --- /dev/null +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/transformer/vorc_transformer.h" + +#include + +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/table/iceberg/schema_parser.h" +#include "io/fs/local_file_system.h" +#include "runtime/runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" +#include "util/uid_util.h" + +namespace doris { + +class VOrcTransformerTest : public testing::Test { +protected: + void SetUp() override { + _file_path = "./vorc_transformer_" + UniqueId::gen_uid().to_string() + ".orc"; + _fs = io::global_local_filesystem(); + } + + void TearDown() override { static_cast(_fs->delete_file(_file_path)); } + + std::string _file_path; + std::shared_ptr _fs; +}; + +TEST_F(VOrcTransformerTest, CollectsBoundsForTopLevelFieldAfterStruct) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto string_type = std::make_shared(); + VExprContextSPtrs output_exprs = + MockSlotRef::create_mock_contexts(DataTypes {struct_type, string_type}); + + const std::string schema_json = R"({ + "type": "struct", + "fields": [ + { + "id": 1, + "name": "s", + "required": true, + "type": { + "type": "struct", + "fields": [ + {"id": 2, "name": "a", "required": true, "type": "int"} + ] + } + }, + {"id": 3, "name": "b", "required": true, "type": "string"} + ] + })"; + std::unique_ptr schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + VOrcTransformer transformer(&state, file_writer.get(), output_exprs, "", {"s", "b"}, false, + TFileCompressType::PLAIN, schema.get(), _fs); + ASSERT_TRUE(transformer.open().ok()); + + auto nested_column = ColumnInt32::create(); + nested_column->insert_value(-1); + Columns struct_columns; + struct_columns.emplace_back(std::move(nested_column)); + auto struct_column = ColumnStruct::create(std::move(struct_columns)); + auto string_column = ColumnString::create(); + string_column->insert_data("hello", 5); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(struct_column), struct_type, "s")); + block.insert(ColumnWithTypeAndName(std::move(string_column), string_type, "b")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + TIcebergColumnStats stats; + ASSERT_TRUE(transformer.collect_file_statistics_after_close(&stats).ok()); + ASSERT_TRUE(stats.__isset.lower_bounds); + ASSERT_TRUE(stats.__isset.upper_bounds); + ASSERT_EQ(1, stats.lower_bounds.count(3)); + ASSERT_EQ(1, stats.upper_bounds.count(3)); + EXPECT_EQ("hello", stats.lower_bounds.at(3)); + EXPECT_EQ("hello", stats.upper_bounds.at(3)); +} + +} // namespace doris diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index d241059d6e8202..d3428b6134a479 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -28,6 +28,7 @@ #include "common/consts.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_decimal.h" @@ -38,6 +39,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/vin_predicate.h" @@ -253,6 +255,81 @@ TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) { } } +TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { + auto table_child = field_id_col("a", 1, i32()); + auto table_struct = struct_col("s", 10, {table_child}); + auto file_child = field_id_col("legacy_a", 1, i32(), 0); + auto file_struct = struct_name_col("s", {file_child}, 0); + + TableColumnMapper parquet_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(parquet_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(parquet_mapper.mappings().size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(parquet_mapper.mappings()[0].child_mappings.size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].child_mappings[0].file_local_id, 0); + + TableColumnMapper orc_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(orc_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + auto table_struct = struct_col("s", 10, {field_id_col("a", 2, i32())}); + table_struct.has_name_mapping = true; + auto file_struct = struct_name_col("s", {field_id_col("a", 2, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + EXPECT_TRUE(mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetRetainsRecursiveIdlessWrapperWithNestedFieldId) { + auto table_inner = struct_col("inner", 20, {field_id_col("leaf", 30, i32())}); + auto table_outer = struct_col("outer", 10, {table_inner}); + auto file_inner = struct_name_col("inner", {field_id_col("leaf", 30, i32(), 0)}, 0); + auto file_outer = struct_col("outer", 10, {file_inner}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_outer}, {}, {file_outer}).ok()); + + const auto& outer_mapping = mapper.mappings()[0]; + ASSERT_TRUE(outer_mapping.file_local_id.has_value()); + ASSERT_EQ(outer_mapping.child_mappings.size(), 1); + const auto& inner_mapping = outer_mapping.child_mappings[0]; + ASSERT_TRUE(inner_mapping.file_local_id.has_value()); + ASSERT_EQ(inner_mapping.child_mappings.size(), 1); + EXPECT_TRUE(inner_mapping.child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { + auto defaulted_child = field_id_col("data", 2, varbinary()); + defaulted_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + defaulted_child.initial_default_value_is_base64 = true; + auto table_struct = struct_col("s", 10, {field_id_col("a", 1, i32()), defaulted_child}); + auto file_struct = struct_col("s", 10, {field_id_col("a", 1, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + const auto& missing = mapper.mappings()[0].child_mappings[1]; + ASSERT_TRUE(missing.initial_default_column); + Field value; + missing.initial_default_column->get(0, value); + EXPECT_EQ(value.get_type(), TYPE_VARBINARY); + EXPECT_EQ(std::string(value.get()), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + void expect_mapping(const ColumnMapping& mapping, size_t global_index, const std::string& table_name, int32_t file_local_id, const std::string& file_name, const DataTypePtr& file_type, @@ -316,6 +393,38 @@ class TestFunctionExpr final : public VExpr { std::string _expr_name; }; +class ExecutableStructElementExpr final : public VExpr { +public: + explicit ExecutableStructElementExpr(DataTypePtr child_type) + : VExpr(std::move(child_type), false) { + set_node_type(TExprNodeType::FUNCTION_CALL); + TFunctionName fn_name; + fn_name.__set_function_name(_expr_name); + _fn.__set_name(fn_name); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(data_type()); + return Status::OK(); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr struct_column; + RETURN_IF_ERROR( + get_child(0)->execute_column(context, block, selector, count, struct_column)); + const auto& input = assert_cast(*struct_column); + result_column = input.get_column_ptr(0); + return Status::OK(); + } + +private: + const std::string _expr_name = "element_at"; +}; + VExprSPtr table_slot(int slot_id, int column_id, DataTypePtr type, const std::string& name) { return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name); } @@ -340,6 +449,40 @@ VExprSPtr element_at(const VExprSPtr& parent, DataTypePtr child_type, return expr; } +VExprSPtr executable_struct_element(const VExprSPtr& parent, DataTypePtr child_type, + const std::string& child_name) { + auto expr = std::make_shared(std::move(child_type)); + expr->add_child(parent); + expr->add_child(literal(str(), Field::create_field(child_name))); + return expr; +} + +VExprSPtr executable_binary_predicate(TExprOpcode::type opcode, const VExprSPtr& left, + const VExprSPtr& right) { + const auto result_type = u8(); + TFunctionName fn_name; + fn_name.__set_function_name(opcode == TExprOpcode::GT ? "gt" : "eq"); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({left->data_type()->to_thrift(), right->data_type()->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + + auto expr = VectorizedFnCall::create_shared(node); + expr->add_child(left); + expr->add_child(right); + return expr; +} + VExprSPtr array_element_at(const VExprSPtr& parent, DataTypePtr child_type, int64_t ordinal) { auto expr = std::make_shared("element_at", std::move(child_type)); expr->add_child(parent); @@ -474,7 +617,7 @@ class Int64ChildGreaterThanExpr final : public VExpr { size_t count, ColumnPtr& result_column) const override { ColumnPtr child_column; RETURN_IF_ERROR( - get_child(0)->execute_column_impl(context, block, selector, count, child_column)); + get_child(0)->execute_column(context, block, selector, count, child_column)); const auto& input = assert_cast(*child_column); auto result = ColumnUInt8::create(); auto& result_data = result->get_data(); @@ -510,11 +653,10 @@ class Int64BinaryPredicateExpr final : public VExpr { Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override { ColumnPtr left_column; - RETURN_IF_ERROR( - get_child(0)->execute_column_impl(context, block, selector, count, left_column)); + RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left_column)); ColumnPtr right_column; RETURN_IF_ERROR( - get_child(1)->execute_column_impl(context, block, selector, count, right_column)); + get_child(1)->execute_column(context, block, selector, count, right_column)); auto result = ColumnUInt8::create(); auto& result_data = result->get_data(); @@ -1929,7 +2071,8 @@ TEST(ColumnMapperConstantTest, PartitionDefaultAndVirtualColumnsUseDedicatedBran {"dt", Field::create_field("2026-06-11")}, }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + TableColumnMapper mapper( + {.mode = TableColumnMappingMode::BY_NAME, .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, partition_values, {}).ok()); ASSERT_EQ(mapper.mappings().size(), 5); @@ -1950,7 +2093,8 @@ TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { name_col("_last_updated_sequence_number", make_nullable(i64()), 2147483539), }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + TableColumnMapper mapper( + {.mode = TableColumnMappingMode::BY_NAME, .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); ASSERT_EQ(mapper.mappings().size(), 2); @@ -1983,6 +2127,25 @@ TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { std::vector({2147483540, 2147483539})); } +TEST(ColumnMapperConstantTest, GenericByNameKeepsRowLineageNamesPhysical) { + const std::vector table_schema = { + name_col("_row_id", make_nullable(i64())), + name_col("_last_updated_sequence_number", make_nullable(i64())), + }; + const std::vector file_schema = { + name_col("_row_id", make_nullable(i64()), 0), + name_col("_last_updated_sequence_number", make_nullable(i64()), 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + EXPECT_EQ(mapper.mappings()[0].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::COPY_DIRECTLY); + EXPECT_EQ(mapper.mappings()[1].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[1].filter_conversion, FilterConversionType::COPY_DIRECTLY); +} + TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMapping) { auto id_column = field_id_col("id", 1, make_nullable(i32())); auto row_id_column = field_id_col("renamed_row_id", 2147483540, make_nullable(i64())); @@ -1999,7 +2162,8 @@ TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMappi field_id_col("name", 2, make_nullable(str()), 1), }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); ASSERT_EQ(mapper.mappings().size(), 3); @@ -2173,6 +2337,39 @@ TEST(ColumnMapperLocalizeFiltersTest, VisibleLocalFilterAddsPredicateColumnAndCo EXPECT_TRUE(localized_slot->data_type()->equals(*int_type)); } +TEST(ColumnMapperLocalizeFiltersTest, ReportsLocalizationForEachSplitMapping) { + const auto int_type = i32(); + auto table_column = name_col("id", int_type); + const std::vector table_schema = {table_column}; + TableFilter filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(0, 0, int_type, "id"), 1)), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper local_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(local_mapper.create_mapping(table_schema, {}, {name_col("id", int_type, 7)}).ok()); + FileScanRequest local_request; + FilterLocalizationResult local_result; + ASSERT_TRUE(local_mapper + .create_scan_request({filter}, table_schema, &local_request, nullptr, + &local_result) + .ok()); + ASSERT_EQ(local_result.localized_filters.size(), 1); + EXPECT_TRUE(local_result.localized_filters[0]); + ASSERT_EQ(local_request.conjuncts.size(), 1); + + TableColumnMapper missing_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(missing_mapper.create_mapping(table_schema, {}, {}).ok()); + FileScanRequest missing_request; + FilterLocalizationResult missing_result; + ASSERT_TRUE(missing_mapper + .create_scan_request({filter}, table_schema, &missing_request, nullptr, + &missing_result) + .ok()); + ASSERT_EQ(missing_result.localized_filters.size(), 1); + EXPECT_FALSE(missing_result.localized_filters[0]); + EXPECT_TRUE(missing_request.conjuncts.empty()); +} + TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { const auto binary_type = varbinary(); const auto table_column = name_col("partition_key", binary_type); @@ -2198,6 +2395,35 @@ TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { EXPECT_TRUE(request.conjuncts.empty()); } +TEST(ColumnMapperLocalizeFiltersTest, VarcharWidthTruncationFilterStaysAboveFileReader) { + const auto table_type = std::make_shared(3, TYPE_VARCHAR); + const auto file_type = std::make_shared(10, TYPE_VARCHAR); + const auto table_column = name_col("value", table_type); + const auto file_column = name_col("value", file_type, 7); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(binary_predicate( + TExprOpcode::EQ, table_slot(0, 0, table_type, "value"), + literal(table_type, Field::create_field("abc")))), + .global_indices = {GlobalIndex(0)}}; + TQueryOptions query_options; + query_options.__set_truncate_char_or_varchar_columns(true); + RuntimeState state {query_options, TQueryGlobals()}; + FileScanRequest request; + FilterLocalizationResult localization_result; + + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request, &state, + &localization_result) + .ok()); + ASSERT_EQ(localization_result.localized_filters.size(), 1); + EXPECT_FALSE(localization_result.localized_filters[0]); + EXPECT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(7)); +} + TEST(ColumnMapperLocalizeFiltersTest, NestedVarbinaryFilterStaysAboveFileReader) { const auto table_column = struct_name_col( "payload", {name_col("id", i32()), name_col("binary_value", varbinary())}); @@ -2343,10 +2569,41 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(1)})); ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } +TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsOutputPayload) { + const auto int_type = i32(); + auto quantity = name_col("ss_quantity", int_type); + auto tax = name_col("ss_ext_tax", int_type); + const std::vector table_schema = {quantity, tax}; + const std::vector file_schema = { + name_col("ss_quantity", int_type, 0), + name_col("ss_ext_tax", int_type, 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + + auto filter_expr = int_gt(table_slot(7, 0, int_type, "ss_quantity"), 20); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, table_schema, &request).ok()); + + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); + // A visible predicate slot is still part of the table output and cannot be replaced with a + // default-valued placeholder after file-local filtering. + EXPECT_TRUE(request.predicate_only_columns.empty()); +} + TEST(ColumnMapperScanRequestTest, StructOutputAndFilterOnlyChildAreMerged) { const auto int_type = i32(); const auto string_type = str(); @@ -2868,6 +3125,7 @@ TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(10)})); EXPECT_TRUE(request.predicate_columns[0].project_all_children); EXPECT_TRUE(request.predicate_columns[0].children.empty()); @@ -2994,6 +3252,251 @@ TEST(ColumnMapperScanRequestTest, NestedElementAtConjunctUsesFileChildTypeForRen EXPECT_EQ(localized_parent_type->get_element_name(1), "bb"); } +// Scenario: Iceberg promotes a nested struct leaf from INT to BIGINT while an old file still +// stores INT. Because 15 is exactly representable as INT and every INT survives promotion to +// BIGINT, localize `s.b::BIGINT > 15::BIGINT` as `file_s.b::INT > 15::INT`. Rewriting one literal +// avoids casting every file value while preserving the table predicate exactly. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctRewritesExactLiteralToFileType) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, table_leaf, + literal(table_bigint_type, Field::create_field(15))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + ASSERT_EQ(localized_root->get_num_children(), 2); + const auto& localized_leaf = localized_root->children()[0]; + EXPECT_EQ(localized_leaf->expr_name(), "element_at"); + EXPECT_TRUE(localized_leaf->data_type()->equals(*file_int_type)); + const auto& localized_literal = localized_root->children()[1]; + EXPECT_TRUE(localized_literal->is_literal()); + EXPECT_TRUE(localized_literal->data_type()->equals(*file_int_type)); + + auto values = ColumnInt32::create(); + values->insert_value(10); + values->insert_value(20); + MutableColumns children; + children.push_back(std::move(values)); + Block block; + block.insert({ColumnStruct::create(std::move(children)), mapper.mappings()[0].file_type, "s"}); + + auto* conjunct = request.conjuncts[0].get(); + auto status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter result(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, result.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + EXPECT_EQ(result, IColumn::Filter({0, 1})); + conjunct->close(); +} + +// Scenario: an old file allows NULL for a nested leaf that the current table declares required. +// Although every non-NULL INT value and the literal 15 can be promoted to BIGINT exactly, the +// predicate must stay above TableReader. If `file_s.b > 15` ran first for rows [NULL, 20], it would +// discard NULL and prevent table-schema materialization from reporting the nullable-to-required +// contract violation. +TEST_F(ColumnMapperCastTest, + NestedElementAtConjunctStaysTableLevelForNullableFileLeafMappedToRequiredTableLeaf) { + const auto file_nullable_int_type = make_nullable(i32()); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_nullable_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, table_leaf, + literal(table_bigint_type, Field::create_field(15))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.predicate_columns.size() + request.non_predicate_columns.size(), 1); + const auto& scan_column = request.predicate_columns.empty() ? request.non_predicate_columns[0] + : request.predicate_columns[0]; + EXPECT_EQ(scan_column.column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +// Scenario: a narrowing file-to-table cast can produce NULL or an error for values that do not fit +// the table leaf. Evaluating that cast below TableReader can filter those rows before +// _align_column_nullability() validates the required table child. Keep the predicate at table level +// so schema materialization observes every source row first. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctStaysTableLevelForNonLosslessFileToTableCast) { + const auto file_bigint_type = i64(); + const auto table_int_type = i32(); + + auto table_a = field_id_col("a", 11, table_int_type); + auto table_struct = struct_col("s", 10, {table_a}); + auto file_a = field_id_col("a", 11, file_bigint_type, 0); + auto file_struct = struct_col("s", 10, {file_a}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_int_type, "a"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::EQ, table_leaf, literal(table_int_type, Field::create_field(1))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.predicate_columns.size() + request.non_predicate_columns.size(), 1); + const auto& scan_column = request.predicate_columns.empty() ? request.non_predicate_columns[0] + : request.predicate_columns[0]; + EXPECT_EQ(scan_column.column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +// Scenario: the table literal is outside the old file leaf's INT range. Rewriting +// BIGINT 2147483648 to INT would change the predicate, so keep the literal as BIGINT and cast the +// file leaf instead: `CAST(file_s.b::INT AS BIGINT) = 2147483648::BIGINT`. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctFallsBackForOutOfRangeLiteral) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::EQ, table_leaf, + literal(table_bigint_type, Field::create_field(2147483648LL))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + ASSERT_EQ(localized_root->get_num_children(), 2); + const auto& localized_cast = localized_root->children()[0]; + ASSERT_NE(dynamic_cast(localized_cast.get()), nullptr); + EXPECT_TRUE(localized_cast->data_type()->equals(*table_bigint_type)); + ASSERT_EQ(localized_cast->get_num_children(), 1); + EXPECT_EQ(localized_cast->children()[0]->expr_name(), "element_at"); + EXPECT_TRUE(localized_cast->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(localized_root->children()[1]->data_type()->equals(*table_bigint_type)); +} + +// Scenario: the struct leaf is on the right side of the comparison. Literal localization must not +// depend on operand order: `15::BIGINT > s.b::BIGINT` becomes `15::INT > file_s.b::INT`. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctRewritesReverseComparisonLiteral) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, literal(table_bigint_type, Field::create_field(15)), + table_leaf); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + EXPECT_TRUE(localized_root->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(localized_root->children()[0]->is_literal()); + EXPECT_EQ(localized_root->children()[1]->expr_name(), "element_at"); + EXPECT_TRUE(localized_root->children()[1]->data_type()->equals(*file_int_type)); +} + +// Scenario: IN uses one probe type for every candidate. All exact literals may move to the INT +// file domain, but one out-of-range literal makes the complete IN predicate fall back to BIGINT. +TEST_F(ColumnMapperCastTest, NestedElementAtInPredicateUsesAllOrNothingLiteralRewrite) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + const auto build_filter = [&](int64_t second_value) { + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto predicate = create_in_predicate(); + predicate->add_child(table_leaf); + predicate->add_child(literal(table_bigint_type, Field::create_field(10))); + predicate->add_child( + literal(table_bigint_type, Field::create_field(second_value))); + return TableFilter {.conjunct = VExprContext::create_shared(predicate), + .global_indices = {GlobalIndex(0)}}; + }; + + TableColumnMapper exact_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(exact_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest exact_request; + ASSERT_TRUE( + exact_mapper + .create_scan_request({build_filter(20)}, {table_struct}, &exact_request, &state) + .ok()); + ASSERT_EQ(exact_request.conjuncts.size(), 1); + const auto& exact_root = exact_request.conjuncts[0]->root(); + EXPECT_EQ(exact_root->children()[0]->expr_name(), "element_at"); + for (const auto& child : exact_root->children()) { + EXPECT_TRUE(child->data_type()->equals(*file_int_type)); + } + + TableColumnMapper fallback_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(fallback_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest fallback_request; + ASSERT_TRUE(fallback_mapper + .create_scan_request({build_filter(2147483648LL)}, {table_struct}, + &fallback_request, &state) + .ok()); + ASSERT_EQ(fallback_request.conjuncts.size(), 1); + const auto& fallback_root = fallback_request.conjuncts[0]->root(); + const auto& fallback_cast = fallback_root->children()[0]; + ASSERT_NE(dynamic_cast(fallback_cast.get()), nullptr); + EXPECT_TRUE(fallback_cast->data_type()->equals(*table_bigint_type)); + EXPECT_TRUE(fallback_cast->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(fallback_root->children()[1]->data_type()->equals(*table_bigint_type)); + EXPECT_TRUE(fallback_root->children()[2]->data_type()->equals(*table_bigint_type)); +} + // Scenario: output projection reads one struct child while the row filter reads a different nested // struct child. File-local conjunct rewrite must use the merged scan projection type. In the SQL // shape below, `SELECT element_at(s, 'c') WHERE element_at(element_at(s, 'b'), 'cc') LIKE ...` @@ -3548,116 +4051,6 @@ TEST_F(ColumnMapperCastTest, ColumnMapperCastsLiteralForLiteralSlotPredicateType file_request.conjuncts[0]->close(); } -// Scenario: Nereids may represent `BIGINT value = 1` as `value = CAST(INT 1 AS BIGINT)`. -// Strip a provably lossless literal widening so metadata readers can recognize slot-literal -// predicates for zone-map pruning. -TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesImplicitlyCastLiteral) { - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); - auto table_column = name_col("value", i64()); - std::vector projected_columns {table_column}; - auto file_field = name_col("value", i64(), 0); - std::vector file_schema {file_field}; - - auto status = mapper.create_mapping(projected_columns, {}, file_schema); - ASSERT_TRUE(status.ok()) << status; - - auto predicate = std::make_shared(TExprOpcode::GT); - predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); - predicate->add_child(cast_expr(VLiteral::create_shared(i32(), Field::create_field(1)), - table_column.type)); - TableFilter table_filter; - table_filter.conjunct = VExprContext::create_shared(predicate); - table_filter.global_indices = {GlobalIndex(0)}; - - FileScanRequest file_request; - ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) - .ok()); - ASSERT_EQ(file_request.conjuncts.size(), 1); - const auto& localized_expr = file_request.conjuncts[0]->root(); - ASSERT_EQ(localized_expr->get_num_children(), 2); - EXPECT_TRUE(localized_expr->children()[1]->is_literal()); - EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); - - Block block; - block.insert(ColumnHelper::create_column_with_name({1, 2})); - auto* conjunct = file_request.conjuncts[0].get(); - ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok()); - ASSERT_TRUE(conjunct->open(&state).ok()); - IColumn::Filter filter(block.rows(), 1); - bool can_filter_all = false; - ASSERT_TRUE( - conjunct->execute_filter(&block, filter.data(), block.rows(), false, &can_filter_all) - .ok()); - EXPECT_EQ(filter, IColumn::Filter({0, 1})); - conjunct->close(); -} - -// Scenario: Nereids may keep a narrow literal directly under an implicitly coerced comparison, -// for example `nullable INT value = TINYINT 1`. Normalize the literal to the table type so the -// file predicate remains recognizable to Parquet zone-map pruning. -TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesImplicitlyTypedLiteral) { - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); - auto column_type = make_nullable(i32()); - auto table_column = name_col("value", column_type); - std::vector projected_columns {table_column}; - auto file_field = name_col("value", column_type, 0); - std::vector file_schema {file_field}; - - auto status = mapper.create_mapping(projected_columns, {}, file_schema); - ASSERT_TRUE(status.ok()) << status; - - auto predicate = std::make_shared(TExprOpcode::GT); - predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); - predicate->add_child( - VLiteral::create_shared(std::make_shared(), - Field::create_field(static_cast(1)))); - TableFilter table_filter; - table_filter.conjunct = VExprContext::create_shared(predicate); - table_filter.global_indices = {GlobalIndex(0)}; - - FileScanRequest file_request; - ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) - .ok()); - ASSERT_EQ(file_request.conjuncts.size(), 1); - const auto& localized_expr = file_request.conjuncts[0]->root(); - ASSERT_EQ(localized_expr->get_num_children(), 2); - EXPECT_TRUE(localized_expr->children()[1]->is_literal()); - EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); -} - -// Scenario: branch-4.1 Nereids comparisons can be resolved by function name without retaining the -// legacy opcode. The node kind still identifies a binary comparison, so its narrow literal must be -// normalized exactly like the opcode-bearing form used above. -TEST_F(ColumnMapperCastTest, ColumnMapperLocalizesBinaryPredicateWithoutLegacyOpcode) { - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); - auto column_type = make_nullable(i32()); - auto table_column = name_col("value", column_type); - std::vector projected_columns {table_column}; - auto file_field = name_col("value", column_type, 0); - - auto status = mapper.create_mapping(projected_columns, {}, {file_field}); - ASSERT_TRUE(status.ok()) << status; - - auto predicate = std::make_shared(TExprOpcode::INVALID_OPCODE); - predicate->add_child(VSlotRef::create_shared(0, 0, -1, table_column.type, "value")); - predicate->add_child( - VLiteral::create_shared(std::make_shared(), - Field::create_field(static_cast(1)))); - TableFilter table_filter; - table_filter.conjunct = VExprContext::create_shared(predicate); - table_filter.global_indices = {GlobalIndex(0)}; - - FileScanRequest file_request; - ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request, &state) - .ok()); - ASSERT_EQ(file_request.conjuncts.size(), 1); - const auto& localized_expr = file_request.conjuncts[0]->root(); - ASSERT_EQ(localized_expr->get_num_children(), 2); - EXPECT_TRUE(localized_expr->children()[0]->data_type()->equals(*file_field.type)); - EXPECT_TRUE(localized_expr->children()[1]->is_literal()); - EXPECT_TRUE(localized_expr->children()[1]->data_type()->equals(*file_field.type)); -} - // Scenario: a fractional table literal cannot be localized to an integral file type without // changing the predicate boundary, so the mapper must cast the file slot instead. TEST_F(ColumnMapperCastTest, ColumnMapperRejectsLossyBinaryLiteralConversion) { diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp index 0ce99630c7af8b..c04e17e07f64b8 100644 --- a/be/test/format_v2/delimited_text/csv_reader_test.cpp +++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp @@ -446,7 +446,7 @@ TEST_F(CsvV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); @@ -755,6 +755,20 @@ TEST_F(CsvV2ReaderTest, CountAggregateScansRows) { EXPECT_EQ(aggregate_result.count, 2); } +TEST_F(CsvV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + aggregate_request.columns.push_back( + {.projection = LocalColumnIndex::top_level(LocalColumnId(1))}); + FileAggregateResult aggregate_result; + EXPECT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); +} + // Scenario: CSV v2 parses enclosed fields itself instead of delegating to the old CsvReader. A // separator inside an enclosed string must stay inside the same CSV field. TEST_F(CsvV2ReaderTest, EnclosedFieldKeepsSeparatorInsideStringValue) { diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp b/be/test/format_v2/delimited_text/text_reader_test.cpp index f5f7309e490ff1..4927ee7622e4d4 100644 --- a/be/test/format_v2/delimited_text/text_reader_test.cpp +++ b/be/test/format_v2/delimited_text/text_reader_test.cpp @@ -41,6 +41,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_profile.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "testutil/desc_tbl_builder.h" @@ -414,7 +415,7 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); @@ -423,6 +424,33 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); } +TEST_F(TextV2ReaderTest, FormatProfilesKeepDistinctCountersInBothInitializationOrders) { + auto expect_format_children = [](RuntimeProfile* profile) { + TRuntimeProfileTree tree; + profile->to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("DelimitedTextReader")); + EXPECT_TRUE(children.at("DelimitedTextReader").contains("DelimitedRowsFilteredByConjunct")); + EXPECT_FALSE(children.at("DelimitedTextReader").contains("RowsFilteredByConjunct")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowsFilteredByConjunct")); + EXPECT_FALSE(children.at("ParquetReader").contains("DelimitedRowsFilteredByConjunct")); + }; + + RuntimeProfile parquet_first("parquet_first"); + parquet::ParquetProfile parquet_first_counters; + parquet_first_counters.init(&parquet_first); + auto parquet_first_text = create_reader(_file_path, &_params, _slots, &_state, &parquet_first); + expect_format_children(&parquet_first); + + RuntimeProfile text_first("text_first"); + auto text_first_reader = create_reader(_file_path, &_params, _slots, &_state, &text_first); + parquet::ParquetProfile text_first_parquet_counters; + text_first_parquet_counters.init(&text_first); + expect_format_children(&text_first); +} + // Scenario: Hive text has no embedded nested schema, but TableColumnMapper still needs semantic // children for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the // slot type while keeping the top-level local id as the text field ordinal from column_idxs. @@ -909,6 +937,20 @@ TEST_F(TextV2ReaderTest, CountAggregateScansRows) { EXPECT_EQ(aggregate_result.count, 2); } +TEST_F(TextV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + aggregate_request.columns.push_back( + {.projection = LocalColumnIndex::top_level(LocalColumnId(1))}); + FileAggregateResult aggregate_result; + EXPECT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); +} + // Scenario: a non-first split starts inside a text record and must skip the partial first line. TEST_F(TextV2ReaderTest, NonFirstSplitSkipsPartialFirstRecord) { const auto split_path = (_test_dir / "split.text").string(); diff --git a/be/test/format_v2/jni/jdbc_reader_test.cpp b/be/test/format_v2/jni/jdbc_reader_test.cpp new file mode 100644 index 00000000000000..5763a6deb00895 --- /dev/null +++ b/be/test/format_v2/jni/jdbc_reader_test.cpp @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/jdbc_reader.h" + +#include + +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" + +namespace doris::format::jdbc { + +TEST(JdbcJniReaderTest, NonNullableSpecialTypeRejectsCastNull) { + auto data = ColumnString::create(); + data->insert_default(); + auto null_map = ColumnUInt8::create(); + null_map->get_data().push_back(1); + auto result = ColumnNullable::create(std::move(data), std::move(null_map)); + + EXPECT_TRUE(validate_non_nullable_special_type_result(*result, 1) + .is()); +} + +TEST(JdbcJniReaderTest, NonNullableSpecialTypeAcceptsSuccessfulCast) { + auto data = ColumnString::create(); + data->insert_data("ok", 2); + auto null_map = ColumnUInt8::create(); + null_map->get_data().push_back(0); + auto result = ColumnNullable::create(std::move(data), std::move(null_map)); + + EXPECT_TRUE(validate_non_nullable_special_type_result(*result, 1).ok()); +} + +} // namespace doris::format::jdbc diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index 7feebdca113dcc..95eda4a557e790 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -19,9 +19,12 @@ #include +#include #include #include +#include #include +#include #include #include "io/io_common.h" @@ -39,9 +42,15 @@ class FakeJniTableReader final : public JniTableReader { std::vector propagated_batch_sizes; std::vector close_results; bool next_eof = false; + std::chrono::milliseconds init_delay {0}; + std::chrono::milliseconds open_delay {0}; + std::chrono::milliseconds close_delay {0}; protected: - std::string connector_class() const override { return "test/FakeJniScanner"; } + std::string connector_class() const override { + std::this_thread::sleep_for(init_delay); + return "test/FakeJniScanner"; + } Status build_scanner_params(std::map* params) const override { params->clear(); @@ -56,6 +65,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _close_jni_scanner() override { + std::this_thread::sleep_for(close_delay); if (!TEST_scanner_opened()) { return Status::OK(); } @@ -79,6 +89,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _open_jni_scanner() override { + std::this_thread::sleep_for(open_delay); open_batch_sizes.push_back(TEST_batch_size()); TEST_set_split_state(true, false); return Status::OK(); @@ -177,6 +188,7 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { .partition_values = {}, .conjuncts = std::nullopt, .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, .condition_cache_digest = std::nullopt, .cache = nullptr, .current_range = {}, @@ -189,5 +201,36 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { EXPECT_TRUE(reader.TEST_scanner_opened()); } +TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { + constexpr auto delay = std::chrono::milliseconds(8); + RuntimeProfile profile("JniLifecycleContainment"); + FakeJniTableReader reader; + reader.init_delay = delay; + ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok()); + ASSERT_GE(profile.get_counter("InitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.open_delay = delay; + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + ASSERT_GE(profile.get_counter("PrepareSplitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.close_delay = delay; + ASSERT_TRUE(reader.close().ok()); + EXPECT_GE(profile.get_counter("CloseTime")->value(), + std::chrono::duration_cast(delay).count()); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp index 683a47dee299cc..e8d9aec40af34a 100644 --- a/be/test/format_v2/json/json_reader_test.cpp +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -173,14 +173,21 @@ struct ReadResult { bool eof = false; size_t second_rows = 0; bool second_eof = false; + size_t document_buffer_size = 0; std::vector schema; }; ReadResult read_once(const std::string& file_name, const std::string& content, TFileScanRangeParams params, const std::vector& slots, - const std::vector& requested_local_ids, bool read_twice = false) { + const std::vector& requested_local_ids, bool read_twice = false, + bool is_hive_table = false) { const auto file_path = write_json_file(file_name, content); auto range = file_range(file_path); + if (is_hive_table) { + TTableFormatFileDesc table_format; + table_format.__set_table_format_type("hive"); + range.__set_table_format_params(std::move(table_format)); + } auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; @@ -210,6 +217,7 @@ ReadResult read_once(const std::string& file_name, const std::string& content, result.block = make_block(result.schema, requested_local_ids); result.status = reader.get_block(&result.block, &result.rows, &result.eof); + result.document_buffer_size = reader.TEST_document_buffer_size(); if (result.status.ok() && read_twice) { auto eof_block = make_block(result.schema, requested_local_ids); result.second_status = @@ -318,6 +326,20 @@ TEST(JsonReaderTest, ReadsRequestedColumnsInFileScanRequestOrder) { ASSERT_TRUE(result.second_status.ok()) << result.second_status.to_string(); EXPECT_EQ(result.second_rows, 0); EXPECT_TRUE(result.second_eof); + EXPECT_EQ(result.document_buffer_size, 0); +} + +TEST(JsonReaderTest, HiveColumnLookupIsCaseInsensitiveWithoutNormalizedKeys) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("hive_case.jsonl", + R"({"ID":7,"NaMe":"alice"})" + "\n", + json_scan_params(), slots, {0, 1}, false, true); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 7); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "alice"); } TEST(JsonReaderTest, ReadsSingleDocumentOuterArray) { diff --git a/be/test/format_v2/native/native_reader_test.cpp b/be/test/format_v2/native/native_reader_test.cpp index aaa7aa90e0681e..2745ecf852c38b 100644 --- a/be/test/format_v2/native/native_reader_test.cpp +++ b/be/test/format_v2/native/native_reader_test.cpp @@ -295,7 +295,7 @@ TEST(NativeV2ReaderTest, RejectsInvalidHeaderAndEmptyFile) { static_cast(io::global_local_filesystem()->delete_file(empty_path)); } -TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { +TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndReportsHeaderOnlyFileAsEmpty) { std::filesystem::create_directories("./log"); RuntimeState state; RuntimeProfile profile("native_v2_reader_header_boundary_test"); @@ -322,7 +322,8 @@ TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { auto header_only_reader = create_reader(header_only_path, &state, &profile); ASSERT_TRUE(header_only_reader->init(&state).ok()); std::vector schema; - EXPECT_FALSE(header_only_reader->get_schema(&schema).ok()); + const auto header_only_status = header_only_reader->get_schema(&schema); + EXPECT_TRUE(header_only_status.is()) << header_only_status; static_cast(io::global_local_filesystem()->delete_file(header_only_path)); } @@ -350,6 +351,34 @@ TEST(NativeV2ReaderTest, RejectsTruncatedBlockDuringSchemaProbe) { static_cast(io::global_local_filesystem()->delete_file(path)); } +TEST(NativeV2ReaderTest, RejectsPartialBlockLengthAsCorruption) { + std::filesystem::create_directories("./log"); + RuntimeState state; + RuntimeProfile profile("native_v2_reader_partial_length_test"); + + std::string header; + header.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + header.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + + for (size_t prefix_bytes = 1; prefix_bytes < sizeof(uint64_t); ++prefix_bytes) { + const auto path = "./log/native_v2_partial_length_" + std::to_string(prefix_bytes) + "_" + + UniqueId::gen_uid().to_string() + ".native"; + auto content = header; + content.append(prefix_bytes, '\0'); + ASSERT_TRUE(write_file(path, content).ok()); + + auto reader = create_reader(path, &state, &profile); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + const auto status = reader->get_schema(&schema); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("truncated native block length"), std::string::npos); + static_cast(io::global_local_filesystem()->delete_file(path)); + } +} + TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { std::filesystem::create_directories("./log"); RuntimeState state; @@ -374,7 +403,9 @@ TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { auto zero_len_reader = create_reader(zero_len_path, &state, &profile); ASSERT_TRUE(zero_len_reader->init(&state).ok()); std::vector schema; - EXPECT_FALSE(zero_len_reader->get_schema(&schema).ok()); + const auto zero_len_status = zero_len_reader->get_schema(&schema); + EXPECT_TRUE(zero_len_status.is()) << zero_len_status; + EXPECT_NE(zero_len_status.to_string().find("zero-length native block"), std::string::npos); static_cast(io::global_local_filesystem()->delete_file(zero_len_path)); const auto invalid_pblock_path = diff --git a/be/test/format_v2/orc/orc_file_input_stream_test.cpp b/be/test/format_v2/orc/orc_file_input_stream_test.cpp index 54c89505edc3f5..8d63a4ab21ae65 100644 --- a/be/test/format_v2/orc/orc_file_input_stream_test.cpp +++ b/be/test/format_v2/orc/orc_file_input_stream_test.cpp @@ -329,14 +329,49 @@ TEST(OrcFileInputStreamTest, PublishesClusterProfileExactlyOnce) { input.beforeReadStripe( std::make_unique(200, std::vector {}), selected_columns({}), next_streams); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); } - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); +} + +TEST(OrcFileInputStreamTest, MergedIoChildrenStayIsolatedInBothInitializationOrders) { + for (const bool generic_first : {false, true}) { + auto reader = std::make_shared(256); + RuntimeProfile profile(generic_first ? "generic_first" : "orc_first"); + auto register_generic = [&] { + ADD_TIMER(&profile, "MergedSmallIO"); + return ADD_CHILD_COUNTER_WITH_LEVEL(&profile, "RequestIO", TUnit::UNIT, "MergedSmallIO", + 1); + }; + RuntimeProfile::Counter* generic_request_io = nullptr; + if (generic_first) { + generic_request_io = register_generic(); + } + { + OrcFileInputStream input("test.orc", reader, nullptr, &profile, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + if (!generic_first) { + generic_request_io = register_generic(); + } + StripeStreamMap streams; + input.beforeReadStripe( + std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + std::array data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 100); + } + ASSERT_NE(generic_request_io, nullptr); + EXPECT_EQ(generic_request_io->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 1); + } } } // namespace diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 6d41c98534eb38..63a09b04b1cced 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -62,17 +62,18 @@ #include "core/data_type/primitive_type.h" #include "core/value/timestamptz_value.h" #include "exprs/create_predicate_function.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/orc/orc_memory_stream_test.h" #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" #include "io/io_common.h" @@ -359,6 +360,21 @@ class NullableInt32GreaterThanExpr final : public VExpr { const std::string _expr_name = "NullableInt32GreaterThanExpr"; }; +class FailingRowFilterExpr final : public VExpr { +public: + FailingRowFilterExpr() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InvalidArgument("synthetic row filter failure"); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _expr_name = "FailingRowFilterExpr"; +}; + class NullableInt32LessThanExpr final : public VExpr { public: NullableInt32LessThanExpr(int column_id, int32_t value) @@ -618,8 +634,8 @@ class CompoundPredicateExpr final : public VExpr { if (_opcode == TExprOpcode::COMPOUND_NOT) { DORIS_CHECK(_children.size() == 1); ColumnPtr child_column; - RETURN_IF_ERROR(_children[0]->execute_column_impl(context, block, selector, count, - child_column)); + RETURN_IF_ERROR( + _children[0]->execute_column(context, block, selector, count, child_column)); for (size_t row = 0; row < count; ++row) { result_data[row] = !bool_value(*child_column, row); } @@ -633,8 +649,7 @@ class CompoundPredicateExpr final : public VExpr { child_columns.reserve(_children.size()); for (const auto& child : _children) { ColumnPtr child_column; - RETURN_IF_ERROR( - child->execute_column_impl(context, block, selector, count, child_column)); + RETURN_IF_ERROR(child->execute_column(context, block, selector, count, child_column)); child_columns.push_back(std::move(child_column)); } for (size_t row = 0; row < count; ++row) { @@ -688,30 +703,9 @@ class RuntimeFilterWrapperExpr final : public VExpr { VExprSPtr get_impl() const override { return _impl; } - const VExprSPtrs& children() const override { return _impl->children(); } - - TExprNodeType::type node_type() const override { return _impl->node_type(); } - - Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { - RETURN_IF_ERROR(_impl->prepare(state, desc, context)); - _prepare_finished = true; - return Status::OK(); - } - - Status open(RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope) override { - RETURN_IF_ERROR(_impl->open(state, context, scope)); - _open_finished = true; - return Status::OK(); - } - - void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override { - _impl->close(context, scope); - } - Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override { - return _impl->execute_column_impl(context, block, selector, count, result_column); + return _impl->execute_column(context, block, selector, count, result_column); } const std::string& expr_name() const override { return _expr_name; } @@ -4492,8 +4486,6 @@ format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t class NewOrcReaderTest : public testing::Test { protected: - static void SetUpTestSuite() { TimezoneUtils::load_timezones_to_cache(); } - enum class DirectInPredicateShape { ROOT, AND, @@ -4589,7 +4581,7 @@ class NewOrcReaderTest : public testing::Test { auto node = make_filter_in_node(TExprNodeType::IN_PRED); auto direct_in = VDirectInPredicate::create_shared(node, filter, true); direct_in->add_child(TableSlotRef::create_shared(0, 0, -1, schema[0].type, "id")); - VExprSPtr predicate = std::make_shared(direct_in); + VExprSPtr predicate = RuntimeFilterExpr::create_shared(node, direct_in, 0.0, false, 7); if (shape == DirectInPredicateShape::AND) { predicate = std::make_shared( TExprOpcode::COMPOUND_AND, @@ -5286,16 +5278,17 @@ TEST_F(NewOrcReaderTest, StripePrefetchPublishesMergedReadProfile) { } ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - ASSERT_NE(profile.get_counter("ApplyBytes"), nullptr); - ASSERT_NE(profile.get_counter("ClusterNum"), nullptr); - ASSERT_NE(profile.get_counter("OverReadBytes"), nullptr); - EXPECT_GT(profile.get_counter("RequestIO")->value(), profile.get_counter("MergedIO")->value()); - EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); - EXPECT_GT(profile.get_counter("ApplyBytes")->value(), 0); - EXPECT_GT(profile.get_counter("ClusterNum")->value(), 0); - EXPECT_GE(profile.get_counter("OverReadBytes")->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedApplyBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedClusterNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedOverReadBytes"), nullptr); + EXPECT_GT(profile.get_counter("OrcMergedRequestIO")->value(), + profile.get_counter("OrcMergedIO")->value()); + EXPECT_GT(profile.get_counter("OrcMergedIO")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedApplyBytes")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedClusterNum")->value(), 0); + EXPECT_GE(profile.get_counter("OrcMergedOverReadBytes")->value(), 0); } TEST_F(NewOrcReaderTest, StripePrefetchCanBeDisabledByZeroOnceMaxReadBytes) { @@ -5686,6 +5679,29 @@ TEST_F(NewOrcReaderTest, ConjunctFiltersRows) { EXPECT_EQ(values.get_data_at(2).to_string(), "five"); } +TEST_F(NewOrcReaderTest, RowFilterFailureRetainsNextBatchContext) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared())); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_NE(status.to_string().find("nextBatch failed"), std::string::npos) << status; + EXPECT_NE(status.to_string().find("synthetic row filter failure"), std::string::npos) << status; +} + TEST_F(NewOrcReaderTest, OrcLazyDecodesOnlySelectedNonPredicateRows) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -5746,8 +5762,8 @@ TEST_F(NewOrcReaderTest, ClosePublishesOrcLazyStatisticsToRuntimeProfile) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 2); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 2); } TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { @@ -5776,8 +5792,8 @@ TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); } TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSurvivingGranules) { @@ -6725,13 +6741,13 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { ASSERT_EQ(result_rows, 200); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByGroup"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - ASSERT_NE(profile.get_counter("FilteredBytes"), nullptr); - ASSERT_NE(profile.get_counter("FileNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByGroup"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcFileNum"), nullptr); const std::array orc_reader_metric_counters { "ReaderCall", "ReaderInclusiveLatencyUs", @@ -6750,19 +6766,63 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { for (const auto counter_name : orc_reader_metric_counters) { ASSERT_NE(profile.get_counter(std::string(counter_name)), nullptr) << counter_name; } - EXPECT_EQ(profile.get_counter("RowGroupsFiltered")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFiltered")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsReadNum")->value(), 1); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 1); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 3); - EXPECT_EQ(profile.get_counter("FilteredRowsByGroup")->value(), 400); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); - EXPECT_GT(profile.get_counter("FilteredBytes")->value(), 0); - EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByGroup")->value(), 400); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); + EXPECT_GT(profile.get_counter("OrcFilteredBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcFileNum")->value(), 1); EXPECT_EQ(profile.get_counter("ReadRowCount")->value(), static_cast(file_reader_stats.read_rows)); } +TEST_F(NewOrcReaderTest, ProfileKeepsPruningCountersBelowOrcReader) { + RuntimeProfile profile("new_orc_reader_hierarchy_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("OrcReader")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsFiltered")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsReadNum")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcFilteredRowsByGroup")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenParquetInitializesFirst) { + RuntimeProfile profile("parquet_then_orc_profile"); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenOrcInitializesFirst) { + RuntimeProfile profile("orc_then_parquet_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { const auto multi_stripe_file_path = (_test_dir / "profile_minmax_disabled.orc").string(); write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); @@ -6799,10 +6859,10 @@ TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { ASSERT_NE(profile.get_counter("SelectedRowGroupCount"), nullptr); ASSERT_NE(profile.get_counter("EvaluatedRowGroupCount"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 0); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 0); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 0); } TEST_F(NewOrcReaderTest, SargConjunctPrunesStripes) { @@ -7055,7 +7115,7 @@ TEST_F(NewOrcReaderTest, SargNullAwareRuntimeFilterDoesNotPruneNullStripe) { auto node = make_filter_in_node(TExprNodeType::NULL_AWARE_IN_PRED); auto direct_in = VDirectInPredicate::create_shared(node, filter, true); direct_in->add_child(TableSlotRef::create_shared(0, 0, -1, schema[0].type, "id")); - auto runtime_filter = VRuntimeFilterWrapper::create_shared(node, direct_in, 0.0, true, 7); + auto runtime_filter = RuntimeFilterExpr::create_shared(node, direct_in, 0.0, true, 7); auto runtime_filter_context = VExprContext::create_shared(std::move(runtime_filter)); ASSERT_TRUE(runtime_filter_context->prepare(&state, RowDescriptor()).ok()); ASSERT_TRUE(runtime_filter_context->open(&state).ok()); @@ -9289,6 +9349,7 @@ TEST_F(NewOrcReaderTest, SargTimestampInstantRepeatedCivilTimeDoesNotPruneStripe auto reader = create_reader_for_path(multi_stripe_file_path); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TimezoneUtils::load_timezones_to_cache(); state.set_timezone("America/New_York"); ASSERT_TRUE(reader->init(&state).ok()); diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp new file mode 100644 index 00000000000000..599c930f312856 --- /dev/null +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -0,0 +1,4146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/column_reader.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader.h" +#include "util/block_compression.h" +#include "util/coding.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/thrift_util.h" +#include "util/timezone_utils.h" + +namespace doris::format::parquet::native { +namespace { + +VExprSPtr create_int32_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t literal, + bool literal_on_left = false) { + const auto int_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + auto slot = VSlotRef::create_shared(column_id, column_id, -1, int_type, "plain_int"); + auto value = VLiteral::create_shared(int_type, Field::create_field(literal)); + if (literal_on_left) { + root->add_child(value); + root->add_child(slot); + } else { + root->add_child(slot); + root->add_child(value); + } + return root; +} + +VExprSPtr create_float_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, float literal) { + const auto float_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({float_type->to_thrift(), float_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + root->add_child(VSlotRef::create_shared(column_id, column_id, -1, float_type, "plain_float")); + root->add_child(VLiteral::create_shared(float_type, Field::create_field(literal))); + return root; +} + +TEST(ParquetV2NativeDecoderTest, RawExprEvaluatesPhysicalValuesWithoutColumn) { + const std::array values = {1, 4, 7, 10, 13, 16}; + std::array matches {}; + matches.fill(1); + const auto greater_equal = create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 7); + const auto less_than = create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 16); + const auto int_type = std::make_shared(); + + ASSERT_TRUE(greater_equal->can_execute_on_raw_fixed_values(int_type, 0)); + ASSERT_TRUE(greater_equal + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + ASSERT_TRUE(less_than + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1, 1, 0})); + + matches.fill(1); + const auto literal_less_than_slot = + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 7, true); + ASSERT_TRUE(literal_less_than_slot + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawExprPreservesFloatNanOrdering) { + const float nan = std::numeric_limits::quiet_NaN(); + const std::array values {-1, 0, 1, nan}; + const auto float_type = std::make_shared(); + + std::array matches {}; + matches.fill(1); + const auto greater_than_zero = create_float_raw_comparison(0, "gt", TExprOpcode::GT, 0); + ASSERT_TRUE(greater_than_zero + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1})); + + matches.fill(1); + const auto equals_nan = create_float_raw_comparison(0, "eq", TExprOpcode::EQ, nan); + ASSERT_TRUE(equals_nan + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawFixedFilterSupportsIdentityWidthEncodingTypes) { + using Reader = ColumnChunkReader; + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::INT32)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::INT64)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::FLOAT)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::DOUBLE)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::INT32)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::INT64)); + EXPECT_FALSE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::FLOAT)); + EXPECT_FALSE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::RLE_DICTIONARY, + tparquet::Type::INT32)); +} + +class RejectFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return Status::InternalError("Unexpected fixed dictionary"); + } +}; + +class CaptureBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + refs.assign(values, values + num_values); + return Status::OK(); + } + + std::vector refs; +}; + +class CapturePlainBinaryLayoutConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + legacy_consume_called = true; + return Status::InternalError("PLAIN BYTE_ARRAY used the legacy StringRef path"); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + base = encoded_data; + source_offsets.assign(payload_offsets, payload_offsets + num_values); + output_offsets.assign(value_offsets, value_offsets + num_values + 1); + spans = value_spans; + return Status::OK(); + } + + const char* base = nullptr; + bool legacy_consume_called = false; + std::vector source_offsets; + std::vector output_offsets; + std::vector spans; +}; + +class CaptureFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + ++consume_calls; + if (width == 0) { + width = value_width; + } + DORIS_CHECK_EQ(width, value_width); + bytes.insert(bytes.end(), values, values + num_values * value_width); + return Status::OK(); + } + + template + std::vector values() const { + DORIS_CHECK_EQ(width, sizeof(T)); + DORIS_CHECK_EQ(bytes.size() % sizeof(T), 0); + std::vector result(bytes.size() / sizeof(T)); + memcpy(result.data(), bytes.data(), bytes.size()); + return result; + } + + size_t width = 0; + size_t consume_calls = 0; + std::vector bytes; +}; + +TEST(ParquetV2NativeDecoderTest, FragmentedPlainSelectionUsesOneConsumerBatch) { + std::array input {}; + std::iota(input.begin(), input.end(), 0); + Slice data(reinterpret_cast(input.data()), input.size() * sizeof(int32_t)); + FixLengthPlainDecoder decoder; + decoder.set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder.set_data(&data).ok()); + + ParquetSelection selection { + .total_values = input.size(), .selected_values = input.size() / 2, .ranges = {}}; + for (size_t row = 0; row < input.size(); row += 2) { + selection.ranges.push_back({.first = row, .count = 1}); + } + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder.decode_selected_fixed_values(selection, consumer).ok()); + + EXPECT_EQ(consumer.consume_calls, 1); + std::vector expected; + for (int32_t value = 0; value < static_cast(input.size()); value += 2) { + expected.push_back(value); + } + EXPECT_EQ(consumer.values(), expected); +} + +class ScriptedDictionaryMaterializationSource final : public ParquetDecodeSource { +public: + ScriptedDictionaryMaterializationSource(std::vector ids, bool prefer_indices) + : _ids(std::move(ids)), _prefer_indices(prefer_indices) {} + + Status decode_fixed_values(size_t, ParquetFixedValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no fixed values"); + } + + Status decode_binary_values(size_t, ParquetBinaryValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no binary values"); + } + + Status skip_values(size_t) override { return Status::OK(); } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++index_batches; + *indices = _ids; + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++direct_batches; + return consumer.consume_indices(_ids.data(), _ids.size()); + } + + bool prefer_dictionary_index_materialization(size_t) const override { return _prefer_indices; } + + size_t direct_batches = 0; + size_t index_batches = 0; + +private: + std::vector _ids; + bool _prefer_indices; +}; + +const RowRanges& scripted_row_ranges() { + static const RowRanges ranges; + return ranges; +} + +class ScriptedColumnReader final : public ColumnReader { +public: + ScriptedColumnReader(size_t rows, size_t values, bool eof, std::vector rep_levels, + std::vector def_levels) + : ColumnReader(scripted_row_ranges(), rows, nullptr, nullptr), + _rows(rows), + _values(values), + _eof(eof), + _rep_levels(std::move(rep_levels)), + _def_levels(std::move(def_levels)) {} + + Status read_column_data(ColumnPtr& column, const DataTypePtr&, + const std::shared_ptr&, FilterMap&, size_t, + size_t* read_rows, bool* eof, bool, int64_t = -1) override { + if (_used) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + column = IColumn::mutate(std::move(column)); + column->assert_mutable()->insert_many_defaults(_values); + *read_rows = _rows; + *eof = _eof; + _used = true; + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t* read_rows, bool* eof) override { + *read_rows = _rows; + *eof = _eof; + return Status::OK(); + } + + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { return {}; } + void close() override {} + void release_batch_scratch(size_t) override {} + void reset_filter_map_index() override {} + +private: + size_t _rows; + size_t _values; + bool _eof; + bool _used = false; + std::vector _rep_levels; + std::vector _def_levels; +}; + +class MemoryBufferedReader final : public io::BufferedStreamReader { +public: + explicit MemoryBufferedReader(std::vector data) : _data(std::move(data)) {} + + Status read_bytes(const uint8_t** buf, uint64_t offset, size_t bytes_to_read, + const io::IOContext*) override { + if (offset > _data.size() || bytes_to_read > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + *buf = _data.data() + offset; + return Status::OK(); + } + Status read_bytes(Slice& slice, uint64_t offset, const io::IOContext*) override { + if (offset > _data.size() || slice.size > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + slice.data = reinterpret_cast(_data.data() + offset); + return Status::OK(); + } + std::string path() override { return "memory.parquet"; } + int64_t mtime() const override { return 0; } + +private: + std::vector _data; +}; + +class NativeDecoderMemoryFileReader final : public io::FileReader { +public: + explicit NativeDecoderMemoryFileReader(std::vector data) + : _data(std::move(data)), _path("native-decoder-memory.parquet") {} + + Status close() override { + _closed = true; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t size() const override { return _data.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _data.size() || result.size > _data.size() - offset) { + return Status::IOError("native decoder memory read exceeds file"); + } + memcpy(result.data, _data.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); + } + +private: + std::vector _data; + io::Path _path; + bool _closed = false; +}; + +std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type physical_type) { + auto node = ::parquet::schema::PrimitiveNode::Make("value", ::parquet::Repetition::REQUIRED, + physical_type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +DorisUniqueBufferPtr make_byte_array_dictionary(const std::vector& values, + int32_t* length) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + *length = static_cast(total_size); + auto dictionary = make_unique_buffer(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(dictionary.get() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(dictionary.get() + offset, value.data(), value.size()); + offset += value.size(); + } + return dictionary; +} + +std::vector encode_plain_byte_arrays(const std::vector& values) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + std::vector encoded(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(encoded.data() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(encoded.data() + offset, value.data(), value.size()); + offset += value.size(); + } + return encoded; +} + +std::vector serialize_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +Status materialize_level_only_page(bool data_page_v2, tparquet::Type::type physical_type, + tparquet::Encoding::type encoding, bool all_null) { + constexpr size_t VALUE_COUNT = 3; + const std::vector encoded_levels {static_cast(VALUE_COUNT << 1), + static_cast(all_null ? 0 : 1)}; + std::vector payload; + tparquet::PageHeader header; + if (data_page_v2) { + payload = encoded_levels; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(VALUE_COUNT); + header.data_page_header_v2.__set_num_nulls(all_null ? VALUE_COUNT : 0); + header.data_page_header_v2.__set_num_rows(VALUE_COUNT); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_definition_levels_byte_length(encoded_levels.size()); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + header.type = tparquet::PageType::DATA_PAGE; + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(VALUE_COUNT); + header.data_page_header.__set_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(VALUE_COUNT); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BOOLEAN) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.definition_level = 1; + field.parquet_schema.__set_type(physical_type); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, VALUE_COUNT, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + const auto load_status = chunk_reader.load_page_data(); + EXPECT_TRUE(load_status.ok()) << load_status; + RETURN_IF_ERROR(load_status); + + level_t level = -1; + EXPECT_EQ(chunk_reader.def_level_decoder().get_next_run(&level, VALUE_COUNT), VALUE_COUNT); + EXPECT_EQ(level, all_null ? 0 : 1); + FilterMap filter; + RETURN_IF_ERROR(filter.init(nullptr, VALUE_COUNT, false)); + NullMap null_map; + ColumnSelectVector select_vector; + const std::vector null_runs {static_cast(all_null ? 0 : VALUE_COUNT), + static_cast(all_null ? VALUE_COUNT : 0)}; + RETURN_IF_ERROR(select_vector.init(null_runs, VALUE_COUNT, &null_map, &filter, 0)); + auto column = field.data_type->create_column(); + ParquetDecodeContext decode_context; + static const auto utc = cctz::utc_time_zone(); + RETURN_IF_ERROR(init_decode_context_for_test(field, &utc, &decode_context)); + ParquetMaterializationState state; + state.enable_strict_mode = true; + const auto status = chunk_reader.materialize_values(column, *field.data_type->get_serde(), + decode_context, state, select_vector); + if (status.ok()) { + EXPECT_EQ(column->size(), VALUE_COUNT); + EXPECT_EQ(null_map, NullMap(VALUE_COUNT, all_null ? 1 : 0)); + } + return status; +} + +Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, + tparquet::CompressionCodec::type codec, bool preload_page_cache = false, + tparquet::Type::type physical_type = tparquet::Type::INT32) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + const size_t chunk_size = bytes.size(); + const std::string page_cache_file_key = "native-scripted-page-" + + std::to_string(static_cast(header.type)) + "-" + + std::to_string(header.compressed_page_size) + "-" + + std::to_string(header.uncompressed_page_size); + if (preload_page_cache) { + auto* page = new DataPage(bytes.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), bytes.data(), bytes.size()); + page->reset_size(bytes.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(page_cache_file_key, chunk_size, 0), page, &handle, + segment_v2::DATA_PAGE); + } + MemoryBufferedReader reader(std::move(bytes)); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(codec); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(chunk_size); + if (header.type == tparquet::PageType::DICTIONARY_PAGE) { + chunk.meta_data.__set_dictionary_page_offset(0); + chunk.meta_data.__set_data_page_offset(0); + } else { + chunk.meta_data.__set_data_page_offset(0); + } + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BYTE_ARRAY) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(preload_page_cache, page_cache_file_key); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + return chunk_reader.load_page_data(); +} + +Status load_malformed_nested_page(tparquet::PageHeader header, const std::vector& payload, + int64_t metadata_values = std::numeric_limits::max(), + level_t max_repetition_level = 1) { + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(metadata_values); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = max_repetition_level; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + std::vector rep_levels; + size_t result_rows = 0; + bool cross_page = false; + return chunk_reader.load_page_nested_rows(rep_levels, 1, &result_rows, &cross_page); +} + +Status materialize_plain_int96(const std::vector& values, + const std::vector& filter_values = {}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__set_uncompressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values.size()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page( + header, std::vector(reinterpret_cast(values.data()), + reinterpret_cast(values.data()) + + values.size() * sizeof(ParquetInt96Timestamp))); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT96); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(values.size()); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT96; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, values.size(), + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT96; + static const auto utc = cctz::utc_time_zone(); + decode_context.timezone = &utc; + ParquetMaterializationState state; + state.enable_strict_mode = true; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.empty() ? nullptr : filter_values.data(), + filter_values.size(), false)); + ColumnSelectVector select_vector; + const std::vector run_length_null_map {static_cast(values.size()), 0}; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, values.size(), nullptr, &filter, 0)); + return chunk_reader.materialize_values(column, *type.get_serde(), decode_context, state, + select_vector); +} + +template +Status materialize_selected_plain_fixed( + const std::vector& physical_values, size_t logical_values, + const std::vector& run_length_null_map, const std::vector& filter_values, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const DataType& type, MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics, bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(static_cast(logical_values)); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(PhysicalType)); + if (!payload.empty()) { + memcpy(payload.data(), physical_values.data(), payload.size()); + } + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.enable_strict_mode = strict_mode; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_plain_int32(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT32, + ParquetPhysicalType::INT32, type, column, null_map, + statistics, strict_mode, context_override); +} + +template +Status materialize_selected_plain_int64(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT64, + ParquetPhysicalType::INT64, type, column, null_map, + statistics, false, &context); +} + +template +Status materialize_selected_dictionary_fixed( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const ParquetDecodeContext* context_override = nullptr) { + std::vector dictionary_payload(dictionary.size() * sizeof(PhysicalType)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + // Real Parquet files place column chunks after the file header. Keep the dictionary offset + // non-zero because zero is the metadata sentinel for "no dictionary page". + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 4); + for (const auto id : physical_ids) { + encoder.Put(id); + } + // Parquet bit-packed dictionary runs declare groups of eight; emit the trailing padding so a + // decoder that buffers the declared run never reads beyond the synthetic page. + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 4; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_dictionary_int32( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics) { + return materialize_selected_dictionary_fixed( + dictionary, physical_ids, logical_values, run_length_null_map, filter_values, type, + column, null_map, statistics, tparquet::Type::INT32, ParquetPhysicalType::INT32); +} + +template +Status materialize_selected_dictionary_int64( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_dictionary_fixed(dictionary, physical_ids, logical_values, + run_length_null_map, filter_values, type, column, + null_map, statistics, tparquet::Type::INT64, + ParquetPhysicalType::INT64, &context); +} + +Status materialize_selected_dictionary_strings(const std::vector& dictionary, + const std::vector& physical_ids, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics) { + auto dictionary_payload = encode_plain_byte_arrays(dictionary); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (const auto id : physical_ids) { + encoder.Put(id); + } + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 2; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeString type; + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::BYTE_ARRAY; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +TEST(ParquetV2NativeDecoderTest, RawExprMapsNullableSparseRowsDirectly) { + const std::vector physical_values {1, 4, 7, 10, 13}; + constexpr size_t LOGICAL_VALUES = 7; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(LOGICAL_VALUES); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(int32_t)); + memcpy(payload.data(), physical_values.data(), payload.size()); + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(LOGICAL_VALUES); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, LOGICAL_VALUES, + nullptr, page_context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + + const std::vector null_runs {1, 1, 2, 1, 2}; + const std::vector input_filter {1, 0, 1, 1, 1, 0, 1}; + FilterMap filter; + ASSERT_TRUE(filter.init(input_filter.data(), input_filter.size(), false).ok()); + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector.init(null_runs, LOGICAL_VALUES, nullptr, &filter, 0).ok()); + const VExprSPtrs predicates {create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 4), + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 13)}; + NullMap selected_nulls; + IColumn::Filter physical_matches; + auto projected_column = make_nullable(field.data_type)->create_column(); + IColumn::Filter row_filter; + bool used_filter = false; + ASSERT_TRUE(chunk_reader + .filter_fixed_width_values(predicates, 0, select_vector, &selected_nulls, + &physical_matches, projected_column.get(), + &row_filter, &used_filter) + .ok()); + + EXPECT_TRUE(used_filter); + EXPECT_EQ(selected_nulls, (NullMap {0, 0, 0, 1, 0})); + EXPECT_EQ(physical_matches, (IColumn::Filter {0, 1, 1, 0})); + EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 0, 0})); + ASSERT_EQ(projected_column->size(), 2); + EXPECT_FALSE(projected_column->is_null_at(0)); + EXPECT_FALSE(projected_column->is_null_at(1)); + const auto& projected_values = + assert_cast( + assert_cast(*projected_column).get_nested_column()) + .get_data(); + EXPECT_EQ(projected_values[0], 4); + EXPECT_EQ(projected_values[1], 7); + EXPECT_EQ(chunk_reader.remaining_num_values(), 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainPrimitiveSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 0, 500, 0, 1000, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeDecimal32 type(9, 0); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 9, + .decimal_scale = 0}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + const auto& values = assert_cast(*column).get_data(); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values[0].value, 0); + EXPECT_EQ(values[1].value, 500); + EXPECT_EQ(values[2].value, 0); + EXPECT_EQ(values[3].value, 1000); + EXPECT_EQ(values[4].value, 0); + EXPECT_EQ(null_map, (NullMap {0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateSelectionBatchesPhysicalPayload) { + const std::vector physical_days {0, 1, 2, 3, 4}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateV2 type; + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_days, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-04"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhysicalPayload) { + const std::vector physical_micros {0, 1'000'000, 2'000'000, 3'000'000, 4'000'000}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS}; + ASSERT_TRUE(materialize_selected_plain_int64(physical_micros, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000000"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-01 00:00:03.000000"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NegativeNanosFloorAcrossPlainAndDictionaryTimestamps) { + const std::vector nanos {-1, -999, -1000, -1001, 0, 1001}; + const std::vector dictionary_ids {0, 1, 2, 3, 4, 5}; + const std::vector no_nulls {static_cast(nanos.size()), 0}; + const std::vector select_all(nanos.size(), 1); + const std::vector expected { + "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999999", + "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999998", + "1970-01-01 00:00:00.000000", "1970-01-01 00:00:00.000001"}; + const ParquetDecodeContext local_context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS}; + const ParquetDecodeContext utc_context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS, + .timestamp_is_adjusted_to_utc = true}; + + for (const bool dictionary : {false, true}) { + SCOPED_TRACE(dictionary ? "dictionary" : "plain"); + DataTypeDateTimeV2 datetime_type(6); + auto datetime_column = datetime_type.create_column(); + NullMap datetime_nulls; + ColumnChunkReaderStatistics datetime_statistics; + const auto datetime_status = + dictionary ? materialize_selected_dictionary_int64( + nanos, dictionary_ids, nanos.size(), no_nulls, select_all, + datetime_type, &datetime_column, &datetime_nulls, + &datetime_statistics, local_context) + : materialize_selected_plain_int64(nanos, nanos.size(), no_nulls, + select_all, datetime_type, + &datetime_column, &datetime_nulls, + &datetime_statistics, local_context); + ASSERT_TRUE(datetime_status.ok()) << datetime_status; + ASSERT_EQ(datetime_column->size(), expected.size()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(datetime_type.to_string(*datetime_column, row), expected[row]); + } + + DataTypeTimeStampTz timestamptz_type(6); + auto timestamptz_column = timestamptz_type.create_column(); + NullMap timestamptz_nulls; + ColumnChunkReaderStatistics timestamptz_statistics; + const auto timestamptz_status = + dictionary ? materialize_selected_dictionary_int64( + nanos, dictionary_ids, nanos.size(), no_nulls, select_all, + timestamptz_type, ×tamptz_column, ×tamptz_nulls, + ×tamptz_statistics, utc_context) + : materialize_selected_plain_int64( + nanos, nanos.size(), no_nulls, select_all, timestamptz_type, + ×tamptz_column, ×tamptz_nulls, + ×tamptz_statistics, utc_context); + ASSERT_TRUE(timestamptz_status.ok()) << timestamptz_status; + const auto& tz_column = assert_cast(*timestamptz_column); + const auto utc = cctz::utc_time_zone(); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(tz_column.get_element(row).to_string(utc, 6), expected[row] + "+00:00"); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector dictionary(16); + std::iota(dictionary.begin(), dictionary.end(), 100); + // One value followed by nineteen NULLs exercises the sparse-null threshold (<10%). + std::vector null_runs; + for (size_t row = 0; row < LOGICAL_VALUES;) { + null_runs.push_back(1); + ++row; + const auto null_count = std::min(19, LOGICAL_VALUES - row); + null_runs.push_back(cast_set(null_count)); + row += null_count; + } + std::vector physical_ids((LOGICAL_VALUES + 19) / 20); + for (size_t index = 0; index < physical_ids.size(); ++index) { + physical_ids[index] = static_cast(index % dictionary.size()); + } + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + const auto status = materialize_selected_dictionary_int32( + dictionary, physical_ids, LOGICAL_VALUES, null_runs, filter, type, &column, &null_map, + &statistics); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 100, 102, 0, 104, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationSkipsFailureScanWhenDictionaryIsClean) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + auto& dictionary = assert_cast(*state.typed_dictionary).get_data(); + dictionary = {10, 20, 30, 40}; + state.dictionary_indices = {3, 0, 2, 1, 3}; + state.dictionary_conversion_failures.resize_fill(dictionary.size(), 0); + + auto output = ColumnInt32::create(); + ASSERT_TRUE(state.materialize_dictionary(*output).ok()); + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_failure_scan_rows, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationUsesCacheAwareExecutionShape) { + auto verify_strategy = [](bool prefer_indices, + ParquetDictionaryMaterializationStrategy expected_strategy, + size_t expected_direct_batches, size_t expected_index_batches) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + assert_cast(*state.typed_dictionary).get_data() = {10, 20, 30, 40}; + ScriptedDictionaryMaterializationSource source({3, 0, 2, 1, 3}, prefer_indices); + auto output = ColumnInt32::create(); + + const auto status = state.materialize_dictionary(*output, source, 5); + EXPECT_TRUE(status.ok()) << status; + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_materialization_strategy, expected_strategy); + EXPECT_EQ(source.direct_batches, expected_direct_batches); + EXPECT_EQ(source.index_batches, expected_index_batches); + }; + + verify_strategy(false, ParquetDictionaryMaterializationStrategy::DIRECT, 1, 0); + verify_strategy(true, ParquetDictionaryMaterializationStrategy::INDICES, 0, 1); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryProbeMaterializesTypedValuesOnlyOnce) { + const std::array dictionary {10, 20}; + std::vector dictionary_payload(sizeof(dictionary)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values(dictionary.size()); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + const auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (const uint32_t id : {1U, 0U, 0U, 0U, 0U, 0U, 0U, 0U}) { + encoder.Put(id); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 1; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(2); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(data_page_offset); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + auto file = std::make_shared(bytes); + const auto row_ranges = ::doris::RowRanges::create_single(2); + ScalarColumnReader reader(row_ranges, 2, chunk, nullptr, nullptr, nullptr); + ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) + .ok()); + auto dictionary_result = reader.dictionary_values(field.data_type); + ASSERT_TRUE(dictionary_result.has_value()) << dictionary_result.error(); + EXPECT_EQ(reader.dictionary_materialization_count_for_test(), 1); + + FilterMap filter; + ASSERT_TRUE(filter.init(nullptr, 2, false).ok()); + ColumnPtr ids = ColumnInt32::create(); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader.read_column_data(ids, field.data_type, nullptr, filter, 2, &rows, &eof, true) + .ok()); + ASSERT_EQ(rows, 2); + auto matched_values = reader.materialize_dictionary_values( + &assert_cast(*ids), field.data_type); + ASSERT_TRUE(matched_values.has_value()) << matched_values.error(); + EXPECT_EQ(reader.dictionary_materialization_count_for_test(), 1); + EXPECT_EQ(assert_cast(**matched_values).get_data(), + (ColumnInt32::Container {20, 10})); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryRepeatedRunsGatherDirectlyIntoDestination) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (size_t row = 0; row < 64; ++row) { + encoder.Put(1); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 1; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 64, state).ok()); + EXPECT_EQ(state.dictionary_materialization_strategy, + ParquetDictionaryMaterializationStrategy::DIRECT); + ASSERT_EQ(output->size(), 64); + for (size_t row = 0; row < output->size(); ++row) { + EXPECT_EQ(assert_cast(*output).get_element(row), 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DictionaryDirectGatherRollsBackLateCorruptRun) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (size_t row = 0; row < 8; ++row) { + encoder.Put(0); + } + for (size_t row = 0; row < 8; ++row) { + encoder.Put(3); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 2; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + const auto status = + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 16, state); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(output->empty()); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryIndexBoundsCheckHandlesVectorTailAndUnsignedIds) { + const std::vector valid {0, 7, 1, 6, 2, 5, 3, 4, 7, 0, 6}; + EXPECT_TRUE(native::dictionary_indices_in_bounds(valid.data(), valid.size(), 8)); + + auto invalid_tail = valid; + invalid_tail.back() = 8; + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_tail.data(), invalid_tail.size(), 8)); + + auto invalid_unsigned = valid; + invalid_unsigned[3] = std::numeric_limits::max(); + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_unsigned.data(), + invalid_unsigned.size(), 8)); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionRemapsConversionFailures) { + const std::vector dictionary {1, 1000, 2, 3}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeInt8 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(9); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_int32(dictionary, physical_ids, 7, null_runs, + filter, type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {9, 0, 0, 2, 0})); + EXPECT_EQ(null_map, (NullMap {0, 1, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionExpandsStringsInPlace) { + const std::vector dictionary {"a", "bbb", "cc", "dddd"}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeString type; + auto column = type.create_column(); + assert_cast(*column).insert_data("prefix", 6); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_strings(dictionary, physical_ids, 7, null_runs, + filter, &column, &null_map, &statistics) + .ok()); + + ASSERT_EQ(column->size(), 5); + EXPECT_EQ(column->get_data_at(0).to_string_view(), "prefix"); + EXPECT_EQ(column->get_data_at(1).to_string_view(), ""); + EXPECT_EQ(column->get_data_at(2).to_string_view(), "bbb"); + EXPECT_EQ(column->get_data_at(3).to_string_view(), "cc"); + EXPECT_EQ(column->get_data_at(4).to_string_view(), ""); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes) { + DataTypeInt32 type; + ColumnChunkReaderStatistics statistics; + + auto all_null_column = type.create_column(); + assert_cast(*all_null_column).get_data().push_back(8); + NullMap all_null_map; + all_null_map.push_back(0); + const std::vector select_alternating {1, 0, 1, 0, 1}; + ASSERT_TRUE(materialize_selected_plain_int32({}, 5, {0, 5}, select_alternating, type, + &all_null_column, &all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_null_column = type.create_column(); + assert_cast(*dictionary_all_null_column).get_data().push_back(8); + NullMap dictionary_all_null_map; + dictionary_all_null_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10}, {}, 5, {0, 5}, select_alternating, type, + &dictionary_all_null_column, + &dictionary_all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(dictionary_all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto all_filtered_column = type.create_column(); + assert_cast(*all_filtered_column).get_data().push_back(9); + NullMap all_filtered_map; + all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_plain_int32( + {10, 20, 30}, 5, {1, 1, 1, 1, 1}, std::vector(5, 0), type, + &all_filtered_column, &all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_filtered_column = type.create_column(); + assert_cast(*dictionary_all_filtered_column).get_data().push_back(9); + NullMap dictionary_all_filtered_map; + dictionary_all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10, 20, 30}, {0, 1, 2}, 5, {1, 1, 1, 1, 1}, + std::vector(5, 0), type, + &dictionary_all_filtered_column, + &dictionary_all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(dictionary_all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); + const uint8_t* dictionary_address = dictionary.get(); + ByteArrayDictDecoder decoder; + + ASSERT_TRUE(decoder.set_dict(dictionary, dictionary_length, 2).ok()); + EXPECT_EQ(dictionary.get(), nullptr); + + RejectFixedConsumer fixed_consumer; + CaptureBinaryConsumer binary_consumer; + ASSERT_TRUE(decoder.decode_dictionary(fixed_consumer, binary_consumer).ok()); + ASSERT_EQ(binary_consumer.refs.size(), 2); + EXPECT_EQ(binary_consumer.refs[0].to_string_view(), "alpha"); + EXPECT_EQ(binary_consumer.refs[1].to_string_view(), "beta"); + EXPECT_EQ(binary_consumer.refs[0].data, + reinterpret_cast(dictionary_address + sizeof(uint32_t))); + + // bit width 1, an RLE run of three values (header = 3 << 1), dictionary id 1. + char valid_indices[] = {1, 6, 1}; + Slice valid_slice(valid_indices, sizeof(valid_indices)); + ASSERT_TRUE(decoder.set_data(&valid_slice).ok()); + std::vector decoded_indices; + ASSERT_TRUE(decoder.decode_dictionary_indices(3, &decoded_indices).ok()); + EXPECT_EQ(decoded_indices, std::vector({1, 1, 1})); + + // bit width 2, one RLE value with dictionary id 3. Skipping still validates the encoded id so + // filter selection cannot hide a corrupt dictionary stream. + char invalid_indices[] = {2, 2, 3}; + Slice invalid_slice(invalid_indices, sizeof(invalid_indices)); + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetDecodeSource& source = decoder; + EXPECT_TRUE(source.skip_values(1).is()); + + // Sparse decode must validate filtered dictionary ids too. Predicate selection must never + // turn a corrupt page into a successful read merely because the bad row was not selected. + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetSelection filtered_corrupt {.total_values = 1, .selected_values = 0, .ranges = {}}; + EXPECT_TRUE(decoder.decode_selected_dictionary_indices(filtered_corrupt, &decoded_indices) + .is()); + + Slice empty_indices; + EXPECT_TRUE(decoder.set_data(&empty_indices).is()); + + // Dictionary indices are uint32_t. Wider external bit widths would make the RLE decoder copy + // a five-byte repeated value into four-byte state before it can validate an index. + char oversized_bit_width[] = {33, 2, 0}; + Slice oversized_bit_width_slice(oversized_bit_width, sizeof(oversized_bit_width)); + EXPECT_TRUE(decoder.set_data(&oversized_bit_width_slice).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryBoundsEntryCountBeforeReserve) { + auto dictionary = make_unique_buffer(1); + dictionary.get()[0] = 0; + ByteArrayDictDecoder decoder; + EXPECT_TRUE(decoder.set_dict(dictionary, 1, std::numeric_limits::max()) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { + TimezoneUtils::load_timezones_to_cache(); + for (const auto converted_type : + {tparquet::ConvertedType::TIMESTAMP_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS}) { + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT64; + field.parquet_schema.__set_converted_type(converted_type); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.logical_type, ParquetLogicalType::TIMESTAMP); + EXPECT_TRUE(context.timestamp_is_adjusted_to_utc); + + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + ASSERT_TRUE(init_decode_context_for_test(field, &shanghai, &context).ok()); + int64_t epoch = 0; + Slice epoch_slice(reinterpret_cast(&epoch), sizeof(epoch)); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(sizeof(epoch)); + ASSERT_TRUE(decoder->set_data(&epoch_slice).ok()); + ParquetMaterializationState state; + DataTypeDateTimeV2 type(0); + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, *decoder, context, 1, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 08:00:00"); + } +} + +TEST(ParquetV2NativeDecoderTest, GeospatialByteArrayAnnotationsDecodeAsRawBytes) { + const std::string wkb("\x01\x01\x00\x00\x00", 5); + for (const bool geometry : {true, false}) { + tparquet::LogicalType logical; + if (geometry) { + logical.__set_GEOMETRY(tparquet::GeometryType()); + } else { + logical.__set_GEOGRAPHY(tparquet::GeographyType()); + } + + NativeFieldSchema field; + field.name = geometry ? "geometry" : "geography"; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.physical_type, ParquetPhysicalType::BYTE_ARRAY); + EXPECT_EQ(context.logical_type, ParquetLogicalType::NONE); + + DataTypeString type; + auto plain_column = type.create_column(); + auto encoded = encode_plain_byte_arrays({wkb}); + Slice plain_slice(encoded.data(), encoded.size()); + std::unique_ptr plain_decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, + plain_decoder) + .ok()); + ASSERT_TRUE(plain_decoder->set_data(&plain_slice).ok()); + ParquetMaterializationState plain_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*plain_column, *plain_decoder, context, 1, + plain_state) + .ok()); + ASSERT_EQ(plain_column->size(), 1); + EXPECT_EQ(plain_column->get_data_at(0).to_string_view(), wkb); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({wkb, "unused"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 2).ok()); + char dictionary_index[] = {1, 2, 0}; + Slice dictionary_slice(dictionary_index, sizeof(dictionary_index)); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + context.encoding = ParquetValueEncoding::DICTIONARY; + auto dictionary_column = type.create_column(); + ParquetMaterializationState dictionary_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, dictionary_decoder, + context, 1, dictionary_state) + .ok()); + ASSERT_EQ(dictionary_column->size(), 1); + EXPECT_EQ(dictionary_column->get_data_at(0).to_string_view(), wkb); + + field.physical_type = tparquet::Type::INT32; + EXPECT_TRUE( + init_decode_context_for_test(field, nullptr, &context).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, InvalidLogicalPhysicalPairsFailBeforeDecode) { + auto invalid = [](tparquet::Type::type physical, const tparquet::LogicalType& logical) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + + tparquet::LogicalType date; + date.__set_DATE(tparquet::DateType()); + EXPECT_FALSE(invalid(tparquet::Type::BOOLEAN, date).ok()); + + tparquet::LogicalType timestamp; + timestamp.__set_TIMESTAMP(tparquet::TimestampType()); + timestamp.TIMESTAMP.__set_unit(tparquet::TimeUnit()); + timestamp.TIMESTAMP.unit.__set_MICROS(tparquet::MicroSeconds()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, timestamp).ok()); + + tparquet::LogicalType integer; + integer.__set_INTEGER(tparquet::IntType()); + integer.INTEGER.__set_bitWidth(64); + integer.INTEGER.__set_isSigned(true); + EXPECT_FALSE(invalid(tparquet::Type::INT32, integer).ok()); + + tparquet::LogicalType string; + string.__set_STRING(tparquet::StringType()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, string).ok()); + + tparquet::LogicalType uuid; + uuid.__set_UUID(tparquet::UUIDType()); + EXPECT_FALSE(invalid(tparquet::Type::BYTE_ARRAY, uuid).ok()); + + auto invalid_converted = [](tparquet::Type::type physical, + tparquet::ConvertedType::type converted, int type_length = -1, + int precision = -1, int scale = -1) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_converted_type(converted); + if (type_length >= 0) { + field.parquet_schema.__set_type_length(type_length); + } + if (precision >= 0) { + field.parquet_schema.__set_precision(precision); + } + if (scale >= 0) { + field.parquet_schema.__set_scale(scale); + } + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UTF8).ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::TIME_MICROS).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UINT_64).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::ConvertedType::INTERVAL, 8) + .ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::DECIMAL, -1, 10, 2) + .ok()); + + NativeFieldSchema fixed; + fixed.name = "fixed"; + fixed.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; + ParquetDecodeContext context; + EXPECT_FALSE(init_decode_context_for_test(fixed, nullptr, &context).ok()); + fixed.parquet_schema.__set_type_length(4); + EXPECT_TRUE(init_decode_context_for_test(fixed, nullptr, &context).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema int96_field; + int96_field.physical_type = tparquet::Type::INT96; + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, false)); + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, true)); + + NativeFieldSchema utc_field; + utc_field.physical_type = tparquet::Type::INT64; + utc_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + utc_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + utc_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(utc_field, datetime_type, false)); + + NativeFieldSchema converted_utc_field; + converted_utc_field.physical_type = tparquet::Type::INT64; + converted_utc_field.parquet_schema.__set_converted_type( + tparquet::ConvertedType::TIMESTAMP_MICROS); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(converted_utc_field, datetime_type, + false)); + + NativeFieldSchema local_field = utc_field; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + EXPECT_FALSE( + preserves_timestamp_conversion_default_for_test(local_field, datetime_type, false)); + + NativeFieldSchema integer_field; + integer_field.physical_type = tparquet::Type::INT64; + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test( + integer_field, make_nullable(std::make_shared()), false)); +} + +TEST(ParquetV2NativeDecoderTest, FastInt96NormalizesLegacyOutOfDayNanos) { + EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema local_field; + local_field.physical_type = tparquet::Type::INT64; + local_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + local_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + const uint64_t invalid_datetime = 0; + column->insert_data(reinterpret_cast(&invalid_datetime), sizeof(invalid_datetime)); + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 1); + + null_map[0] = 0; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { + const ParquetSelection selection { + .total_values = 7, + .selected_values = 3, + .ranges = {{.first = 1, .count = 2}, {.first = 6, .count = 1}}}; + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {10, 11, 12, 13, 14, 15, 16, 17}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({11, 12, 16})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({17})); + + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded_strings = encode_plain_byte_arrays(strings); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + Slice string_slice(encoded_strings.data(), encoded_strings.size()); + ASSERT_TRUE(decoder->set_data(&string_slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 3); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "one"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "two"); + EXPECT_EQ(selected_strings.refs[2].to_string_view(), "six"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "seven"); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char booleans[] = {static_cast(0b10001101)}; + Slice boolean_slice(booleans, sizeof(booleans)); + ASSERT_TRUE(decoder->set_data(&boolean_slice).ok()); + CaptureFixedConsumer selected_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_booleans).ok()); + EXPECT_EQ(selected_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_boolean).ok()); + EXPECT_EQ(trailing_boolean.values(), std::vector({1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_booleans[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_booleans, sizeof(rle_booleans)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + CaptureFixedConsumer selected_rle_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_rle_booleans).ok()); + EXPECT_EQ(selected_rle_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_rle_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_rle_boolean).ok()); + EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); +} + +TEST(ParquetV2NativeDecoderTest, PlainByteArrayPublishesOffsetsAndCoalescedSelectionSpans) { + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded = encode_plain_byte_arrays(strings); + Slice slice(encoded.data(), encoded.size()); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + + const ParquetSelection selection { + .total_values = 7, + .selected_values = 4, + .ranges = {{.first = 1, .count = 2}, {.first = 5, .count = 2}}}; + CapturePlainBinaryLayoutConsumer consumer; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, consumer).ok()); + EXPECT_FALSE(consumer.legacy_consume_called); + EXPECT_EQ(consumer.output_offsets, std::vector({0, 3, 6, 10, 13})); + ASSERT_EQ(consumer.source_offsets.size(), selection.selected_values); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[0], 3), "one"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[1], 3), "two"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[2], 4), "five"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[3], 3), "six"); + ASSERT_EQ(consumer.spans.size(), 2); + EXPECT_EQ(consumer.spans[0].first, 0); + EXPECT_EQ(consumer.spans[0].count, 2); + EXPECT_EQ(consumer.spans[1].first, 2); + EXPECT_EQ(consumer.spans[1].count, 2); + + CaptureBinaryConsumer trailing; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing).ok()); + ASSERT_EQ(trailing.refs.size(), 1); + EXPECT_EQ(trailing.refs[0].to_string_view(), "seven"); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainFixedDecodeDoesNotRetainGatherBuffer) { + constexpr size_t value_count = 1UL << 18; + std::vector integers(value_count); + std::iota(integers.begin(), integers.end(), 0); + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 2, + .ranges = {{.first = 1, .count = 1}, {.first = value_count - 1, .count = 1}}}; + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), + std::vector({1, static_cast(value_count - 1)})); + // Sparse PLAIN spans can be consumed directly from the encoded page. Retaining a gather buffer + // makes a highly selective batch allocate in proportion to its selected width for no benefit. + EXPECT_EQ(decoder->retained_scratch_bytes(), 0); +} + +TEST(ParquetV2NativeDecoderTest, FixedPlainLargeSkipCannotWrapPageOffset) { + std::vector values {11, 22}; + Slice slice(reinterpret_cast(values.data()), values.size() * sizeof(int64_t)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int64_t)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(536870913).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({11})); +} + +TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {11, 22, 33}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer integer_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, integer_consumer).ok()); + EXPECT_EQ(integer_consumer.values(), std::vector({22, 33})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0b10001101)}; + Slice plain_boolean_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_boolean_slice).ok()); + CaptureFixedConsumer plain_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(8, plain_boolean_consumer).ok()); + EXPECT_EQ(plain_boolean_consumer.values(), + std::vector({1, 0, 1, 1, 0, 0, 0, 1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_boolean[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_boolean, sizeof(rle_boolean)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + ASSERT_TRUE(decoder->skip_values(3).ok()); + CaptureFixedConsumer rle_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(5, rle_boolean_consumer).ok()); + EXPECT_EQ(rle_boolean_consumer.values(), std::vector({1, 0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer first_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, first_integer).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer remaining_integers; + ASSERT_TRUE(decoder->decode_fixed_values(2, remaining_integers).ok()); + EXPECT_EQ(first_integer.values(), std::vector({100})); + EXPECT_EQ(remaining_integers.values(), std::vector({99, 1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(3, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 3); + EXPECT_EQ(consumer.refs[0].to_string_view(), "prefix-b"); + EXPECT_EQ(consumer.refs[1].to_string_view(), "other"); + EXPECT_EQ(consumer.refs[2].to_string_view(), "other-tail"); + } +} + +TEST(ParquetV2NativeDecoderTest, DeltaBinaryPackedRejectsNonIntegralBlockGeometry) { + std::vector encoded(32); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, 3200); + cursor = encode_varint32(cursor, 33); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + Slice slice(encoded.data(), encoded.size()); + EXPECT_FALSE(decoder->set_data(&slice).ok()); +} + +TEST(ParquetV2NativeDecoderTest, SparseStatefulEncodingsBatchDecodeAndCompact) { + const ParquetSelection selection { + .total_values = 3, + .selected_values = 2, + .ranges = {{.first = 0, .count = 1}, {.first = 2, .count = 1}}}; + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({100, 99})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 2); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "prefix-a"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "other"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "other-tail"); + } + + const std::vector floats {1.0F, -2.5F, 3.25F, 9.5F}; + std::vector encoded_floats(floats.size() * sizeof(float)); + for (size_t row = 0; row < floats.size(); ++row) { + const auto* bytes = reinterpret_cast(&floats[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded_floats[byte * floats.size() + row] = bytes[byte]; + } + } + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice float_slice(encoded_floats.data(), encoded_floats.size()); + ASSERT_TRUE(decoder->set_data(&float_slice).ok()); + CaptureFixedConsumer selected_floats; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_floats).ok()); + EXPECT_EQ(selected_floats.values(), std::vector({1.0F, 3.25F})); + CaptureFixedConsumer trailing_float; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_float).ok()); + EXPECT_EQ(trailing_float.values(), std::vector({9.5F})); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { + const std::vector values {1.0F, -2.5F, 3.25F}; + std::vector encoded(values.size() * sizeof(float)); + for (size_t row = 0; row < values.size(); ++row) { + const auto* bytes = reinterpret_cast(&values[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded[byte * values.size() + row] = bytes[byte]; + } + } + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) { + char encoded[] = {static_cast(0b00001101)}; + Slice levels(encoded, sizeof(encoded)); + LevelDecoder decoder; + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 1, 4).ok()); + + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 4), 1); + EXPECT_EQ(value, 1); + EXPECT_EQ(decoder.get_next(), 0); + decoder.rewind_one(); + EXPECT_EQ(decoder.get_next(), 0); + level_t tail[2] = {-1, -1}; + EXPECT_EQ(decoder.get_levels(tail, 2), 2); + EXPECT_EQ(tail[0], 1); + EXPECT_EQ(tail[1], 1); + + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice truncated_levels(truncated_rle, sizeof(truncated_rle)); + LevelDecoder rle_decoder; + ASSERT_TRUE(rle_decoder.init(&truncated_levels, tparquet::Encoding::RLE, 1, 1).ok()); + level_t truncated_value = -1; + EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelByteCountDoesNotWrapAtLargeCounts) { + char placeholder[8] = {}; + constexpr uint32_t num_levels = 1'500'000'000; + constexpr size_t expected_bytes = 562'500'000; + Slice levels(placeholder, expected_bytes); + LevelDecoder decoder; + + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 4, num_levels).ok()); + EXPECT_EQ(levels.size, 0); +} + +TEST(ParquetV2NativeDecoderTest, RejectsLevelsAboveSchemaMaximumOnEveryDecodePath) { + auto make_decoder = [](tparquet::Encoding::type encoding) { + static char encoded[] = {0x03}; // width=2 value=3, while the schema maximum is 2. + static char rle_encoded[] = {0x02, 0x00, 0x00, 0x00, 0x02, 0x03}; + Slice levels = encoding == tparquet::Encoding::BIT_PACKED + ? Slice(encoded, sizeof(encoded)) + : Slice(rle_encoded, sizeof(rle_encoded)); + LevelDecoder decoder; + EXPECT_TRUE(decoder.init(&levels, encoding, 2, 1).ok()); + return decoder; + }; + + for (auto encoding : {tparquet::Encoding::BIT_PACKED, tparquet::Encoding::RLE}) { + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_levels(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + EXPECT_EQ(decoder.get_next(), -1); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NestedReadersRejectRleAndBitPackedLevelsAboveMaximum) { + for (auto encoding : {tparquet::Encoding::RLE, tparquet::Encoding::BIT_PACKED}) { + const std::vector payload = encoding == tparquet::Encoding::RLE + ? std::vector {2, 0, 0, 0, 2, 3} + : std::vector {3}; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + + EXPECT_TRUE(load_malformed_nested_page(header, payload, 1, 2).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0xFF)}; + Slice plain_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_slice).ok()); + EXPECT_FALSE(decoder->skip_values(9).ok()); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + // A bit-packed run header for eight values without its required payload byte. + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice rle_slice(truncated_rle, sizeof(truncated_rle)); + ASSERT_TRUE(decoder->set_data(&rle_slice).ok()); + EXPECT_FALSE(decoder->skip_values(1).ok()); + + // The first literal group is complete but the second has only its header. Exact-count checks + // must reject both dense and selected decodes instead of exposing the retained vector tail. + char partially_truncated_rle[] = {0x03, 0x00, 0x00, 0x00, 0x03, static_cast(0xFF), 0x03}; + Slice partially_truncated_slice(partially_truncated_rle, sizeof(partially_truncated_rle)); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer dense_consumer; + EXPECT_FALSE(decoder->decode_fixed_values(9, dense_consumer).ok()); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_tail { + .total_values = 9, .selected_values = 1, .ranges = {{.first = 8, .count = 1}}}; + EXPECT_FALSE(decoder->decode_selected_fixed_values(select_tail, selected_consumer).ok()); +} + +TEST(ParquetV2NativeDecoderTest, CompactRleSkipsKeepScratchBounded) { + constexpr uint32_t value_count = 1U << 20; + uint8_t header[16]; + uint8_t* header_end = encode_varint32(header, value_count << 1); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"only"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 1).ok()); + std::vector dictionary_indices {0}; + dictionary_indices.insert(dictionary_indices.end(), reinterpret_cast(header), + reinterpret_cast(header_end)); + Slice dictionary_slice(dictionary_indices.data(), dictionary_indices.size()); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + ParquetDecodeSource& dictionary_source = dictionary_decoder; + ASSERT_TRUE(dictionary_source.skip_values(value_count).ok()); + EXPECT_LE(dictionary_decoder.retained_scratch_bytes(), 4096 * sizeof(uint32_t)); + + std::vector boolean_payload(reinterpret_cast(header), + reinterpret_cast(header_end)); + boolean_payload.push_back(0); + std::vector boolean_page(sizeof(uint32_t) + boolean_payload.size()); + encode_fixed32_le(reinterpret_cast(boolean_page.data()), boolean_payload.size()); + memcpy(boolean_page.data() + sizeof(uint32_t), boolean_payload.data(), boolean_payload.size()); + Slice boolean_slice(boolean_page.data(), boolean_page.size()); + std::unique_ptr boolean_decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, boolean_decoder) + .ok()); + ASSERT_TRUE(boolean_decoder->set_data(&boolean_slice).ok()); + ASSERT_TRUE(boolean_decoder->skip_values(value_count).ok()); + EXPECT_LE(boolean_decoder->retained_scratch_bytes(), 4096); +} + +TEST(ParquetV2NativeDecoderTest, CompactDeltaSkipKeepsScratchBounded) { + constexpr uint32_t delta_count = 1U << 20; + std::vector encoded(64); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, delta_count); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, delta_count + 1); + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta + *cursor++ = 0; // zero-width miniblock + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(delta_count + 1); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(delta_count + 1).ok()); + EXPECT_LE(decoder->retained_scratch_bytes(), 4096 * sizeof(int32_t) + 1); +} + +TEST(ParquetV2NativeDecoderTest, DeltaPaddingBitCountIsWidenedBeforeCursorAdvance) { + int64_t padding_bits = 0; + ASSERT_TRUE(detail::checked_delta_padding_bits(64, std::numeric_limits::max(), + &padding_bits) + .ok()); + EXPECT_EQ(padding_bits, 64LL * std::numeric_limits::max()); + EXPECT_GT(padding_bits, std::numeric_limits::max()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArraySkipsKeepScratchBounded) { + constexpr size_t value_count = 1U << 16; + std::vector<::parquet::ByteArray> values(value_count); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + static_cast(encoding), decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(value_count).ok()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayLongPrefixSparseWorkIsByteBounded) { + constexpr size_t value_count = 128; + const std::string repeated_value(256UL << 10, 'x'); + std::vector<::parquet::ByteArray> values( + value_count, + ::parquet::ByteArray(static_cast(repeated_value.size()), + reinterpret_cast(repeated_value.data()))); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + auto make_decoder = [&]() { + std::unique_ptr decoder; + EXPECT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + EXPECT_TRUE(decoder->set_data(&slice).ok()); + return decoder; + }; + + auto skipped = make_decoder(); + ASSERT_TRUE(skipped->skip_values(value_count).ok()); + EXPECT_LT(skipped->retained_scratch_bytes(), 8UL << 20); + + auto sparse = make_decoder(); + CaptureBinaryConsumer consumer; + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 1, + .ranges = {{.first = value_count - 1, .count = 1}}, + }; + ASSERT_TRUE(sparse->decode_selected_binary_values(selection, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 1); + EXPECT_EQ(consumer.refs[0].to_string_view(), repeated_value); + EXPECT_LT(sparse->retained_scratch_bytes(), 8UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValues) { + const std::vector values {"good", "bad"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_type_length(4); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_good { + .total_values = 2, .selected_values = 1, .ranges = {{.first = 0, .count = 1}}}; + EXPECT_TRUE(decoder->decode_selected_fixed_values(select_good, selected_consumer) + .is()); + + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_TRUE(decoder->skip_values(2).is()); +} + +TEST(ParquetV2NativeDecoderTest, TruncatedFixedWidthPlainKeepsPublicErrorKeyword) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(4); + char truncated[] = {'a', 'b', 'c'}; + Slice slice(truncated, sizeof(truncated)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + const auto status = decoder->decode_fixed_values(1, consumer); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unexpected end of stream"), std::string::npos); +} + +TEST(ParquetV2NativeDecoderTest, DeltaSkipRequiresTheRequestedValueCount) { + const std::vector integers {7}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(integers.data(), static_cast(integers.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(2).ok()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaHeadersAndLengthsAreBoundedBeforeAllocation) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + // block_size=128, miniblocks=4, total_values=INT_MAX, first_value=0. + char oversized_count[] = {static_cast(0x80), + 0x01, + 0x04, + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + 0x07, + 0x00}; + Slice oversized_count_slice(oversized_count, sizeof(oversized_count)); + EXPECT_TRUE(decoder->set_data(&oversized_count_slice).is()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + // A single decoded length of 1 GiB with no following payload bytes must fail before the + // decoder resizes its backing data buffer. + char oversized_length[] = {static_cast(0x80), + 0x01, + 0x04, + 0x01, + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + 0x08}; + Slice oversized_length_slice(oversized_length, sizeof(oversized_length)); + CaptureBinaryConsumer consumer; + EXPECT_TRUE(decoder->set_data(&oversized_length_slice).is()); + EXPECT_TRUE(consumer.refs.empty()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + std::vector huge_prefix(64); + uint8_t* cursor = huge_prefix.data(); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, std::numeric_limits::max() - 1); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); // one empty suffix + huge_prefix.resize(cursor - huge_prefix.data()); + Slice huge_prefix_slice(huge_prefix.data(), huge_prefix.size()); + ASSERT_TRUE(decoder->set_data(&huge_prefix_slice).ok()); + EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDeltaLengthPageResetsDecoderState) { + const std::vector strings {"old", "state"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + Slice valid(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&valid).ok()); + Slice empty; + EXPECT_TRUE(decoder->set_data(&empty).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRejectsPartialRowsAtPageBoundary) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + char partial_row[] = {0, 1, 2, 3, 4}; + Slice slice(partial_row, sizeof(partial_row)); + EXPECT_TRUE(decoder->set_data(&slice).is()); +} + +Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t value_rows, + size_t key_values, size_t value_values, + std::vector key_rep_levels, std::vector value_rep_levels, + bool value_eof, std::vector key_def_levels = {}, + std::vector value_def_levels = {}) { + auto value_type = make_nullable(std::make_shared()); + auto map_type = std::make_shared(key_type, value_type); + ColumnPtr column = map_type->create_column(); + NativeFieldSchema field; + field.name = "m"; + field.data_type = map_type; + field.definition_level = 1; + field.repetition_level = 1; + field.repeated_parent_def_level = 0; + + if (key_def_levels.empty()) { + key_def_levels.assign(key_rep_levels.size(), 1); + } + if (value_def_levels.empty()) { + value_def_levels.assign(value_rep_levels.size(), 1); + } + auto key_reader = std::make_unique( + key_rows, key_values, true, std::move(key_rep_levels), std::move(key_def_levels)); + auto value_reader = std::make_unique(value_rows, value_values, value_eof, + std::move(value_rep_levels), + std::move(value_def_levels)); + MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); + RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); + + auto root = std::make_shared(std::make_shared(), + std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + return reader.read_column_data(column, map_type, root, filter, key_rows, &read_rows, &eof, + false); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderUsesKeyShapeForNestedValues) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 2, 1}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryDistribution) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 4, 4, {0, 1, 0, 1}, {0, 1, 1, 0}, true) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryPresence) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 2, 2, {0, 0}, {0, 0}, true, {0, 1}, {1, 0}) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { + auto int_type = std::make_shared(); + EXPECT_TRUE( + read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderPreservesNullableKeysWrittenByDoris) { + auto nullable_key = make_nullable(std::make_shared()); + EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 0}); + children["b"] = std::make_unique(1, 1, true, std::vector {0}, + std::vector {0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.repetition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique( + 2, 3, true, std::vector {0, 1, 0}, std::vector {0, 0, 0}); + children["b"] = std::make_unique( + 2, 3, true, std::vector {0, 0, 1}, std::vector {0, 0, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedOptionalParentPresence) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.definition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 1}); + children["b"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {1, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { + constexpr size_t value_count = 1UL << 20; + std::vector encoded(value_count * sizeof(float), 0); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(value_count, consumer).ok()); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + EXPECT_EQ(decoder->active_scratch_bytes(), value_count * sizeof(float)); + + std::vector ordinary_encoded(sizeof(float), 0); + Slice ordinary_slice(ordinary_encoded.data(), ordinary_encoded.size()); + ASSERT_TRUE(decoder->set_data(&ordinary_slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + // Capacity records the high-water allocation while size records this batch. The reader needs + // both to distinguish stable large batches from a one-off outlier before releasing capacity. + EXPECT_EQ(decoder->active_scratch_bytes(), sizeof(float)); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + decoder->release_scratch(64UL << 10); + EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, DecompressionScratchStaysActiveUntilPageExhaustion) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + auto compressed_page = [&](uint32_t value_count) { + std::vector encoded_levels(8); + uint8_t* level_end = encode_varint32(encoded_levels.data(), value_count << 1); + *level_end++ = 1; + encoded_levels.resize(level_end - encoded_levels.data()); + std::vector payload(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + payload.resize(payload.size() + static_cast(value_count) * sizeof(int32_t)); + + faststring compressed; + DORIS_CHECK(codec->compress(Slice(payload.data(), payload.size()), &compressed).ok()); + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(value_count); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, std::vector(compressed.data(), + compressed.data() + compressed.size())); + }; + + constexpr uint32_t LARGE_VALUE_COUNT = 1U << 20; + auto bytes = compressed_page(LARGE_VALUE_COUNT); + const auto ordinary_page = compressed_page(1); + bytes.insert(bytes.end(), ordinary_page.begin(), ordinary_page.end()); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(static_cast(LARGE_VALUE_COUNT) + 1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.definition_level = 1; + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, + static_cast(LARGE_VALUE_COUNT) + 1, nullptr, + context); + ASSERT_TRUE(reader.init().ok()); + ASSERT_TRUE(reader.load_page_data().ok()); + ASSERT_GT(reader.active_decoder_scratch_bytes(), 1UL << 20); + ASSERT_TRUE(reader.skip_values(LARGE_VALUE_COUNT).ok()); + ASSERT_TRUE(reader.next_page().ok()); + ASSERT_TRUE(reader.parse_page_header().ok()); + ASSERT_TRUE(reader.load_page_data().ok()); + EXPECT_LT(reader.active_decoder_scratch_bytes(), 64UL << 10); + const size_t retained_while_active = reader.retained_decoder_scratch_bytes(); + ASSERT_GT(retained_while_active, 1UL << 20); + reader.release_decoder_scratch(64UL << 10); + EXPECT_EQ(reader.retained_decoder_scratch_bytes(), retained_while_active); + + ASSERT_TRUE(reader.skip_values(1).ok()); + ASSERT_TRUE(reader.next_page().ok()); + // A release requested while a decoder points into the buffer must execute at the next safe + // page boundary; otherwise periodic probes can never age out a previous large-page capacity. + EXPECT_LE(reader.retained_decoder_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayScratchReleasePreservesPrefixState) { + const std::vector values {std::string(4096, 'x'), "shared-prefix-a", + "shared-prefix-b", "shared-prefix-c"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + decoder->release_scratch(64); + ASSERT_TRUE(decoder->decode_binary_values(2, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 2); + EXPECT_EQ(consumer.refs[0].to_string_view(), values[2]); + EXPECT_EQ(consumer.refs[1].to_string_view(), values[3]); +} + +TEST(ParquetV2NativeDecoderTest, DeltaBitPackScratchReleasePreservesMiniblockState) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + + std::vector outlier_header(64); + uint8_t* cursor = outlier_header.data(); + cursor = encode_varint32(cursor, 512); // values per block + cursor = encode_varint32(cursor, 16); // miniblocks per block + cursor = encode_varint32(cursor, 2); // total values + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta, zig-zag encoded + memset(cursor, 0, 16); // one zero bit width per miniblock + cursor += 16; + outlier_header.resize(cursor - outlier_header.data()); + decoder->set_expected_values(2); + Slice outlier_slice(outlier_header.data(), outlier_header.size()); + ASSERT_TRUE(decoder->set_data(&outlier_slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + + std::vector values(40); + std::iota(values.begin(), values.end(), 0); + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + decoder->release_scratch(8); + ASSERT_TRUE(decoder->decode_fixed_values(values.size() - 1, consumer).ok()); + EXPECT_EQ(consumer.values().size(), values.size() + 1); +} + +TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption) { + auto parse_header = [](tparquet::PageHeader header) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + if (header.compressed_page_size > 0) { + bytes.resize(bytes.size() + header.compressed_page_size); + } + MemoryBufferedReader reader(bytes); + tparquet::ColumnMetaData metadata; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, bytes.size(), 1, metadata, + context); + return page_reader.parse_page_header(); + }; + + tparquet::PageHeader negative; + negative.type = tparquet::PageType::DATA_PAGE; + negative.__set_compressed_page_size(-1); + negative.__set_uncompressed_page_size(1); + negative.__isset.data_page_header = true; + negative.data_page_header.__set_num_values(1); + EXPECT_TRUE(parse_header(negative).is()); + + tparquet::PageHeader oversized_levels; + oversized_levels.type = tparquet::PageType::DATA_PAGE_V2; + oversized_levels.__set_compressed_page_size(1); + oversized_levels.__set_uncompressed_page_size(1); + oversized_levels.__isset.data_page_header_v2 = true; + oversized_levels.data_page_header_v2.__set_num_values(1); + oversized_levels.data_page_header_v2.__set_num_rows(1); + oversized_levels.data_page_header_v2.__set_repetition_levels_byte_length(2); + oversized_levels.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(oversized_levels).is()); + + tparquet::PageHeader impossible_counts = oversized_levels; + impossible_counts.__set_compressed_page_size(0); + impossible_counts.__set_uncompressed_page_size(0); + impossible_counts.data_page_header_v2.__set_repetition_levels_byte_length(0); + impossible_counts.data_page_header_v2.__set_num_nulls(2); + EXPECT_TRUE(parse_header(impossible_counts).is()); + + tparquet::PageHeader missing_layout; + missing_layout.type = tparquet::PageType::DATA_PAGE; + missing_layout.__set_compressed_page_size(0); + missing_layout.__set_uncompressed_page_size(0); + EXPECT_TRUE(parse_header(missing_layout).is()); + + auto swapped_layout = oversized_levels; + swapped_layout.type = tparquet::PageType::DATA_PAGE; + EXPECT_TRUE(parse_header(swapped_layout).is()); + + auto competing_layouts = negative; + competing_layouts.__set_compressed_page_size(0); + competing_layouts.__set_uncompressed_page_size(0); + competing_layouts.__isset.data_page_header_v2 = true; + competing_layouts.data_page_header_v2.__set_num_values(0); + competing_layouts.data_page_header_v2.__set_num_rows(0); + competing_layouts.data_page_header_v2.__set_num_nulls(0); + competing_layouts.data_page_header_v2.__set_repetition_levels_byte_length(0); + competing_layouts.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(competing_layouts).is()); +} + +TEST(ParquetV2NativeDecoderTest, ShiftedOffsetIndexFallsBackToSequentialPages) { + auto data_page = [](int32_t value) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(sizeof(value)); + header.__set_uncompressed_page_size(sizeof(value)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(sizeof(value)); + memcpy(payload.data(), &value, sizeof(value)); + return serialize_page(header, payload); + }; + auto verify_fallback = [&](bool cache_hit) { + const auto first_page = data_page(10); + const auto second_page = data_page(20); + std::vector bytes = first_page; + bytes.insert(bytes.end(), second_page.begin(), second_page.end()); + bytes.push_back(0); + + tparquet::OffsetIndex offset_index; + tparquet::PageLocation first_location; + first_location.__set_offset(0); + first_location.__set_compressed_page_size(cast_set(first_page.size() + 1)); + first_location.__set_first_row_index(0); + tparquet::PageLocation second_location; + second_location.__set_offset(cast_set(first_page.size() + 1)); + second_location.__set_compressed_page_size(cast_set(second_page.size())); + second_location.__set_first_row_index(1); + offset_index.__set_page_locations({first_location, second_location}); + EXPECT_TRUE( + validate_offset_index(offset_index, {.offset = 0, .length = bytes.size()}, 0, 2)); + + const std::string cache_key = cache_hit ? "shifted-index-cache-hit" : ""; + if (cache_hit) { + auto* page = new DataPage(first_page.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), first_page.data(), first_page.size()); + page->reset_size(first_page.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(cache_key, bytes.size(), 0), page, &handle, + segment_v2::DATA_PAGE); + } + + tparquet::ColumnMetaData metadata; + metadata.__set_type(tparquet::Type::INT32); + metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + metadata.__set_num_values(2); + metadata.__set_total_compressed_size(bytes.size()); + metadata.__set_data_page_offset(0); + MemoryBufferedReader stream(std::move(bytes)); + PageReader reader(&stream, nullptr, 0, metadata.total_compressed_size, 2, + metadata, ParquetPageReadContext(cache_hit, cache_key), + &offset_index); + + ASSERT_TRUE(reader.parse_page_header().ok()); + reader.skip_page_data(); + ASSERT_TRUE(reader.next_page().ok()); + // The rectangles above are non-overlapping but shifted one byte from the serialized pages. + // Discarding the optional index must continue from the first page's real payload end. + ASSERT_TRUE(reader.parse_page_header().ok()); + const tparquet::PageHeader* parsed = nullptr; + ASSERT_TRUE(reader.get_page_header(&parsed).ok()); + EXPECT_EQ(parsed->data_page_header.num_values, 1); + }; + verify_fallback(false); + verify_fallback(true); +} + +TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) { + auto init_chunk = [](tparquet::PageHeader header, bool with_offset_index) { + std::vector payload(static_cast(header.compressed_page_size), 0); + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + if (with_offset_index) { + ColumnChunkReader reader_with_index(&reader, &chunk, &field, &offset_index, + 1, nullptr, context); + return reader_with_index.init(); + } + ColumnChunkReader sequential_reader(&reader, &chunk, &field, nullptr, 1, + nullptr, context); + return sequential_reader.init(); + }; + + tparquet::PageHeader v2; + v2.type = tparquet::PageType::DATA_PAGE_V2; + v2.__set_compressed_page_size(8); + v2.__set_uncompressed_page_size(8); + v2.__isset.data_page_header_v2 = true; + v2.data_page_header_v2.__set_num_values(2); + v2.data_page_header_v2.__set_num_rows(1); + v2.data_page_header_v2.__set_num_nulls(0); + v2.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + v2.data_page_header_v2.__set_repetition_levels_byte_length(0); + v2.data_page_header_v2.__set_definition_levels_byte_length(0); + v2.data_page_header_v2.__set_is_compressed(false); + auto status = init_chunk(v2, false); + EXPECT_TRUE(status.is()) << status; + status = init_chunk(v2, true); + EXPECT_TRUE(status.is()) << status; + + tparquet::PageHeader v1; + v1.type = tparquet::PageType::DATA_PAGE; + v1.__set_compressed_page_size(8); + v1.__set_uncompressed_page_size(8); + v1.__isset.data_page_header = true; + v1.data_page_header.__set_num_values(2); + v1.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + v1.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + v1.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + status = init_chunk(v1, true); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsOffsetIndexRowSpanMismatch) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(8); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + auto bytes = serialize_page(header, std::vector(8)); + MemoryBufferedReader stream(bytes); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + field.definition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + ParquetPageReadContext context(false, ""); + + ColumnChunkReader reader(&stream, &chunk, &field, &offset_index, + /*total_rows=*/2, nullptr, context); + const auto status = reader.init(); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyAllNullPagesInitializeZeroValueDecoders) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, true); + EXPECT_TRUE(status.ok()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyPagesRejectDefinitionLevelsThatRequireValues) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, false); + EXPECT_TRUE(status.is()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyReaderSkipsLeadingZeroValuePages) { + auto page = [](bool v2, std::optional value) { + std::vector payload; + if (value.has_value()) { + const std::vector definition_levels {2, 1}; + if (v2) { + payload = definition_levels; + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), definition_levels.size()); + payload.insert(payload.end(), definition_levels.begin(), definition_levels.end()); + } + const auto* value_bytes = reinterpret_cast(&*value); + payload.insert(payload.end(), value_bytes, value_bytes + sizeof(*value)); + } else if (!v2) { + // V1 prefixes its RLE definition-level stream even when that stream has no values. + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), 0); + } + tparquet::PageHeader header; + header.type = v2 ? tparquet::PageType::DATA_PAGE_V2 : tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + if (v2) { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(value.has_value() ? 1 : 0); + header.data_page_header_v2.__set_num_rows(value.has_value() ? 1 : 0); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_definition_levels_byte_length(value.has_value() ? 2 + : 0); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(value.has_value() ? 1 : 0); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + return serialize_page(header, payload); + }; + + static const auto utc = cctz::utc_time_zone(); + for (const bool v2 : {false, true}) { + SCOPED_TRACE(v2 ? "v2" : "v1"); + auto bytes = page(v2, std::nullopt); + const auto nonempty_page = page(v2, 7); + bytes.insert(bytes.end(), nonempty_page.begin(), nonempty_page.end()); + auto file = std::make_shared(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = make_nullable(std::make_shared()); + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + auto row_ranges = ::doris::RowRanges::create_single(1); + ScalarColumnReader scalar_reader(row_ranges, 1, chunk, nullptr, &utc, + nullptr); + ASSERT_TRUE( + scalar_reader + .init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) + .ok()); + FilterMap filter; + ASSERT_TRUE(filter.init(nullptr, 1, false).ok()); + ColumnPtr scalar_column = field.data_type->create_column(); + size_t scalar_rows = 0; + bool scalar_eof = false; + while (scalar_rows == 0 && !scalar_eof) { + size_t read_now = 0; + ASSERT_TRUE(scalar_reader + .read_column_data(scalar_column, field.data_type, nullptr, filter, + 1, &read_now, &scalar_eof, false) + .ok()); + scalar_rows += read_now; + } + ASSERT_EQ(scalar_rows, 1); + const auto& scalar_nullable = assert_cast(*scalar_column); + EXPECT_FALSE(scalar_nullable.is_null_at(0)); + EXPECT_EQ( + assert_cast(scalar_nullable.get_nested_column()).get_element(0), + 7); + + std::unique_ptr reader; + ASSERT_TRUE(LevelReader::create(file, chunk, &field, 1, bytes.size(), nullptr, false, "", + ParquetReaderCompat {}, &reader) + .ok()); + + std::vector repetition_levels; + std::vector definition_levels; + size_t rows_read = 0; + const auto status = + reader->read_rows(1, &repetition_levels, &definition_levels, &rows_read); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows_read, 1); + EXPECT_EQ(repetition_levels, (std::vector {0})); + EXPECT_EQ(definition_levels, (std::vector {1})); + } +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(12); + header.__set_uncompressed_page_size(12); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(2); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(4); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + // Two values [0, 1] contain only one repetition-level zero, so they describe one row. + EXPECT_TRUE(load_malformed_nested_page(header, {2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 2) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, FirstNestedV1PageRejectsOrphanContinuation) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + // A two-value RLE run at repetition level 1 has no level-0 row start. + const std::vector payload {2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(2); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + EXPECT_TRUE(load_malformed_nested_page(header, payload, 2).is()); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1ContinuationRemainsValidAfterFirstRowStart) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + first_page.insert(first_page.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader reader(first_page); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(first_page.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1IgnoresUnverifiableOffsetIndexRows) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + const auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + const auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + std::vector bytes = first_page; + bytes.insert(bytes.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation first_location; + first_location.__set_offset(0); + first_location.__set_compressed_page_size(first_page.size()); + first_location.__set_first_row_index(0); + tparquet::PageLocation second_location; + second_location.__set_offset(first_page.size()); + second_location.__set_compressed_page_size(second_page.size()); + // This V1 page continues row zero, but its index claims a new logical row. Repetition levels + // are the only authoritative source, so indexed seeking must be disabled for this chunk. + second_location.__set_first_row_index(1); + offset_index.__set_page_locations({first_location, second_location}); + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&stream, &chunk, &field, &offset_index, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1DiscardedOffsetIndexStopsAtLogicalChunkEnd) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + const std::vector payload {2, 0, 0, 0, 2, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page(header, payload); + const size_t page_size = bytes.size(); + // A compatibility reader may pad the validated physical range beyond the encoded chunk. + bytes.resize(page_size + 16, 0); + + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(page_size); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(page_size); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + ColumnChunkRange padded_range {.offset = 0, .length = bytes.size()}; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&stream, &chunk, &field, &offset_index, 1, nullptr, + context, &padded_range); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + EXPECT_EQ(rows, 1); + EXPECT_FALSE(cross_page); + EXPECT_FALSE(chunk_reader.has_next_page()); +} + +TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { + for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(std::numeric_limits::max()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(std::numeric_limits::max()); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } + // The four-byte V1 level length (or V2 value payload) contains no advertised levels. The + // reader must report corruption without reserving INT_MAX level slots first. + EXPECT_TRUE(load_malformed_nested_page(header, {0, 0, 0, 0}).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, PageDecompressionRejectsBothSizeMismatchDirections) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + std::vector input(8, 7); + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(input.data(), input.size()), &compressed).ok()); + std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + for (const int32_t advertised_size : {4, 12}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(advertised_size); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + EXPECT_FALSE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndLogicalSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(1); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(load_scripted_page(header, std::vector(8), + tparquet::CompressionCodec::UNCOMPRESSED) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDictionaryRejectsDeclaredPayloadBeforeAllocation) { + tparquet::PageHeader header; + header.__set_uncompressed_page_size(std::numeric_limits::max()); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(0); + EXPECT_TRUE(validate_dictionary_page_size(header).is()); +} + +TEST(ParquetV2NativeDecoderTest, NonemptyFixedWidthDictionaryRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(validate_dictionary_page_size(header, sizeof(int32_t)).is()); + + const std::vector payload(compressed.data(), compressed.data() + compressed.size()); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, RequiredFixedWidthPageRejectsImpossibleExtentBeforeAllocation) { + for (const auto encoding : {tparquet::Encoding::PLAIN, tparquet::Encoding::BYTE_STREAM_SPLIT}) { + tparquet::PageHeader header; + header.__set_uncompressed_page_size(std::numeric_limits::max()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(encoding); + EXPECT_TRUE(validate_fixed_width_page_size(header, sizeof(int32_t), 0, 0) + .is()); + header.__set_uncompressed_page_size(sizeof(int32_t)); + EXPECT_TRUE(validate_fixed_width_page_size(header, sizeof(int32_t), 0, 0).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, OptionalV2FixedWidthPageRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const std::vector levels {2, 1}; + + for (const auto encoding : {tparquet::Encoding::PLAIN, tparquet::Encoding::BYTE_STREAM_SPLIT}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(levels.size() + compressed.size()); + header.__set_uncompressed_page_size((8 << 20) + levels.size()); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(1); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(levels.size()); + header.data_page_header_v2.__set_is_compressed(true); + std::vector payload = levels; + payload.insert(payload.end(), compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); + } +} + +TEST(ParquetV2NativeDecoderTest, RepeatedV2FixedWidthPageRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const std::vector levels {2, 0, 2, 1}; + + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(levels.size() + compressed.size()); + header.__set_uncompressed_page_size((8 << 20) + levels.size()); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(1); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(2); + header.data_page_header_v2.__set_definition_levels_byte_length(2); + header.data_page_header_v2.__set_is_compressed(true); + std::vector payload = levels; + payload.insert(payload.end(), compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, VariableWidthDataPagePreflightsCompressedExtent) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::vector value {1, 0, 0, 0, 'x'}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + const std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::BYTE_ARRAY); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY, true, + tparquet::Type::BYTE_ARRAY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, VariableWidthDictionaryPreflightsCompressedExtent) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::vector value {1, 0, 0, 0, 'x'}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const Slice payload(compressed.data(), compressed.size()); + + EXPECT_TRUE( + validate_compressed_page_size(tparquet::CompressionCodec::SNAPPY, payload, value.size()) + .ok()); + EXPECT_TRUE(validate_compressed_page_size(tparquet::CompressionCodec::SNAPPY, payload, 8 << 20) + .is()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE( + load_scripted_page( + header, + std::vector(compressed.data(), compressed.data() + compressed.size()), + tparquet::CompressionCodec::SNAPPY, false, tparquet::Type::BYTE_ARRAY) + .is()); + EXPECT_TRUE( + load_scripted_page( + header, + std::vector(compressed.data(), compressed.data() + compressed.size()), + tparquet::CompressionCodec::SNAPPY, true, tparquet::Type::BYTE_ARRAY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDataPagesRequireEqualPhysicalAndLogicalSizes) { + for (const auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + } + const std::vector payload(8); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED) + .is()); + EXPECT_TRUE( + load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED, true) + .is()); + } +} + +TEST(ParquetV2NativeDecoderTest, LegacyV2CompressedOverrideAllowsDifferentSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(16); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, true) + .ok()); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { + MemoryBufferedReader reader(std::vector {0}); + tparquet::ColumnMetaData metadata; + tparquet::OffsetIndex empty_index; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, 1, 1, metadata, context, &empty_index); + EXPECT_FALSE(page_reader.has_next_page()); + EXPECT_TRUE(page_reader.next_page().is()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkRangeRejectsSignedOverflowAndBoundsLegacyPadding) { + tparquet::ColumnMetaData metadata; + metadata.__set_data_page_offset(-1); + metadata.__set_total_compressed_size(10); + ColumnChunkRange range; + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(95); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(10); + metadata.__set_total_compressed_size(std::numeric_limits::max()); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_total_compressed_size(20); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, true, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 25); + + metadata.__set_dictionary_page_offset(0); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 20); + + metadata.__set_dictionary_page_offset(5); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 5); + EXPECT_EQ(range.length, 20); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlappingLocations) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(110); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(120); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_offset(140); + second.__set_first_row_index(0); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_TRUE(validate_offset_index(index, range, 110, 20)); + + range = {.offset = std::numeric_limits::max(), .length = 2}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsShiftedFirstDataPage) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(120); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(140); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + + EXPECT_FALSE(validate_offset_index(index, range, 100, 20)); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingDataDecoder) { + tparquet::PageHeader index_header; + index_header.type = tparquet::PageType::INDEX_PAGE; + index_header.__set_compressed_page_size(3); + index_header.__set_uncompressed_page_size(3); + index_header.__set_index_page_header(tparquet::IndexPageHeader()); + auto bytes = serialize_page(index_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsUnknownAuxiliaryPage) { + tparquet::PageHeader unknown_header; + unknown_header.type = static_cast(127); + unknown_header.__set_compressed_page_size(3); + unknown_header.__set_uncompressed_page_size(3); + auto bytes = serialize_page(unknown_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); +} + +TEST(ParquetV2NativeDecoderTest, LegacyDataPageV2OverridesFalseCompressedFlag) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(1024); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + EXPECT_TRUE(should_cache_decompressed(&header, metadata, false)); + EXPECT_FALSE(should_cache_decompressed(&header, metadata, true)); + + EXPECT_TRUE(parquet_reader_compat("parquet-cpp version 1.5.0").data_page_v2_always_compressed); + EXPECT_FALSE(parquet_reader_compat("parquet-cpp version 2.0.0").data_page_v2_always_compressed); + EXPECT_TRUE(parquet_reader_compat("parquet-mr version 1.2.8").parquet_816_padding); + EXPECT_FALSE(parquet_reader_compat("parquet-mr version 1.2.9").parquet_816_padding); +} + +TEST(ParquetV2NativeDecoderTest, NullableNumericOverflowIsNullOnlyOutsideStrictMode) { + const int32_t overflow = 1000; + auto decode = [&](bool strict, MutableColumnPtr* column, IColumn::Filter* null_map) { + std::unique_ptr decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder)); + decoder->set_type_length(sizeof(overflow)); + Slice slice(reinterpret_cast(&overflow), sizeof(overflow)); + RETURN_IF_ERROR(decoder->set_data(&slice)); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + MutableColumnPtr column; + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ASSERT_TRUE(decode(false, &column, &null_map).ok()); + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + + null_map.clear(); + null_map.resize_fill(1, 0); + EXPECT_FALSE(decode(true, &column, &null_map).ok()); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryConversionFailureIsDeferredUntilReferenced) { + auto decode_dictionary_id = [](uint8_t dictionary_id, bool strict, IColumn::Filter* null_map, + MutableColumnPtr* column) { + const std::array dictionary_values {1, 1000}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + RETURN_IF_ERROR(Decoder::get_decoder(tparquet::Type::INT32, + tparquet::Encoding::RLE_DICTIONARY, decoder)); + decoder->set_type_length(sizeof(int32_t)); + RETURN_IF_ERROR( + decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size())); + char encoded_id[] = {1, 2, static_cast(dictionary_id)}; + Slice id_slice(encoded_id, sizeof(encoded_id)); + RETURN_IF_ERROR(decoder->set_data(&id_slice)); + + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + for (const bool strict : {false, true}) { + IColumn::Filter null_map; + IColumn::Filter* output_null_map = nullptr; + if (strict) { + null_map.resize_fill(1, 0); + output_null_map = &null_map; + } + + MutableColumnPtr column; + const auto unused_bad = decode_dictionary_id(0, strict, output_null_map, &column); + ASSERT_TRUE(unused_bad.ok()) << unused_bad; + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(assert_cast(*column).get_element(0), 1); + + const auto referenced_bad = decode_dictionary_id(1, strict, output_null_map, &column); + EXPECT_TRUE(referenced_bad.is()) << referenced_bad; + EXPECT_TRUE(column->empty()); + if (output_null_map != nullptr) { + EXPECT_EQ((*output_null_map)[0], 0); + } + } +} + +TEST(ParquetV2NativeDecoderTest, FixedLengthStringsAppendAsOneContiguousSpan) { + ColumnString column; + const std::string values = "aaabbbccc"; + column.insert_many_fixed_length_data(values.data(), 3, 3); + ASSERT_EQ(column.size(), 3); + EXPECT_EQ(column.get_data_at(0).to_string_view(), "aaa"); + EXPECT_EQ(column.get_data_at(1).to_string_view(), "bbb"); + EXPECT_EQ(column.get_data_at(2).to_string_view(), "ccc"); +} + +TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) { + ColumnChunkReaderStatistics first_chunk; + first_chunk.page_read_counter = 1; + first_chunk.data_page_read_counter = 1; + ColumnChunkReaderStatistics second_chunk; + second_chunk.page_read_counter = 1; + second_chunk.data_page_read_counter = 1; + ColumnReader::ColumnStatistics combined; + ColumnReader::ColumnStatistics first(first_chunk, 0); + ColumnReader::ColumnStatistics second(second_chunk, 0); + combined.merge(first); + combined.merge(second); + + EXPECT_EQ(combined.page_read_counter, 2); + ASSERT_EQ(combined.leaf_page_read_counters.size(), 2); + EXPECT_EQ(combined.leaf_page_read_counters[0], 1); + EXPECT_EQ(combined.leaf_page_read_counters[1], 1); +} + +TEST(ParquetV2NativeDecoderTest, NativePageCacheUsesStableFileDescriptionIdentity) { + ParquetPageCacheKeyBuilder first; + ParquetPageCacheKeyBuilder replaced; + first.init("s3://bucket/object::etag-v1::1234"); + replaced.init("s3://bucket/object::etag-v2::1234"); + EXPECT_EQ(first.make_key(4096, 128).fname, "s3://bucket/object::etag-v1::1234"); + EXPECT_NE(first.make_key(4096, 128).encode(), replaced.make_key(4096, 128).encode()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompressed) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(100); + header.__set_uncompressed_page_size(1000); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + + // The representation is explicit in V2; a codec and compression ratio cannot override it on + // a warm cache hit. + EXPECT_TRUE(should_cache_decompressed(&header, metadata)); +} + +TEST(ParquetV2NativeDecoderTest, CacheDisabledPagesDoNotPrepareCopiedPayload) { + EXPECT_FALSE(can_prepare_page_cache_payload(false, false, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, true, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, false, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, true, false)); + EXPECT_TRUE(can_prepare_page_cache_payload(true, false, true, true)); +} + +TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { + ::doris::RowRanges row_ranges; + tparquet::ColumnChunk chunk; + ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); + + constexpr size_t max_retained_bytes = 64UL << 10; + reader.reserve_batch_scratch_for_test(1UL << 16); + const size_t oversized_bytes = reader.retained_batch_scratch_bytes_for_test(); + ASSERT_GT(oversized_bytes, max_retained_bytes); + + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + const size_t released_bytes = reader.retained_batch_scratch_bytes_for_test(); + EXPECT_LT(released_bytes, oversized_bytes); + EXPECT_LE(released_bytes, max_retained_bytes + sizeof(void*)); +} + +} // namespace +} // namespace doris::format::parquet::native diff --git a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp new file mode 100644 index 00000000000000..490a0bde204a74 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "../../../benchmark/parquet/parquet_benchmark_scenarios.h" + +#include + +#include +#include +#include +#include + +namespace doris::parquet_benchmark { +namespace { + +TEST(ParquetBenchmarkScenariosTest, DecoderMatrixCoversNativeEncodingAndTypeFamilies) { + const auto scenarios = decoder_scenarios(); + const std::set> actual = [&] { + std::set> values; + for (const auto& scenario : scenarios) { + values.emplace(scenario.encoding, scenario.value_type); + } + return values; + }(); + + const std::set> expected { + {Encoding::PLAIN, ValueType::INT32}, + {Encoding::PLAIN, ValueType::INT64}, + {Encoding::PLAIN, ValueType::FLOAT}, + {Encoding::PLAIN, ValueType::DOUBLE}, + {Encoding::PLAIN, ValueType::BYTE_ARRAY}, + {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::INT32}, + {Encoding::DICTIONARY, ValueType::INT64}, + {Encoding::DICTIONARY, ValueType::FLOAT}, + {Encoding::DICTIONARY, ValueType::DOUBLE}, + {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, + {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + }; + EXPECT_EQ(actual, expected); +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversNullableSparseAndProjectionAxes) { + const auto scenarios = reader_scenarios(); + for (const int null_percent : {0, 1, 10, 50, 90}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == Encoding::PLAIN && + scenario.null_percent == null_percent && + scenario.null_pattern == pattern && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32; + })) << "missing nullable sparse axis combination"; + } + } + } + } +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversOperationsEncodingsAndSchemaWidths) { + const auto scenarios = reader_scenarios(); + for (const auto operation : + {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, + ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == operation; + })); + } + for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, + Encoding::DELTA_BINARY_PACKED}) { + for (const auto operation : {ReaderOperation::FULL_SCAN, ReaderOperation::PREDICATE_SCAN}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.encoding == encoding && scenario.operation == operation; + })); + } + } + for (const int width : {4, 32, 128, 512}) { + for (const int predicate_position : {0, width - 1}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.schema_width == width && + scenario.predicate_position == predicate_position; + })); + } + } +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversFixedWidthRawFilterAxes) { + const auto scenarios = reader_scenarios(); + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == encoding && scenario.null_percent == 10 && + scenario.null_pattern == Pattern::ALTERNATING && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32 && + scenario.predicate_position == 0; + })) << "missing fixed-width raw filter axis combination"; + } + } + } +} + +TEST(ParquetBenchmarkScenariosTest, SelectionPlanDistinguishesClusteredAndSparseRuns) { + const auto clustered = make_selection_plan(1000, 10, Pattern::CLUSTERED); + EXPECT_EQ(clustered.total_rows, 1000); + EXPECT_EQ(clustered.selected_rows, 100); + ASSERT_EQ(clustered.ranges.size(), 1); + EXPECT_EQ(clustered.ranges.front().count, 100); + + const auto alternating = make_selection_plan(1000, 10, Pattern::ALTERNATING); + EXPECT_EQ(alternating.total_rows, 1000); + EXPECT_EQ(alternating.selected_rows, 100); + EXPECT_EQ(alternating.ranges.size(), 100); + + EXPECT_EQ(make_selection_plan(1000, 0, Pattern::ALTERNATING).ranges.size(), 0); + EXPECT_EQ(make_selection_plan(1000, 100, Pattern::ALTERNATING).ranges.size(), 1); +} + +} // namespace +} // namespace doris::parquet_benchmark diff --git a/be/test/format_v2/parquet/parquet_column_reader_test.cpp b/be/test/format_v2/parquet/parquet_column_reader_test.cpp deleted file mode 100644 index fb4cd129e1b03d..00000000000000 --- a/be/test/format_v2/parquet/parquet_column_reader_test.cpp +++ /dev/null @@ -1,3682 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_decimal.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "core/types.h" -#include "format_v2/file_reader.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/selection_vector.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -template -const ColumnType& get_nullable_nested_column(const IColumn& column) { - // File-local schema exposed by the parquet reader follows Doris external-table semantics: - // nested STRUCT fields, LIST elements, and MAP keys/values are nullable even when the parquet - // field is required. - const auto& nullable_column = assert_cast(column); - return assert_cast(nullable_column.get_nested_column()); -} - -ParquetColumnSchema mock_column_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "mock"; - schema.type = std::make_shared(); - return schema; -} - -class BaseUnsupportedReader final : public ParquetColumnReader { -public: - BaseUnsupportedReader() - : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } -}; - -class DefaultSelectReader final : public ParquetColumnReader { -public: - DefaultSelectReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - auto& values = assert_cast(*column); - for (int64_t row = 0; row < rows; ++row) { - values.insert_value(static_cast(_cursor + row)); - } - _cursor += rows; - *rows_read = rows; - _read_ranges.push_back(rows); - return Status::OK(); - } - - Status skip(int64_t rows) override { - _cursor += rows; - _skip_ranges.push_back(rows); - return Status::OK(); - } - - const std::vector& read_ranges() const { return _read_ranges; } - const std::vector& skip_ranges() const { return _skip_ranges; } - -private: - int64_t _cursor = 0; - std::vector _read_ranges; - std::vector _skip_ranges; -}; - -class NestedSkipReader final : public ParquetColumnReader { -public: - NestedSkipReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - *values_consumed = length_upper_bound; - return Status::OK(); - } -}; - -class ParquetColumnReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_column_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "reader.parquet").string(); - _plain_file_path = (_test_dir / "plain_reader.parquet").string(); - write_parquet_file(); - _file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto metadata = _file_reader->metadata(); - ASSERT_EQ(metadata->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(_fields.size(), _expected_by_field.size()); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_all_null_int32_array() { - arrow::Int32Builder builder; - for (int64_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(builder.AppendNull().ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_required_struct_array() { - auto struct_type = arrow::struct_({arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - const std::vector a_values = {101, 102, 103, 104, 105}; - const std::vector b_values = {"sa", "sb", "sc", "sd", "se"}; - for (size_t row = 0; row < a_values.size(); ++row) { - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(a_values[row]).ok()); - EXPECT_TRUE(b_builder->Append(b_values[row]).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(201).ok()); - EXPECT_TRUE(b_builder->Append("nsa").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(203).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(204).ok()); - EXPECT_TRUE(b_builder->Append("nsd").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_decimal_array() { - auto decimal_type = arrow::decimal128(38, 6); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("d", decimal_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto d_array_builder = std::make_unique( - decimal_type, arrow::default_memory_pool()); - field_builders.push_back(std::shared_ptr(std::move(d_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* d_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(123456789)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(d_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(-987654321)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("xs", list_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - field_builders.push_back(list_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(305).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_map_array() { - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("kv", map_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - field_builders.push_back(map_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(401).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("one").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(403).ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(404).ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(405).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->Append("five").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_nested_struct_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto nested_type = arrow::struct_({arrow::field("xs", list_type, true)}); - auto struct_type = arrow::struct_({arrow::field("nested", nested_type, true)}); - - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - std::vector> nested_field_builders; - nested_field_builders.push_back(list_builder); - auto nested_builder = std::make_shared( - nested_type, arrow::default_memory_pool(), std::move(nested_field_builders)); - std::vector> field_builders; - field_builders.push_back(nested_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(7).ok()); - EXPECT_TRUE(value_builder->Append(8).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), false))); - const std::vector> values = { - {1, 2}, {3}, {4, 5, 6}, {7}, {8, 9}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto value : row) { - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append(20).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(30).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(110).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(120).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(130).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_list_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - auto* a_builder = assert_cast(struct_builder->field_builder(0)); - auto* b_builder = assert_cast(struct_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(11).ok()); - EXPECT_TRUE(b_builder->Append("la").ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(12).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(13).ok()); - EXPECT_TRUE(b_builder->Append("ld").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(14).ok()); - EXPECT_TRUE(b_builder->Append("le").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_list_list_int_array() { - auto value_builder = std::make_shared(); - auto inner_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto inner_list_builder = std::make_shared( - arrow::default_memory_pool(), value_builder, inner_list_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), inner_list_builder, - arrow::list(arrow::field("element", inner_list_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->AppendNull().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - const std::vector>> values = { - {{1, "a"}, {2, "b"}}, {{3, "c"}}, {{4, "d"}, {5, "e"}, {6, "f"}}, - {{7, "g"}}, {{8, "h"}, {9, "i"}}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto& [key, value] : row) { - EXPECT_TRUE(key_builder->Append(key).ok()); - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append("bb").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_struct_map_array() { - auto key_builder = std::make_shared(); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto value_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - auto* a_builder = assert_cast(value_builder->field_builder(0)); - auto* b_builder = assert_cast(value_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(21).ok()); - EXPECT_TRUE(b_builder->Append("ma").ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(22).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(24).ok()); - EXPECT_TRUE(b_builder->Append("me").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(201).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(key_builder->Append(202).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(203).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(204).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(205).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_map_list_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), map_builder, - arrow::list(arrow::field("element", map_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("a").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(value_builder->Append("d").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_map_map_array() { - auto key_builder = std::make_shared(); - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, nested_map_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(101).ok()); - EXPECT_TRUE(nested_value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(401).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_list_struct_map_list_array() { - auto element_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - element_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - auto map_builder = std::make_shared(arrow::default_memory_pool(), - key_builder, list_builder, map_type); - auto struct_type = arrow::struct_({arrow::field("kv", map_type, true)}); - std::vector> struct_field_builders; - struct_field_builders.push_back(map_builder); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(struct_field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(10).ok()); - EXPECT_TRUE(element_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_map_list_map_array() { - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto list_type = arrow::list(arrow::field("element", nested_map_type, true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - nested_map_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(1).ok()); - EXPECT_TRUE(nested_value_builder->Append("a").ok()); - EXPECT_TRUE(nested_key_builder->Append(2).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(3).ok()); - EXPECT_TRUE(nested_value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(50).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(4).ok()); - EXPECT_TRUE(nested_value_builder->Append("d").ok()); - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, std::shared_ptr array, - std::function validator) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - _expected_by_field.push_back(std::move(validator)); - } - - void write_parquet_file() { - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); - add_field(arrow::field("all_null_int_col", arrow::int32(), true), - build_all_null_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - for (size_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(nullable_column.is_null_at(row)); - } - }); - add_field(arrow::field("struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false), - }), - false), - build_required_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_STRUCT); - const auto& struct_column = assert_cast(column); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - get_nullable_nested_column(struct_column.get_column(1)); - EXPECT_EQ(a_values.get_element(0), 101); - EXPECT_EQ(a_values.get_element(4), 105); - EXPECT_EQ(b_values.get_data_at(1).to_string(), "sb"); - EXPECT_EQ(b_values.get_data_at(4).to_string(), "se"); - }); - add_field(arrow::field("nullable_struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }), - true), - build_nullable_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_nested = - assert_cast(b_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 201); - EXPECT_EQ(a_values.get_element(2), 203); - EXPECT_EQ(a_values.get_element(3), 204); - EXPECT_FALSE(b_values.is_null_at(0)); - EXPECT_TRUE(b_values.is_null_at(2)); - EXPECT_FALSE(b_values.is_null_at(3)); - EXPECT_EQ(b_nested.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(b_nested.get_data_at(3).to_string(), "nsd"); - }); - add_field(arrow::field("nullable_struct_decimal_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("d", arrow::decimal128(38, 6), true), - }), - true), - build_nullable_struct_with_decimal_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& d_values = - assert_cast(struct_column.get_column(1)); - const auto& d_nested = - assert_cast(d_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_FALSE(d_values.is_null_at(0)); - EXPECT_TRUE(d_values.is_null_at(2)); - EXPECT_FALSE(d_values.is_null_at(3)); - EXPECT_EQ(d_nested.get_element(0), Decimal128V3(123456789)); - EXPECT_EQ(d_nested.get_element(3), Decimal128V3(-987654321)); - }); - auto struct_list_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("xs", arrow::list(arrow::field("element", arrow::int32(), true)), - true), - }); - add_field(arrow::field("nullable_struct_list_col", struct_list_type, true), - build_nullable_struct_with_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_EQ(a_values.get_element(4), 305); - - const auto& xs_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = - assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = - assert_cast(xs_array.get_data()); - ASSERT_EQ(elements.size(), 4); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - const auto& values = - assert_cast(elements.get_nested_column()); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_EQ(values.get_element(3), 5); - }); - auto struct_map_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("kv", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - }); - add_field(arrow::field("nullable_struct_map_col", struct_map_type, true), - build_nullable_struct_with_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(2), 403); - EXPECT_EQ(a_values.get_element(3), 404); - EXPECT_EQ(a_values.get_element(4), 405); - - const auto& kv_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = - assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); - }); - auto nested_struct_list_type = arrow::struct_({ - arrow::field("nested", - arrow::struct_({ - arrow::field("xs", - arrow::list(arrow::field("element", - arrow::int32(), true)), - true), - }), - true), - }); - add_field(arrow::field("nullable_struct_nested_struct_list_col", nested_struct_list_type, - true), - build_nullable_struct_with_nested_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = - assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); - }); - add_field(arrow::field("list_int_col", - arrow::list(arrow::field("element", arrow::int32(), false)), false), - build_required_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_ARRAY); - const auto* array_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ( - remove_nullable(array_type->get_nested_type())->get_primitive_type(), - TYPE_INT); - const auto& array_column = assert_cast(column); - ASSERT_EQ(array_column.size(), ROW_COUNT); - const auto array_size_at = [&array_column](size_t row_idx) { - return array_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : array_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(array_size_at(0), 2); - EXPECT_EQ(array_size_at(1), 1); - EXPECT_EQ(array_size_at(2), 3); - EXPECT_EQ(array_size_at(4), 2); - const auto& values = - get_nullable_nested_column(array_column.get_data()); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(5), 6); - EXPECT_EQ(values.get_element(8), 9); - }); - add_field(arrow::field("nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), true), - build_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 30); - EXPECT_EQ(values.get_element(4), 40); - }); - add_field(arrow::field("required_nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), false), - build_required_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& array_column = assert_cast(column); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 5); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - ASSERT_EQ(elements.size(), 5); - EXPECT_TRUE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(4)); - }); - auto list_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field("nullable_list_struct_col", - arrow::list(arrow::field("element", list_struct_type, true)), true), - build_nullable_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - - const auto& elements = - assert_cast(array_column.get_data()); - const auto& struct_column = - assert_cast(elements.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 11); - EXPECT_EQ(a_values.get_element(1), 12); - EXPECT_EQ(a_values.get_element(3), 13); - EXPECT_EQ(a_values.get_element(4), 14); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); - }); - auto nested_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field(arrow::field("nullable_list_list_int_col", - arrow::list(arrow::field("element", nested_list_type, true)), true), - build_nullable_list_list_int_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 4); - EXPECT_EQ(outer_offsets[3], 5); - EXPECT_EQ(outer_offsets[4], 7); - - const auto& inner_nullable = - assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_FALSE(inner_nullable.is_null_at(6)); - - const auto& inner_array = - assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[5], 5); - EXPECT_EQ(inner_offsets[6], 7); - - const auto& elements = - assert_cast(inner_array.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 7); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 3); - EXPECT_EQ(values.get_element(4), 4); - EXPECT_EQ(values.get_element(5), 5); - EXPECT_TRUE(elements.is_null_at(6)); - }); - add_field(arrow::field( - "map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)), - false), - build_required_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_MAP); - const auto* map_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), - TYPE_INT); - EXPECT_EQ(remove_nullable(map_type->get_value_type())->get_primitive_type(), - TYPE_STRING); - const auto& map_column = assert_cast(column); - ASSERT_EQ(map_column.size(), ROW_COUNT); - const auto map_size_at = [&map_column](size_t row_idx) { - return map_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : map_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(map_size_at(0), 2); - EXPECT_EQ(map_size_at(1), 1); - EXPECT_EQ(map_size_at(2), 3); - EXPECT_EQ(map_size_at(4), 2); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(keys.size(), 9); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(5), 6); - EXPECT_EQ(keys.get_element(8), 9); - EXPECT_EQ(values.get_data_at(0).to_string(), "a"); - EXPECT_EQ(values.get_data_at(5).to_string(), "f"); - EXPECT_EQ(values.get_data_at(8).to_string(), "i"); - }); - add_field( - arrow::field("nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - build_nullable_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 10); - EXPECT_EQ(keys.get_element(1), 20); - EXPECT_EQ(keys.get_element(3), 40); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(values.is_null_at(3)); - }); - add_field( - arrow::field("required_nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - false), - build_required_nullable_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& map_column = assert_cast(column); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 4); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(3)); - }); - auto map_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field( - "nullable_map_int_struct_col", - arrow::map(arrow::int32(), arrow::field("value", map_struct_type, true)), - true), - build_nullable_int_struct_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& struct_column = - assert_cast(values.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 21); - EXPECT_EQ(a_values.get_element(1), 22); - EXPECT_EQ(a_values.get_element(3), 24); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); - }); - auto map_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field( - arrow::field("nullable_map_int_list_col", - arrow::map(arrow::int32(), arrow::field("value", map_list_type, true)), - true), - build_nullable_int_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), ROW_COUNT); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 4); - EXPECT_EQ(map_offsets[4], 5); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 5); - EXPECT_EQ(keys.get_element(0), 201); - EXPECT_EQ(keys.get_element(1), 202); - EXPECT_EQ(keys.get_element(2), 203); - EXPECT_EQ(keys.get_element(3), 204); - EXPECT_EQ(keys.get_element(4), 205); - - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_FALSE(values.is_null_at(4)); - - const auto& list_column = - assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); - - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(element_values.get_element(0), 1); - EXPECT_EQ(element_values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(element_values.get_element(3), 3); - EXPECT_EQ(element_values.get_element(4), 4); - }); - auto list_map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field("nullable_list_map_int_string_col", - arrow::list(arrow::field("element", list_map_type, true)), true), - build_nullable_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& map_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 3); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(value_data.get_data_at(3).to_string(), "d"); - }); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field( - "nullable_map_int_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)), - true), - build_nullable_int_map_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 4); - - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 4); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - - const auto& inner_values = - assert_cast(outer_map.get_values()); - ASSERT_EQ(inner_values.size(), 4); - EXPECT_FALSE(inner_values.is_null_at(0)); - EXPECT_FALSE(inner_values.is_null_at(1)); - EXPECT_TRUE(inner_values.is_null_at(2)); - EXPECT_FALSE(inner_values.is_null_at(3)); - - const auto& inner_map = - assert_cast(inner_values.get_nested_column()); - const auto& inner_offsets = inner_map.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 4); - EXPECT_EQ(inner_offsets[0], 1); - EXPECT_EQ(inner_offsets[1], 1); - EXPECT_EQ(inner_offsets[2], 1); - EXPECT_EQ(inner_offsets[3], 2); - const auto& inner_keys = - get_nullable_nested_column(inner_map.get_keys()); - const auto& inner_strings = - assert_cast(inner_map.get_values()); - const auto& inner_string_data = - assert_cast(inner_strings.get_nested_column()); - ASSERT_EQ(inner_keys.size(), 2); - EXPECT_EQ(inner_keys.get_element(0), 101); - EXPECT_EQ(inner_keys.get_element(1), 401); - EXPECT_EQ(inner_string_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(inner_strings.is_null_at(1)); - }); - auto deep_list_value_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto deep_list_map_type = - arrow::map(arrow::int32(), arrow::field("value", deep_list_value_type, true)); - auto deep_list_struct_type = arrow::struct_({arrow::field("kv", deep_list_map_type, true)}); - add_field(arrow::field("nullable_list_struct_map_list_col", - arrow::list(arrow::field("element", deep_list_struct_type, true)), - true), - build_deep_list_struct_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& struct_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - - const auto& struct_column = - assert_cast(struct_values.get_nested_column()); - const auto& map_values = - assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - - const auto& lists = - assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 4); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 4); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 3); - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 3); - EXPECT_EQ(element_values.get_element(0), 10); - EXPECT_TRUE(elements.is_null_at(1)); - EXPECT_EQ(element_values.get_element(2), 40); - }); - auto deep_map_nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto deep_map_list_type = - arrow::list(arrow::field("element", deep_map_nested_map_type, true)); - add_field( - arrow::field( - "nullable_map_int_list_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", deep_map_list_type, true)), - true), - build_deep_map_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); - - const auto& inner_maps = - assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 6); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_FALSE(inner_maps.is_null_at(1)); - EXPECT_TRUE(inner_maps.is_null_at(2)); - EXPECT_FALSE(inner_maps.is_null_at(3)); - EXPECT_TRUE(inner_maps.is_null_at(4)); - EXPECT_FALSE(inner_maps.is_null_at(5)); - const auto& inner_map_column = - assert_cast(inner_maps.get_nested_column()); - const auto& inner_offsets = inner_map_column.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 6); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 3); - EXPECT_EQ(inner_offsets[4], 3); - EXPECT_EQ(inner_offsets[5], 4); - const auto& inner_keys = - get_nullable_nested_column(inner_map_column.get_keys()); - ASSERT_EQ(inner_keys.size(), 4); - EXPECT_EQ(inner_keys.get_element(0), 1); - EXPECT_EQ(inner_keys.get_element(1), 2); - EXPECT_EQ(inner_keys.get_element(2), 3); - EXPECT_EQ(inner_keys.get_element(3), 4); - const auto& strings = - assert_cast(inner_map_column.get_values()); - const auto& string_data = - assert_cast(strings.get_nested_column()); - ASSERT_EQ(strings.size(), 4); - EXPECT_EQ(string_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(strings.is_null_at(1)); - EXPECT_EQ(string_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(string_data.get_data_at(3).to_string(), "d"); - }); - - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - - auto file_result = arrow::io::FileOutputStream::Open(_file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ROW_COUNT, builder.build())); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_plain_reader(size_t field_idx) { - // Keep the normal fixture dictionary encoded. This one test writes a plain-encoded copy - // because Arrow BinaryRecordReader has a stricter reset contract than - // DictionaryRecordReader. - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - auto plain_file_result = arrow::io::FileOutputStream::Open(_plain_file_path); - DORIS_CHECK(plain_file_result.ok()); - std::shared_ptr plain_out = *plain_file_result; - ::parquet::WriterProperties::Builder plain_builder; - plain_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - plain_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - plain_builder.compression(::parquet::Compression::UNCOMPRESSED); - plain_builder.disable_dictionary(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), plain_out, ROW_COUNT, plain_builder.build())); - DORIS_CHECK(plain_out->Close().ok()); - - _plain_file_reader = ::parquet::ParquetFileReader::OpenFile(_plain_file_path, false); - auto metadata = _plain_file_reader->metadata(); - DORIS_CHECK(metadata != nullptr); - DORIS_CHECK(metadata->num_row_groups() == 1); - _plain_row_group = _plain_file_reader->RowGroup(0); - DORIS_CHECK(_plain_row_group != nullptr); - - ParquetColumnReaderFactory factory(_plain_row_group, metadata->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_child_reader(size_t field_idx, - size_t child_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[child_idx]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_grandchild_reader( - size_t field_idx, size_t child_idx, size_t grandchild_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - const auto& child_schema = *struct_schema.children[child_idx]; - EXPECT_LT(grandchild_idx, child_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = child_schema.local_id; - child_projection.project_all_children = false; - format::LocalColumnIndex grandchild_projection; - grandchild_projection.index = child_schema.children[grandchild_idx]->local_id; - child_projection.children.push_back(std::move(grandchild_projection)); - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - void read_and_validate(size_t field_idx) const { - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - _expected_by_field[field_idx](*_fields[field_idx], *column); - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet test field " << name; - return _fields.size(); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::string _plain_file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::unique_ptr<::parquet::ParquetFileReader> _plain_file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::shared_ptr<::parquet::RowGroupReader> _plain_row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; - std::vector> _expected_by_field; -}; - -TEST(ParquetColumnReaderBaseTest, SelectionVectorRangesAndValidation) { - SelectionVector identity; - ASSERT_TRUE(identity.verify(4, 5).ok()); - auto ranges = selection_to_ranges(identity, 4); - ASSERT_EQ(ranges.size(), 1); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 4); - - std::array selected = {0, 2, 3, 6, 6}; - SelectionVector external(selected.data(), 4); - auto status = external.verify(3, 7); - ASSERT_TRUE(status.ok()) << status; - ranges = selection_to_ranges(external, 3); - ASSERT_EQ(ranges.size(), 2); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 1); - EXPECT_EQ(ranges[1].start, 2); - EXPECT_EQ(ranges[1].length, 2); - - EXPECT_FALSE(external.verify(8, 7).ok()); - EXPECT_FALSE(external.verify(5, 7).ok()); - EXPECT_FALSE(external.verify(4, 6).ok()); - - std::array duplicate = {0, 2, 2}; - SelectionVector non_strict(duplicate.data(), duplicate.size()); - EXPECT_FALSE(non_strict.verify(3, 5).ok()); - EXPECT_FALSE(identity.verify(1, -1).ok()); -} - -TEST(ParquetColumnReaderBaseTest, DefaultSelectUsesSkipReadRangesAndNestedConsumeIsExplicit) { - DefaultSelectReader reader; - std::array selected = {1, 3, 4}; - SelectionVector selection(selected.data(), selected.size()); - auto column = ColumnInt32::create(); - MutableColumnPtr mutable_column = std::move(column); - auto status = reader.select(selection, selected.size(), 6, mutable_column); - ASSERT_TRUE(status.ok()) << status; - - const auto& values = assert_cast(*mutable_column); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 3); - EXPECT_EQ(values.get_element(2), 4); - EXPECT_EQ(reader.skip_ranges(), std::vector({1, 1, 1})); - EXPECT_EQ(reader.read_ranges(), std::vector({1, 2})); - - BaseUnsupportedReader unsupported_reader; - auto skip_status = unsupported_reader.skip(1); - EXPECT_FALSE(skip_status.ok()); - EXPECT_NE(skip_status.to_string().find("skip is not implemented"), std::string::npos); - EXPECT_FALSE(unsupported_reader.load_nested_batch(1).ok()); - int64_t values_read = 0; - EXPECT_FALSE(unsupported_reader.build_nested_column(1, mutable_column, &values_read).ok()); - EXPECT_FALSE(unsupported_reader.consume_nested_column(1, &values_read).ok()); - - NestedSkipReader nested_reader; - auto nested_status = nested_reader.consume_nested_column(3, &values_read); - ASSERT_TRUE(nested_status.ok()) << nested_status; - EXPECT_EQ(values_read, 3); -} - -TEST_F(ParquetColumnReaderTest, ScalarReadCoversRequiredNullableAllNullAndMultipleBatches) { - read_and_validate(find_field_idx("int32_col")); - read_and_validate(find_field_idx("string_col")); - read_and_validate(find_field_idx("nullable_int_col")); - read_and_validate(find_field_idx("all_null_int_col")); - - auto reader = create_reader(find_field_idx("int32_col")); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - ASSERT_TRUE(reader->read(3, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 3); - const auto& values = assert_cast(*column); - ASSERT_EQ(values.size(), ROW_COUNT); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_EQ(values.get_element(2), 30); - EXPECT_EQ(values.get_element(4), 50); -} - -TEST_F(ParquetColumnReaderTest, ScalarSkipCoversZeroSomeAllAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(0).ok()); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& first_value = assert_cast(*column); - EXPECT_EQ(first_value.get_element(0), 10); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(2).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& skipped_values = assert_cast(*column); - EXPECT_EQ(skipped_values.get_element(0), 30); - EXPECT_EQ(skipped_values.get_element(1), 40); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(ROW_COUNT).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - EXPECT_EQ(rows_read, 0); - EXPECT_EQ(column->size(), 0); - - reader = create_reader(find_field_idx("nullable_int_col")); - ASSERT_TRUE(reader->skip(1).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); -} - -TEST_F(ParquetColumnReaderTest, ScalarSelectCoversAllDisjointSingleZeroThenReadAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector all_selected(ROW_COUNT); - auto column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(all_selected, ROW_COUNT, ROW_COUNT, column).ok()); - const auto& all_values = assert_cast(*column); - ASSERT_EQ(all_values.size(), ROW_COUNT); - EXPECT_EQ(all_values.get_element(0), 10); - EXPECT_EQ(all_values.get_element(4), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array disjoint = {0, 2, 4}; - SelectionVector disjoint_selection(disjoint.data(), disjoint.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(disjoint_selection, disjoint.size(), ROW_COUNT, column).ok()); - const auto& disjoint_values = assert_cast(*column); - ASSERT_EQ(disjoint_values.size(), 3); - EXPECT_EQ(disjoint_values.get_element(0), 10); - EXPECT_EQ(disjoint_values.get_element(1), 30); - EXPECT_EQ(disjoint_values.get_element(2), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array single = {2}; - SelectionVector single_selection(single.data(), single.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(single_selection, single.size(), ROW_COUNT, column).ok()); - const auto& single_value = assert_cast(*column); - ASSERT_EQ(single_value.size(), 1); - EXPECT_EQ(single_value.get_element(0), 30); - - reader = create_reader(find_field_idx("int32_col")); - std::array first_last = {0, 4}; - SelectionVector first_last_selection(first_last.data(), first_last.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(first_last_selection, first_last.size(), ROW_COUNT, column).ok()); - const auto& first_last_values = assert_cast(*column); - ASSERT_EQ(first_last_values.size(), 2); - EXPECT_EQ(first_last_values.get_element(0), 10); - EXPECT_EQ(first_last_values.get_element(1), 50); - - reader = create_reader(find_field_idx("int32_col")); - SelectionVector empty_selection; - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(empty_selection, 0, 2, column).ok()); - ASSERT_EQ(column->size(), 0); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& after_empty_select = assert_cast(*column); - ASSERT_EQ(after_empty_select.size(), 1); - EXPECT_EQ(after_empty_select.get_element(0), 30); - - reader = create_reader(find_field_idx("nullable_int_col")); - std::array nullable_rows = {0, 1, 2}; - SelectionVector nullable_selection(nullable_rows.data(), nullable_rows.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(nullable_selection, nullable_rows.size(), ROW_COUNT, column).ok()); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidScalarInputsAndNestedScalarProjection) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& int_schema = *_fields[find_field_idx("int32_col")]; - ParquetColumnSchema invalid_leaf; - invalid_leaf.kind = ParquetColumnSchemaKind::PRIMITIVE; - invalid_leaf.name = "invalid_leaf"; - invalid_leaf.type = int_schema.type; - invalid_leaf.type_descriptor = int_schema.type_descriptor; - invalid_leaf.descriptor = int_schema.descriptor; - invalid_leaf.leaf_column_id = _file_reader->metadata()->num_columns(); - auto status = factory.create(invalid_leaf, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); - - ParquetColumnSchema null_descriptor; - null_descriptor.kind = ParquetColumnSchemaKind::PRIMITIVE; - null_descriptor.name = "null_descriptor"; - null_descriptor.type = int_schema.type; - null_descriptor.type_descriptor = int_schema.type_descriptor; - null_descriptor.leaf_column_id = int_schema.leaf_column_id; - status = factory.create(null_descriptor, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("descriptor is null"), std::string::npos); - - const auto& list_element_schema = - *_fields[find_field_idx("nullable_list_int_col")]->children[0]; - status = factory.create(list_element_schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("flat primitive columns"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex projection = - format::LocalColumnIndex::partial_local(list_schema.local_id); - format::LocalColumnIndex element_projection = - format::LocalColumnIndex::partial_local(list_element_schema.local_id); - projection.children.push_back(std::move(element_projection)); - status = factory.create(list_schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("scalar projection is invalid"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidComplexProjections) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& struct_schema = *_fields[find_field_idx("struct_col")]; - format::LocalColumnIndex struct_empty = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - auto status = factory.create(struct_schema, &struct_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); - - format::LocalColumnIndex struct_invalid = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - struct_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(struct_schema, &struct_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex list_empty = - format::LocalColumnIndex::partial_local(list_schema.local_id); - status = factory.create(list_schema, &list_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); - - const auto& map_schema = *_fields[find_field_idx("nullable_map_int_struct_col")]; - const auto& value_schema = *map_schema.children[1]; - format::LocalColumnIndex map_invalid = - format::LocalColumnIndex::partial_local(map_schema.local_id); - map_invalid.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - map_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(map_schema, &map_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadSupportedComplexTypes) { - read_and_validate(find_field_idx("struct_col")); - read_and_validate(find_field_idx("nullable_struct_col")); - read_and_validate(find_field_idx("nullable_struct_decimal_col")); - read_and_validate(find_field_idx("list_int_col")); - read_and_validate(find_field_idx("nullable_list_int_col")); - read_and_validate(find_field_idx("required_nullable_list_int_col")); - read_and_validate(find_field_idx("nullable_list_struct_col")); - read_and_validate(find_field_idx("nullable_list_list_int_col")); - read_and_validate(find_field_idx("map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_string_col")); - read_and_validate(find_field_idx("required_nullable_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_struct_col")); - read_and_validate(find_field_idx("nullable_map_int_list_col")); - read_and_validate(find_field_idx("nullable_list_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_map_int_string_col")); - read_and_validate(find_field_idx("nullable_list_struct_map_list_col")); - read_and_validate(find_field_idx("nullable_map_int_list_map_int_string_col")); -} - -TEST_F(ParquetColumnReaderTest, SkipThenRead) { - auto reader = create_reader(find_field_idx("int32_col")); - auto st = reader->skip(2); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 2); - EXPECT_EQ(int_values.get_element(0), 30); - EXPECT_EQ(int_values.get_element(1), 40); -} - -TEST_F(ParquetColumnReaderTest, SelectReadsOnlySelectedRanges) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 2); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 3); - EXPECT_EQ(int_values.get_element(0), 10); - EXPECT_EQ(int_values.get_element(1), 30); - EXPECT_EQ(int_values.get_element(2), 50); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructChildren) { - const auto field_idx = find_field_idx("struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& struct_column = assert_cast(*column); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(values.get_data_at(0).to_string(), "sa"); - EXPECT_EQ(values.get_data_at(4).to_string(), "se"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedNullableStructChildren) { - const auto field_idx = find_field_idx("nullable_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& nullable_column = assert_cast(*column); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = assert_cast(struct_column.get_column(0)); - const auto& nested_values = assert_cast(values.get_nested_column()); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(nested_values.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(nested_values.get_data_at(3).to_string(), "nsd"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedListStructElementChildren) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& list_schema = *_fields[field_idx]; - ASSERT_EQ(list_schema.name, "nullable_list_struct_col"); - ASSERT_EQ(list_schema.children.size(), 1); - const auto& element_schema = *list_schema.children[0]; - ASSERT_EQ(element_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = list_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex element_projection; - element_projection.index = element_schema.local_id; - element_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = element_schema.children[1]->local_id; - element_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(element_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(list_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* array_type = - assert_cast(remove_nullable(reader->type()).get()); - const auto* element_type = assert_cast( - remove_nullable(array_type->get_nested_type()).get()); - ASSERT_EQ(element_type->get_elements().size(), 1); - EXPECT_EQ(element_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& elements = assert_cast(array_column.get_data()); - const auto& struct_column = assert_cast(elements.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedMapStructValueChildren) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.name, "nullable_map_int_struct_col"); - ASSERT_EQ(map_schema.children.size(), 2); - const auto& value_schema = *map_schema.children[1]; - ASSERT_EQ(value_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = map_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex value_projection; - value_projection.index = value_schema.local_id; - value_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = value_schema.children[1]->local_id; - value_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(value_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* map_type = assert_cast(remove_nullable(reader->type()).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), TYPE_INT); - const auto* value_type = - assert_cast(remove_nullable(map_type->get_value_type()).get()); - ASSERT_EQ(value_type->get_elements().size(), 1); - EXPECT_EQ(value_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - const auto& values = assert_cast(map_column.get_values()); - const auto& struct_column = assert_cast(values.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); -} - -TEST_F(ParquetColumnReaderTest, AllowsMapKeyWithValueProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - const auto& value_schema = *map_schema.children[1]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - projection.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_NE(reader, nullptr); -} - -TEST_F(ParquetColumnReaderTest, RejectMapKeyOnlyProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_FALSE(st.ok()); - EXPECT_NE(st.to_string().find("contains no value"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_list_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "xs"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = assert_cast(xs_array.get_data()); - const auto& values = assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 4); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 5); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructListChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_map_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "kv"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesListChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesMapChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesNestedStructComplexChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_nested_struct_list_col"); - auto reader = create_projected_grandchild_reader(field_idx, 0, 0); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructMapChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadStructListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructListWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(1), 304); - EXPECT_EQ(a_values.get_element(2), 305); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadStructMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(1), 404); - EXPECT_EQ(a_values.get_element(2), 405); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, ReadListStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 1); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 1); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 1); - EXPECT_EQ(inner_offsets[0], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectListListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 5); - EXPECT_EQ(outer_offsets[2], 7); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[6], 7); -} - -TEST_F(ParquetColumnReaderTest, ReadMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SkipPlainBinaryMapThenReadResetsArrowBuilder) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_plain_reader(field_idx); - - // Row 0 contains two STRING values. The levels-only skip must release (and discard) those - // Arrow BinaryRecordReader builder chunks before the next normal read. If they leak into the - // next batch, ParquetLeafReader observes more values than current definition/repetition levels. - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 3); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(map_column.get_offsets()[1], 0); - EXPECT_EQ(map_column.get_offsets()[2], 1); - const auto& values = get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(values.size(), 1); - EXPECT_EQ(values.get_data_at(0).to_string(), "cc"); -} - -TEST_F(ParquetColumnReaderTest, SelectMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectMapStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 2); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectMapListWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 4); - EXPECT_EQ(map_offsets[2], 5); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(4)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepListStructMapListAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepListStructMapListThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 3); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_FALSE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 3); - EXPECT_TRUE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_FALSE(map_values.is_null_at(2)); - - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 2); - EXPECT_EQ(keys.get_element(0), 3); - EXPECT_EQ(keys.get_element(1), 4); - const auto& lists = assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 2); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepListStructMapList) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepMapListMapAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepMapListMapThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 3); - EXPECT_EQ(outer_keys.get_element(0), 30); - EXPECT_EQ(outer_keys.get_element(1), 40); - EXPECT_EQ(outer_keys.get_element(2), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 3); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_FALSE(lists.is_null_at(2)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 3); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); - EXPECT_EQ(list_offsets[2], 3); - const auto& inner_maps = assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 3); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_TRUE(inner_maps.is_null_at(1)); - EXPECT_FALSE(inner_maps.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepMapListMap) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp deleted file mode 100644 index 0d0f9a2f8567cc..00000000000000 --- a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp +++ /dev/null @@ -1,506 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" - -namespace doris::format::parquet { -namespace { - -std::shared_ptr fixed_binary_array(const std::vector& values, - int byte_width) { - auto type = arrow::fixed_size_binary(byte_width); - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetLeafReader make_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type) { - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr); -} - -struct CapturedDecodedView { - DecodedValueKind value_kind = DecodedValueKind::INT32; - DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; - int64_t row_count = 0; - int decimal_precision = -1; - int decimal_scale = -1; - int fixed_length = -1; - bool timestamp_is_adjusted_to_utc = false; - bool enable_strict_mode = false; - const cctz::time_zone* timezone = nullptr; - bool null_map_is_null = true; - std::vector null_map; - std::vector fixed_values; - std::vector binary_values; - std::vector owned_binary_values; -}; - -ParquetLeafReader make_spy_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type, - CapturedDecodedView* captured, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false) { - auto appender = [captured](MutableColumnPtr&, const DecodedColumnView& view) { - captured->value_kind = view.value_kind; - captured->time_unit = view.time_unit; - captured->row_count = view.row_count; - captured->decimal_precision = view.decimal_precision; - captured->decimal_scale = view.decimal_scale; - captured->fixed_length = view.fixed_length; - captured->timestamp_is_adjusted_to_utc = view.timestamp_is_adjusted_to_utc; - captured->enable_strict_mode = view.enable_strict_mode; - captured->timezone = view.timezone; - captured->null_map_is_null = view.null_map == nullptr; - captured->null_map.clear(); - if (view.null_map != nullptr) { - captured->null_map.assign(view.null_map, view.null_map + view.row_count); - } - captured->fixed_values.clear(); - if (view.values != nullptr && view.value_kind == DecodedValueKind::INT64) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 8); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::FLOAT) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::INT32) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } - captured->binary_values.clear(); - captured->owned_binary_values.clear(); - if (view.binary_values != nullptr) { - captured->owned_binary_values.reserve(view.binary_values->size()); - for (const auto& value : *view.binary_values) { - captured->owned_binary_values.emplace_back( - value.data == nullptr ? std::string() - : std::string(value.data, value.size)); - } - captured->binary_values.reserve(captured->owned_binary_values.size()); - for (const auto& value : captured->owned_binary_values) { - captured->binary_values.emplace_back(value.data(), value.size()); - } - } - return Status::OK(); - }; - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr, {}, timezone, - enable_strict_mode, std::move(appender)); -} - -} // namespace - -struct ParquetLeafReaderTestAccess { - static ParquetLeafBatch make_fixed_batch(const std::vector& def_levels, - const std::vector& rep_levels, - const std::vector& values, - bool read_dense_for_nullable = false) { - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._consumed_level_count = static_cast(def_levels.size()); - batch._decoded_level_count = static_cast(def_levels.size()); - batch._values_written = static_cast(values.size()); - batch._def_levels = def_levels.data(); - batch._rep_levels = rep_levels.data(); - batch._fixed_values = reinterpret_cast(values.data()); - batch._read_dense_for_nullable = read_dense_for_nullable; - return batch; - } - - static Status build_nested_batch(const ParquetLeafReader& reader, - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, - int16_t value_slot_repetition_level, - ParquetNestedScalarBatch* nested_batch) { - return reader.build_nested_batch_from_leaf_batch(leaf_batch, records_read, - value_slot_definition_level, nested_batch, - value_slot_repetition_level); - } -}; - -std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, - int16_t max_repetition_level) { - auto node = ::parquet::schema::PrimitiveNode::Make("leaf", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32); - return std::make_shared<::parquet::ColumnDescriptor>(node, max_definition_level, - max_repetition_level); -} - -ParquetLeafReader make_nested_leaf_reader( - const std::shared_ptr<::parquet::ColumnDescriptor>& descriptor, DataTypePtr type) { - ParquetTypeDescriptor type_descriptor; - type_descriptor.physical_type = ::parquet::Type::INT32; - type_descriptor.doris_type = type; - return ParquetLeafReader(descriptor.get(), type_descriptor, std::move(type), "nested_leaf", - nullptr); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesAreSpacedBeforeSerde) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30, 50}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - ASSERT_EQ(nullable.size(), 5); - EXPECT_FALSE(nullable.is_null_at(0)); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_FALSE(nullable.is_null_at(2)); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_FALSE(nullable.is_null_at(4)); - const auto& nested = assert_cast(nullable.get_nested_column()); - EXPECT_EQ(nested.get_element(0), 10); - EXPECT_EQ(nested.get_element(2), 30); - EXPECT_EQ(nested.get_element(4), 50); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesRejectCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet values"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, Float16BinaryValuesAreConvertedToFloat) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - descriptor.fixed_length = 2; - auto type = std::make_shared(); - auto reader = make_leaf_reader(descriptor, type); - - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::FIXED_BINARY; - batch._binary_chunks = {fixed_binary_array( - {half(0x0000), half(0x8000), half(0x3E00), half(0x0001), half(0x7E00)}, 2)}; - batch._values_written = 5; - - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, nullptr, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& floats = assert_cast(*column); - ASSERT_EQ(floats.size(), 5); - EXPECT_FLOAT_EQ(floats.get_element(0), 0.0F); - EXPECT_TRUE(std::signbit(floats.get_element(1))); - EXPECT_FLOAT_EQ(floats.get_element(2), 1.5F); - EXPECT_NEAR(floats.get_element(3), 5.9604645e-8F, 1e-12F); - EXPECT_TRUE(std::isnan(floats.get_element(4))); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("aa").ok()); - ASSERT_TRUE(builder.Append("cc").ok()); - ASSERT_TRUE(builder.Append("ee").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 3; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - const auto& strings = assert_cast(nullable.get_nested_column()); - ASSERT_EQ(nullable.size(), 5); - EXPECT_EQ(strings.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_EQ(strings.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("only_one").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 1; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 3, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), - std::string::npos); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT64; - descriptor.time_unit = ParquetTimeUnit::NANOS; - descriptor.decimal_precision = 18; - descriptor.decimal_scale = 4; - descriptor.fixed_length = 12; - descriptor.timestamp_is_adjusted_to_utc = true; - auto type = make_nullable(std::make_shared()); - cctz::time_zone shanghai; - ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); - - CapturedDecodedView captured; - auto reader = make_spy_leaf_reader(descriptor, type, &captured, &shanghai, true); - const std::vector values = {100, 200, 300}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT64; - batch._fixed_values = reinterpret_cast(values.data()); - batch._values_written = values.size(); - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, &null_map, column).ok()); - EXPECT_EQ(captured.value_kind, DecodedValueKind::INT64); - EXPECT_EQ(captured.time_unit, DecodedTimeUnit::NANOS); - EXPECT_EQ(captured.row_count, 3); - EXPECT_EQ(captured.decimal_precision, 18); - EXPECT_EQ(captured.decimal_scale, 4); - EXPECT_EQ(captured.fixed_length, 12); - EXPECT_TRUE(captured.timestamp_is_adjusted_to_utc); - EXPECT_TRUE(captured.enable_strict_mode); - EXPECT_EQ(captured.timezone, &shanghai); - EXPECT_FALSE(captured.null_map_is_null); - EXPECT_EQ(captured.null_map, std::vector({0, 1, 0})); - - auto required_column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, nullptr, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); - - const NullMap empty_null_map; - ASSERT_TRUE(reader.append_values(batch, 3, &empty_null_map, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCapturesBinaryFixedLengthAndFloat16Override) { - ParquetTypeDescriptor binary_descriptor; - binary_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - binary_descriptor.fixed_length = 4; - auto type = std::make_shared(); - - CapturedDecodedView binary_view; - auto binary_reader = make_spy_leaf_reader(binary_descriptor, type, &binary_view); - ParquetLeafBatch binary_batch; - binary_batch._value_kind = DecodedValueKind::FIXED_BINARY; - binary_batch._binary_chunks = {fixed_binary_array({"abcd", "wxyz"}, 4)}; - binary_batch._values_written = 2; - auto binary_column = type->create_column(); - ASSERT_TRUE(binary_reader.append_values(binary_batch, 2, nullptr, binary_column).ok()); - EXPECT_EQ(binary_view.value_kind, DecodedValueKind::FIXED_BINARY); - EXPECT_EQ(binary_view.fixed_length, 4); - ASSERT_EQ(binary_view.owned_binary_values.size(), 2); - EXPECT_EQ(binary_view.owned_binary_values[0], "abcd"); - EXPECT_EQ(binary_view.owned_binary_values[1], "wxyz"); - - ParquetTypeDescriptor float16_descriptor; - float16_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - float16_descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - float16_descriptor.fixed_length = 2; - CapturedDecodedView float16_view; - auto float16_reader = make_spy_leaf_reader(float16_descriptor, - std::make_shared(), &float16_view); - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - ParquetLeafBatch float16_batch; - float16_batch._value_kind = DecodedValueKind::FIXED_BINARY; - float16_batch._binary_chunks = {fixed_binary_array({half(0x3E00), half(0x4000)}, 2)}; - float16_batch._values_written = 2; - auto float16_column = std::make_shared()->create_column(); - ASSERT_TRUE(float16_reader.append_values(float16_batch, 2, nullptr, float16_column).ok()); - EXPECT_EQ(float16_view.value_kind, DecodedValueKind::FLOAT); - ASSERT_EQ(float16_view.fixed_values.size(), sizeof(float) * 2); - const auto* floats = reinterpret_cast(float16_view.fixed_values.data()); - EXPECT_FLOAT_EQ(floats[0], 1.5F); - EXPECT_FLOAT_EQ(floats[1], 2.0F); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLevels) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 2, 2}; - const std::vector rep_levels = {0, 1, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 2, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.records_read, 2); - EXPECT_EQ(nested_batch.levels_written, 3); - EXPECT_EQ(nested_batch.value_indices, std::vector({0, 1, 2})); - const auto& nested_values = assert_cast(*nested_batch.values_column); - ASSERT_EQ(nested_values.size(), 3); - EXPECT_EQ(nested_values.get_element(0), 10); - EXPECT_EQ(nested_values.get_element(2), 30); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutValueSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 777, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 2, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLeafValues) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 1, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutPayloadSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {1, 2, 0, 2}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {777, 10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({-1, 1, -1, 2})); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsMismatchedValueLayout) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 0, 2, 0}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("inconsistent value count"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsDenseNullable) { - auto descriptor = int32_column_descriptor(1, 0); - auto reader = - make_nested_leaf_reader(descriptor, make_nullable(std::make_shared())); - const std::vector def_levels = {1}; - const std::vector rep_levels = {0}; - const std::vector values = {10}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values, true); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 1, 0, 0, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Dense nullable parquet nested reader is not supported"), - std::string::npos); -} - -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index fac89b31c422d8..cb54cab86b27d8 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -26,95 +26,6 @@ namespace doris::format::parquet { namespace { -void expect_plan_entry(const ParquetPageCacheReadPlanEntry& entry, - const ParquetPageCacheRange& cached_range, int64_t copy_offset_in_cache, - int64_t output_offset, int64_t copy_size) { - EXPECT_EQ(entry.cached_range.offset, cached_range.offset); - EXPECT_EQ(entry.cached_range.size, cached_range.size); - EXPECT_EQ(entry.copy_offset_in_cache, copy_offset_in_cache); - EXPECT_EQ(entry.output_offset, output_offset); - EXPECT_EQ(entry.copy_size, copy_size); -} - -TEST(ParquetPageCacheRangeTest, SubsetRequestHitsSingleCachedRange) { - const std::vector cached_ranges = { - {100, 100}, - }; - - // Request [120, 150) is fully inside cached [100, 200). The reader should lookup - // the exact cached key [100, 200), then copy from cached offset 20 into output offset 0. - auto plan = detail::plan_page_cache_range_read(120, 30, cached_ranges); - - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestHitsMultipleAdjacentCachedRanges) { - const std::vector cached_ranges = { - {180, 80}, - {100, 80}, - }; - - // Request [100, 260) is larger than either cached entry, but the two cached ranges - // exactly cover it. The copy plan stitches the two exact cache entries together. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 80}, 0, 0, 80); - expect_plan_entry(plan[1], {180, 80}, 0, 80, 80); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestCanUseOverlappingCachedRanges) { - const std::vector cached_ranges = { - {150, 110}, - {100, 100}, - }; - - // Request [100, 260) is covered by overlapping cached ranges. The first copy uses - // [100, 200); the second resumes at cursor 200 and copies the tail from [150, 260). - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 100}, 0, 0, 100); - expect_plan_entry(plan[1], {150, 110}, 50, 100, 60); -} - -TEST(ParquetPageCacheRangeTest, PartialOverlapWithoutFullCoverageMisses) { - const std::vector cached_ranges = { - {100, 80}, - {200, 60}, - }; - - // Cached ranges cover [100, 180) and [200, 260), but [180, 200) is missing. - // The caller must read the whole request from the file instead of returning - // a partially cached result. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, NonCoveringAndInvalidRangesAreIgnored) { - const std::vector cached_ranges = { - {50, 20}, {100, 0}, {100, -1}, {180, 20}, {120, 30}, - }; - - // Only [120, 150) intersects the request, but it does not cover the request start - // [100, 120), so this is still a miss. - auto plan = detail::plan_page_cache_range_read(100, 50, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { - const std::vector cached_ranges = { - {100, 100}, - }; - - EXPECT_TRUE(detail::plan_page_cache_range_read(-1, 10, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, 0, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); -} - TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { const std::vector ranges = { {100, 50}, @@ -134,6 +45,19 @@ TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) EXPECT_EQ(valid_ranges[1].size, 60); } +TEST(ParquetPageCacheRangeTest, SerializedIndexesAreBoundedIndividuallyAndWhenCoalesced) { + constexpr size_t file_size = 1ULL << 30; + const auto budget = detail::MAX_SERIALIZED_PARQUET_INDEX_BYTES; + + EXPECT_TRUE(detail::is_serialized_index_range_safe(file_size, 0, budget)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, 0, budget + 1)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, -1, 1)); + + EXPECT_TRUE(detail::is_serialized_index_span_safe(100, 100 + budget)); + // Individually small adjacent indexes must not combine into one unbounded allocation. + EXPECT_FALSE(detail::is_serialized_index_span_safe(100, 101 + budget)); +} + TEST(ParquetPageCacheRangeTest, AveragePrefetchRangeSizeUsesOnlyValidRanges) { const std::vector ranges = { {0, 512}, diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 2fd85cefd49ac9..75e6545906f263 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -5,9 +5,7 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// // http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,188 +13,49 @@ // specific language governing permissions and limitations // under the License. -#include #include -#include #include #include #include #include -#include #include #include "common/consts.h" #include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" +#include "core/block/block.h" #include "core/column/column_string.h" -#include "core/column/column_struct.h" #include "core/column/column_vector.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" #include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "format_v2/parquet/selection_vector.h" #include "storage/utils.h" namespace doris::format::parquet { namespace { -ParquetColumnSchema int64_schema(std::string name = "mock") { +ParquetColumnSchema int64_schema() { ParquetColumnSchema schema; schema.local_id = 0; - schema.name = std::move(name); + schema.name = "mock"; schema.type = std::make_shared(); return schema; } -ParquetColumnSchema nested_int64_schema(std::string name, int16_t nullable_definition_level, - int16_t definition_level, int16_t repetition_level = 0, - int16_t repeated_ancestor_definition_level = 0) { - ParquetColumnSchema schema = int64_schema(std::move(name)); - schema.type = make_nullable(std::make_shared()); - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - return schema; -} - -ParquetColumnSchema nested_struct_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "struct"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})); - return schema; -} - -ParquetColumnSchema nested_list_schema(std::string name, DataTypePtr element_type, - int16_t nullable_definition_level, int16_t definition_level, - int16_t repetition_level, - int16_t repeated_ancestor_definition_level) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.kind = ParquetColumnSchemaKind::LIST; - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - schema.type = make_nullable(std::make_shared(std::move(element_type))); - return schema; -} - -ParquetColumnSchema nested_map_schema( - DataTypePtr value_type = make_nullable(std::make_shared())) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "map"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.repetition_level = 1; - schema.repeated_ancestor_definition_level = 2; - schema.type = make_nullable(std::make_shared( - make_nullable(std::make_shared()), std::move(value_type))); - return schema; -} - -ParquetColumnSchema bare_repeated_int64_list_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "repeated"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.definition_level = 1; - schema.repetition_level = 1; - schema.repeated_repetition_level = 1; - schema.repeated_ancestor_definition_level = 1; - schema.type = std::make_shared(std::make_shared()); - return schema; -} - -std::unique_ptr primitive_child(int local_id, std::string name, - DataTypePtr type) { - auto child = std::make_unique(); - child->local_id = local_id; - child->name = std::move(name); - child->kind = ParquetColumnSchemaKind::PRIMITIVE; - child->leaf_column_id = local_id; - child->type = std::move(type); - child->type_descriptor.physical_type = ::parquet::Type::INT32; - child->type_descriptor.doris_type = child->type; - return child; -} - -ParquetColumnSchema struct_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "s"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.children.push_back(primitive_child(0, "a", std::make_shared())); - schema.children.push_back(primitive_child(1, "b", std::make_shared())); - DataTypes types = {make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)}; - Strings names = {"a", "b"}; - schema.type = std::make_shared(types, names); - return schema; -} - -ParquetColumnSchema list_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "xs"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.children.push_back(primitive_child(0, "element", std::make_shared())); - schema.type = std::make_shared(schema.children[0]->type); - return schema; -} - -ParquetColumnSchema map_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "m"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.children.push_back(primitive_child(0, "key", std::make_shared())); - schema.children.push_back(primitive_child(1, "value", std::make_shared())); - schema.type = std::make_shared(make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)); - return schema; -} - class CursorColumnReader final : public ParquetColumnReader { public: CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("invalid mock read arguments"); - } - auto* values = assert_cast(column.get()); + DORIS_CHECK(column); + DORIS_CHECK(rows_read != nullptr); + auto& values = assert_cast(*column); for (int64_t row = 0; row < rows; ++row) { - values->insert_value(_cursor + row); + values.insert_value(_cursor + row); } _read_lengths.push_back(rows); _cursor += rows; @@ -205,271 +64,33 @@ class CursorColumnReader final : public ParquetColumnReader { } Status skip(int64_t rows) override { + DORIS_CHECK(rows >= 0); _skip_lengths.push_back(rows); _cursor += rows; return Status::OK(); } + void flush_profile() override { ++_profile_flushes; } + bool crossed_page_since_last_batch() override { + ++_page_crossing_checks; + return _crossed_page; + } + + void set_crossed_page(bool crossed_page) { _crossed_page = crossed_page; } + int64_t cursor() const { return _cursor; } const std::vector& skip_lengths() const { return _skip_lengths; } const std::vector& read_lengths() const { return _read_lengths; } + int profile_flushes() const { return _profile_flushes; } + int page_crossing_checks() const { return _page_crossing_checks; } private: int64_t _cursor = 0; std::vector _skip_lengths; std::vector _read_lengths; -}; - -class ScriptedNestedReader final : public ParquetColumnReader { -public: - ScriptedNestedReader(ParquetColumnSchema schema, DataTypePtr type, - std::vector def_levels, std::vector rep_levels, - bool has_repeated_child = false, bool build_nulls = false) - : ParquetColumnReader(schema, std::move(type)), - _def_levels(std::move(def_levels)), - _rep_levels(std::move(rep_levels)), - _has_repeated_child(has_repeated_child), - _build_nulls(build_nulls) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - _build_lengths.push_back(length_upper_bound); - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("invalid scripted nested build arguments"); - } - for (int64_t row = 0; row < length_upper_bound; ++row) { - insert_value(column, _next_value++, _build_nulls); - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - set_nested_build_level_cursor(std::min(nested_build_level_cursor() + length_upper_bound, - static_cast(_def_levels.size()))); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return _has_repeated_child; } - - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - -private: - static void insert_value(MutableColumnPtr& column, int64_t value, bool is_null) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - if (is_null) { - nullable_column->insert_default(); - return; - } - assert_cast(nullable_column->get_nested_column()).insert_value(value); - nullable_column->get_null_map_data().push_back(0); - return; - } - assert_cast(*column).insert_value(value); - } - - std::vector _def_levels; - std::vector _rep_levels; - bool _has_repeated_child = false; - bool _build_nulls = false; - int64_t _next_value = 0; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; -}; - -class ChunkedNestedLeafReader final : public ParquetColumnReader { -public: - ChunkedNestedLeafReader() - : ParquetColumnReader(nested_int64_schema("element", 0, 1, 1, 1), - std::make_shared()) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(values_read != nullptr); - _initial_column_sizes.push_back(column->size()); - _build_lengths.push_back(length_upper_bound); - if (auto* nullable = check_and_get_column(*column); nullable != nullptr) { - auto& values = assert_cast(nullable->get_nested_column()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values.insert_value(row); - nullable->get_null_map_data().push_back(0); - } - } else { - auto* values = assert_cast(column.get()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values->insert_value(row); - } - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return true; } - - const std::vector& load_lengths() const { return _load_lengths; } - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - const std::vector& initial_column_sizes() const { return _initial_column_sizes; } - -private: - std::vector _def_levels; - std::vector _rep_levels; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; - std::vector _initial_column_sizes; -}; - -} // namespace - -struct ScalarColumnReaderTestAccess { - static void set_nested_batch(ScalarColumnReader* reader, - std::unique_ptr batch) { - reader->_nested_batch = std::move(batch); - } - - static int64_t page_filtered_rows_to_skip(const ScalarColumnReader& reader, int64_t rows) { - return reader.page_filtered_rows_to_skip(rows); - } - - static void set_row_group_rows_read(ScalarColumnReader* reader, int64_t rows) { - reader->_row_group_rows_read = rows; - } - - static Status append_dictionary_filtered_values( - const ScalarColumnReader& reader, - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { - return reader.append_dictionary_filtered_values(chunks, dictionary_filter, column, - row_filter, matched_rows, used_filter); - } -}; - -namespace { - -ParquetColumnSchema string_schema(std::string name = "string") { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.type = std::make_shared(); - schema.type_descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - schema.type_descriptor.doris_type = schema.type; - return schema; -} - -std::shared_ptr<::arrow::Array> dictionary_array(const std::vector& indices, - const std::vector& values) { - ::arrow::Int8Builder index_builder; - EXPECT_TRUE(index_builder.AppendValues(indices).ok()); - auto index_result = index_builder.Finish(); - EXPECT_TRUE(index_result.ok()) << index_result.status(); - - ::arrow::StringBuilder dictionary_builder; - EXPECT_TRUE(dictionary_builder.AppendValues(values).ok()); - auto dictionary_result = dictionary_builder.Finish(); - EXPECT_TRUE(dictionary_result.ok()) << dictionary_result.status(); - - auto result = ::arrow::DictionaryArray::FromArrays( - ::arrow::dictionary(::arrow::int8(), ::arrow::utf8()), *index_result, - *dictionary_result); - EXPECT_TRUE(result.ok()) << result.status(); - return *result; -} - -std::unique_ptr make_scripted_scalar_reader( - ParquetColumnSchema schema, std::unique_ptr batch) { - auto reader = std::make_unique(schema, nullptr); - ScalarColumnReaderTestAccess::set_nested_batch(reader.get(), std::move(batch)); - return reader; -} - -std::unique_ptr scalar_batch(std::vector def_levels, - std::vector rep_levels, - std::vector value_indices, - std::vector values) { - auto batch = std::make_unique(); - batch->levels_written = static_cast(def_levels.size()); - batch->def_levels = std::move(def_levels); - batch->rep_levels = std::move(rep_levels); - batch->value_indices = std::move(value_indices); - auto column = ColumnInt64::create(); - for (const auto value : values) { - column->insert_value(value); - } - batch->values_column = std::move(column); - return batch; -} - -class DefaultOnlyReader final : public ParquetColumnReader { -public: - DefaultOnlyReader() - : ParquetColumnReader(int64_schema("default_only"), std::make_shared()) { - } - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } + int _profile_flushes = 0; + bool _crossed_page = false; + int _page_crossing_checks = 0; }; GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { @@ -480,21 +101,6 @@ GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { return location; } -TEST(ParquetScalarColumnReaderTest, DictionaryIndexOutsideFilterIsCorruption) { - ScalarColumnReader reader(string_schema("dictionary_value"), nullptr); - MutableColumnPtr column = ColumnString::create(); - IColumn::Filter row_filter; - int64_t matched_rows = 0; - bool used_filter = false; - const std::vector> chunks = { - dictionary_array({0, 1}, {"keep", "out-of-range"})}; - - const auto status = ScalarColumnReaderTestAccess::append_dictionary_filtered_values( - reader, chunks, IColumn::Filter {1}, column, &row_filter, &matched_rows, &used_filter); - EXPECT_EQ(ErrorCode::CORRUPTION, status.code()) << status; - EXPECT_NE(status.to_string().find("Invalid parquet dictionary index 1"), std::string::npos); -} - } // namespace TEST(SelectionVectorTest, IdentitySelectionToRanges) { @@ -520,6 +126,21 @@ TEST(SelectionVectorTest, ExternalBufferSelectionToRanges) { EXPECT_TRUE(selection.verify(std::size(indices), 8).ok()); } +TEST(SelectionVectorTest, OutputRangesReuseCapacity) { + SelectionVector::Index indices[] = {1, 2, 5}; + SelectionVector selection(indices, std::size(indices)); + std::vector ranges; + ranges.reserve(8); + const auto retained_capacity = ranges.capacity(); + + selection_to_ranges(selection, std::size(indices), &ranges); + ASSERT_EQ(ranges.size(), 2); + EXPECT_EQ(ranges.capacity(), retained_capacity); + selection_to_ranges(selection, 1, &ranges); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges.capacity(), retained_capacity); +} + TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { SelectionVector selection(2); EXPECT_FALSE(selection.verify(3, 3).ok()); @@ -534,6 +155,28 @@ TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { EXPECT_FALSE(selection.verify(2, 3).ok()); } +TEST(SelectionVectorTest, MaterializedFilterIsReusedUntilSelectionChanges) { + SelectionVector selection(4); + selection.set_index(0, 1); + selection.set_index(1, 3); + const uint8_t* first_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &first_filter).ok()); + ASSERT_NE(first_filter, nullptr); + EXPECT_EQ(std::vector(first_filter, first_filter + 4), + std::vector({0, 1, 0, 1})); + + const uint8_t* reused_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &reused_filter).ok()); + EXPECT_EQ(reused_filter, first_filter); + + selection.set_index(1, 2); + const uint8_t* updated_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &updated_filter).ok()); + EXPECT_EQ(updated_filter, first_filter); + EXPECT_EQ(std::vector(updated_filter, updated_filter + 4), + std::vector({0, 1, 1, 0})); +} + TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); @@ -559,571 +202,67 @@ TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { SelectionVector selection; auto column = std::make_shared()->create_column(); ASSERT_TRUE(reader.select(selection, 0, 4, column).ok()); - EXPECT_EQ(column->size(), 0); + EXPECT_TRUE(column->empty()); EXPECT_EQ(reader.cursor(), 4); EXPECT_TRUE(reader.read_lengths().empty()); EXPECT_EQ(reader.skip_lengths(), std::vector({4})); } -TEST(ParquetColumnReaderControlTest, BaseNestedDefaultsAndSkipNested) { - DefaultOnlyReader base_reader; - EXPECT_FALSE(base_reader.skip(1).ok()); - EXPECT_FALSE(base_reader.load_nested_batch(1).ok()); - - auto column = std::make_shared()->create_column(); - int64_t values_read = 0; - EXPECT_FALSE(base_reader.build_nested_column(1, column, &values_read).ok()); - - int64_t values_consumed = 0; - EXPECT_FALSE(base_reader.consume_nested_column(1, &values_consumed).ok()); -} - -TEST(ParquetColumnReaderControlTest, NestedSkipConsumesBoundedBatchesWithoutMaterializing) { - auto element_reader = std::make_unique(); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - ASSERT_TRUE(reader.skip(8193).ok()); - EXPECT_TRUE(element_reader_ptr->load_lengths().empty()); - EXPECT_EQ(element_reader_ptr->level_load_lengths(), std::vector({4096, 4096, 1})); - EXPECT_EQ(element_reader_ptr->consume_lengths(), std::vector({4096, 4096, 1})); - EXPECT_TRUE(element_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(element_reader_ptr->initial_column_sizes().empty()); -} - -TEST(ParquetColumnReaderControlTest, MapSkipConsumesBothStreamsWithoutMaterializing) { - const std::vector def_levels {3, 3, 1, 3}; - const std::vector rep_levels {0, 1, 0, 0}; - auto key_reader = std::make_unique( - nested_int64_schema("key", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* key_reader_ptr = key_reader.get(); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(key_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(key_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_TRUE(key_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(value_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, StructSkipConsumesNullSeparatedChildSpans) { - const std::vector def_levels {2, 0, 2}; - const std::vector rep_levels {0, 0, 0}; - auto shape_reader = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* shape_reader_ptr = shape_reader.get(); - auto child_reader = std::make_unique( - nested_int64_schema("child", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* child_reader_ptr = child_reader.get(); - std::vector> children; - children.push_back(std::move(shape_reader)); - children.push_back(std::move(child_reader)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {-1, 0}); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(shape_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(child_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(shape_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_EQ(child_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_TRUE(shape_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(child_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipConsumesRecursivelyWithoutMaterializing) { - auto leaf_reader = std::make_unique( - nested_int64_schema("leaf", 0, 1, 1, 1), std::make_shared(), - std::vector {1, 1}, std::vector {0, 0}); - auto* leaf_reader_ptr = leaf_reader.get(); - const auto inner_type = std::make_shared(std::make_shared()); - auto inner_reader = std::make_unique(bare_repeated_int64_list_schema(), - inner_type, std::move(leaf_reader)); - const auto outer_type = std::make_shared(inner_type); - ListColumnReader reader(bare_repeated_int64_list_schema(), outer_type, std::move(inner_reader)); - - ASSERT_TRUE(reader.skip(2).ok()); - EXPECT_EQ(leaf_reader_ptr->level_load_lengths(), std::vector({2})); - EXPECT_EQ(leaf_reader_ptr->consume_lengths(), std::vector({2})); - EXPECT_TRUE(leaf_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedMaterializerHelpersAppendOffsetsAndParentNulls) { - ColumnArray::Offsets64 offsets; - append_offsets(offsets, {3, 0, 2}); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 3); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 5); - append_offsets(offsets, {1, 4}); - ASSERT_EQ(offsets.size(), 5); - EXPECT_EQ(offsets[3], 6); - EXPECT_EQ(offsets[4], 10); - - const NullMap parent_nulls = {0, 1, 0}; - append_parent_nulls(nullptr, parent_nulls); - NullMap dst = {1}; - append_parent_nulls(&dst, parent_nulls); - EXPECT_EQ(dst, NullMap({1, 0, 1, 0})); -} - -TEST(ParquetColumnReaderControlTest, PageFilteredRowsToSkipUsesOnlyFullSkippedRanges) { - ParquetPageSkipPlan page_skip_plan; - page_skip_plan.skipped_ranges = {RowRange {0, 3}, RowRange {5, 2}, RowRange {10, 4}}; - - auto schema = nested_int64_schema("page_filtered", 0, 0); - ScalarColumnReader reader(schema, nullptr, &page_skip_plan); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 3), 3); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 3); - - ScalarColumnReaderTestAccess::set_row_group_rows_read(&reader, 5); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 2), 2); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 2); -} - -TEST(ParquetColumnReaderControlTest, StructSkipsNullParentForRepeatedChildAndBatchesPresentRows) { - auto repeated_child = std::make_unique( - nested_int64_schema("repeated_shape", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2, 2, 2}, - std::vector {0, 0, 0, 0}, true); - auto* repeated_child_ptr = repeated_child.get(); - auto scalar_child = make_scripted_scalar_reader( - nested_int64_schema("scalar_child", 1, 2), - scalar_batch({2, 0, 2, 2}, {0, 0, 0, 0}, {0, -1, 1, 2}, {10, 20, 30})); - auto* scalar_child_ptr = scalar_child.get(); - - std::vector> children; - children.push_back(std::move(repeated_child)); - children.push_back(std::move(scalar_child)); - StructColumnReader reader(nested_struct_schema(), - make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})), - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_EQ(repeated_child_ptr->build_lengths(), std::vector({1, 2})); - EXPECT_EQ(scalar_child_ptr->nested_build_level_cursor(), 4); -} - -TEST(ParquetColumnReaderControlTest, StructFallsBackToFirstChildWhenAllChildrenAreRepeated) { - auto first_child = std::make_unique( - nested_int64_schema("first", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 0}, std::vector {0, 0}, true); - auto second_child = std::make_unique( - nested_int64_schema("second", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2}, - std::vector {0, 0}, true); - - std::vector> children; - children.push_back(std::move(first_child)); - children.push_back(std::move(second_child)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(rows_read, 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesComplexChildShapeOnly) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 2, 0, 0, 2, 2}, std::vector {0, 0, 0, 0, 0, 0}); - - ParquetColumnSchema map_schema = nested_map_schema(); - map_schema.nullable_definition_level = 2; - map_schema.definition_level = 3; - map_schema.repeated_ancestor_definition_level = 0; - auto key_reader = std::make_unique( - nested_int64_schema("key", 3, 3, 1, 0), - make_nullable(std::make_shared()), - std::vector {3, 3, 0, 0, 3, 3}, std::vector {0, 0, 0, 0, 0, 0}); - auto value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 4, 4, 1, 0), - scalar_batch({4, 4, 0, 0, 4, 4}, {0, 0, 0, 0, 0, 0}, - {0, 1, -1, -1, 2, 3}, {10, 20, 30, 40})); - auto map_reader = std::make_unique( - map_schema, map_schema.type, std::move(key_reader), std::move(value_reader)); - - std::vector> children; - children.push_back(std::move(shape_child)); - children.push_back(std::move(map_reader)); - auto struct_type = make_nullable(std::make_shared(DataTypes {map_schema.type}, - Strings {"partitionValues"})); - StructColumnReader reader(nested_struct_schema(), struct_type, std::move(children), {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(6, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 6); - - const auto& nullable_struct = assert_cast(*column); - ASSERT_EQ(nullable_struct.size(), 6); - EXPECT_FALSE(nullable_struct.is_null_at(0)); - EXPECT_FALSE(nullable_struct.is_null_at(1)); - EXPECT_TRUE(nullable_struct.is_null_at(2)); - EXPECT_TRUE(nullable_struct.is_null_at(3)); - EXPECT_FALSE(nullable_struct.is_null_at(4)); - EXPECT_FALSE(nullable_struct.is_null_at(5)); - - const auto& struct_column = - assert_cast(nullable_struct.get_nested_column()); - const auto& map_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_nullable.size(), 6); - EXPECT_FALSE(map_nullable.is_null_at(0)); - EXPECT_FALSE(map_nullable.is_null_at(1)); - EXPECT_TRUE(map_nullable.is_null_at(2)); - EXPECT_TRUE(map_nullable.is_null_at(3)); - EXPECT_FALSE(map_nullable.is_null_at(4)); - EXPECT_FALSE(map_nullable.is_null_at(5)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 6); - EXPECT_EQ(map_column.get_offsets()[0], 1); - EXPECT_EQ(map_column.get_offsets()[1], 2); - EXPECT_EQ(map_column.get_offsets()[2], 2); - EXPECT_EQ(map_column.get_offsets()[3], 2); - EXPECT_EQ(map_column.get_offsets()[4], 3); - EXPECT_EQ(map_column.get_offsets()[5], 4); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesNestedStructDescendants) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 0, 2}, std::vector {0, 0, 0}); - - auto id_batch = scalar_batch({4, 3, 4}, {0, 0, 0}, {0, -1, 1}, {10, 20}); - id_batch->value_slot_definition_level = 3; - auto id_reader = - make_scripted_scalar_reader(nested_int64_schema("id", 3, 4), std::move(id_batch)); - - ParquetColumnSchema inner_schema; - inner_schema.local_id = 0; - inner_schema.name = "stats_parsed"; - inner_schema.kind = ParquetColumnSchemaKind::STRUCT; - inner_schema.nullable_definition_level = 2; - inner_schema.definition_level = 3; - inner_schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared())}, Strings {"id"})); - - std::vector> inner_children; - inner_children.push_back(std::move(id_reader)); - auto inner_reader = std::make_unique( - inner_schema, inner_schema.type, std::move(inner_children), std::vector {0}); - - std::vector> outer_children; - outer_children.push_back(std::move(shape_child)); - outer_children.push_back(std::move(inner_reader)); - auto outer_type = make_nullable(std::make_shared(DataTypes {inner_schema.type}, - Strings {"stats_parsed"})); - StructColumnReader reader(nested_struct_schema(), outer_type, std::move(outer_children), - {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& outer_nullable = assert_cast(*column); - ASSERT_EQ(outer_nullable.size(), 3); - EXPECT_FALSE(outer_nullable.is_null_at(0)); - EXPECT_TRUE(outer_nullable.is_null_at(1)); - EXPECT_FALSE(outer_nullable.is_null_at(2)); - - const auto& outer_struct = assert_cast(outer_nullable.get_nested_column()); - const auto& inner_nullable = assert_cast(outer_struct.get_column(0)); - ASSERT_EQ(inner_nullable.size(), 3); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_TRUE(inner_nullable.is_null_at(1)); - EXPECT_FALSE(inner_nullable.is_null_at(2)); - - const auto& inner_struct = assert_cast(inner_nullable.get_nested_column()); - const auto& id_nullable = assert_cast(inner_struct.get_column(0)); - const auto& id_values = assert_cast(id_nullable.get_nested_column()); - EXPECT_EQ(id_values.get_element(0), 10); - EXPECT_EQ(id_values.get_element(2), 20); -} - -TEST(ParquetColumnReaderControlTest, ListKeepsEmptyBareRepeatedPrimitiveRows) { - auto element_reader = std::make_unique( - nested_int64_schema("element", 0, 1, 1, 1), std::make_shared(), - std::vector {0, 1, 1, 0}, std::vector {0, 0, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& array_column = assert_cast(*column); - ASSERT_EQ(array_column.get_offsets().size(), 3); - EXPECT_EQ(array_column.get_offsets()[0], 0); - EXPECT_EQ(array_column.get_offsets()[1], 2); - EXPECT_EQ(array_column.get_offsets()[2], 2); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({2})); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipsAncestorEmptyRowsButKeepsNullElements) { - auto element_reader = - std::make_unique(nested_int64_schema("element", 5, 5, 2, 4), - make_nullable(std::make_shared()), - std::vector {1, 5, 5, 5, 2, 5, 2, 0}, - std::vector {0, 0, 2, 1, 0, 1, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - - const auto inner_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()))); - auto inner_reader = std::make_unique( - nested_list_schema("inner", make_nullable(std::make_shared()), 3, 4, 2, - 2), - inner_type, std::move(element_reader)); - auto outer_type = make_nullable(std::make_shared(inner_type)); - ListColumnReader reader(nested_list_schema("outer", inner_type, 1, 2, 1, 2), outer_type, - std::move(inner_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 5); - EXPECT_EQ(outer_offsets[3], 5); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 5); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_TRUE(inner_nullable.is_null_at(4)); - - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 5); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 3); - EXPECT_EQ(inner_offsets[2], 3); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 4); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({4})); -} - -TEST(ParquetColumnReaderControlTest, MapKeepsEmptyMapRows) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 1); +TEST(ParquetColumnReaderControlTest, SchedulerFlushesReaderProfilesAtBatchBoundary) { + ParquetScanScheduler scheduler; + auto reader = std::make_unique(); + auto* reader_ptr = reader.get(); + scheduler._current_predicate_columns.emplace(0, std::move(reader)); - const auto& nullable_map = assert_cast(*column); - EXPECT_FALSE(nullable_map.is_null_at(0)); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(value_reader_ptr->build_lengths(), std::vector({0})); + scheduler.flush_current_reader_profiles(); + EXPECT_EQ(reader_ptr->profile_flushes(), 1); } -TEST(ParquetColumnReaderControlTest, ListMapSkipsAncestorEmptyRowsBeforeScalarValues) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 4, 4, 2, 4), - make_nullable(std::make_shared()), std::vector {1, 4}, - std::vector {0, 0}); - auto value_reader = make_scripted_scalar_reader(nested_int64_schema("value", 5, 5, 2, 4), - scalar_batch({1, 5}, {0, 0}, {-1, 0}, {100})); - - const auto map_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()), - make_nullable(std::make_shared()))); - auto map_reader = std::make_unique( - nested_map_schema(make_nullable(std::make_shared())), map_type, - std::move(key_reader), std::move(value_reader)); - auto outer_type = make_nullable(std::make_shared(map_type)); - ListColumnReader reader(nested_list_schema("outer", map_type, 1, 2, 1, 2), outer_type, - std::move(map_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 2); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); +TEST(ParquetColumnReaderControlTest, SchedulerOrsPageCrossingOncePerBatch) { + ParquetScanScheduler scheduler; + auto predicate_reader = std::make_unique(); + auto* predicate_ptr = predicate_reader.get(); + predicate_ptr->set_crossed_page(true); + scheduler._current_predicate_columns.emplace(0, std::move(predicate_reader)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 2); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 1); + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + lazy_ptr->set_crossed_page(true); + scheduler._current_non_predicate_columns.emplace(1, std::move(lazy_reader)); - const auto& map_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(map_nullable.size(), 1); - EXPECT_FALSE(map_nullable.is_null_at(0)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 1); - - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 1); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_EQ(value_data.get_element(0), 100); + // Both readers are sampled even after the OR becomes true so their next batch starts cleanly. + EXPECT_TRUE(scheduler.finish_current_reader_batch_profiles()); + EXPECT_EQ(predicate_ptr->page_crossing_checks(), 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } -TEST(ParquetColumnReaderControlTest, MapRejectsNullKeysAndMisalignedScalarValueRepLevels) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}, false, true); - auto value_reader = std::make_unique( - nested_int64_schema("value", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}); - MapColumnReader null_key_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(value_reader)); - auto column = null_key_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = null_key_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null key"), std::string::npos); +TEST(ParquetColumnReaderControlTest, PendingOutputDrainsBeforePageCrossingSample) { + ParquetScanScheduler scheduler; + scheduler._batch_size = 1; + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + scheduler._current_non_predicate_columns.emplace(format::LocalColumnId(0), + std::move(lazy_reader)); + scheduler._pending_predicate_batch_rows = 2; + scheduler._pending_predicate_selection = {0, 1}; - auto aligned_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(aligned_key_reader), - std::move(misaligned_value_reader)); - column = misaligned_reader.type()->create_column(); - status = misaligned_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + Block block; + auto type = std::make_shared(); + block.insert({type->create_column(), type, "mock"}); + size_t rows = 0; -TEST(ParquetColumnReaderControlTest, MapConsumePreservesKeyAndValueCorruptionChecks) { - auto null_key_reader = - make_scripted_scalar_reader(nested_int64_schema("key", 2, 3, 1, 2), - scalar_batch({2}, {0}, {-1}, std::vector {})); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {2}, - std::vector {0}); - MapColumnReader null_key_reader_map(nested_map_schema(), nested_map_schema().type, - std::move(null_key_reader), std::move(value_reader)); - int64_t values_consumed = 0; - auto status = null_key_reader_map.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null"), std::string::npos); + ASSERT_TRUE(scheduler.materialize_pending_predicate_batch(request, &block, &rows).ok()); + EXPECT_EQ(rows, 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 0); + lazy_ptr->set_crossed_page(true); - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(misaligned_value_reader)); - status = misaligned_reader.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapBuildsScalarAndComplexValuePaths) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto scalar_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 1}, {0, 1}, {100, 200})); - MapColumnReader scalar_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(scalar_value_reader)); - auto column = scalar_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = scalar_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_map = assert_cast(*column); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 2); - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 2); - EXPECT_EQ(value_data.get_element(0), 100); - EXPECT_EQ(value_data.get_element(1), 200); - - auto complex_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto complex_value_reader = std::make_unique( - nested_int64_schema("complex_value", 2, 3, 1), - make_nullable(std::make_shared()), std::vector {3, 3}, - std::vector {0, 1}); - auto* complex_value_reader_ptr = complex_value_reader.get(); - MapColumnReader complex_reader(nested_map_schema(), nested_map_schema().type, - std::move(complex_key_reader), std::move(complex_value_reader)); - column = complex_reader.type()->create_column(); - status = complex_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(complex_value_reader_ptr->build_lengths(), std::vector({2})); + block.get_by_position(0).column = type->create_column(); + ASSERT_TRUE(scheduler.materialize_pending_predicate_batch(request, &block, &rows).ok()); + EXPECT_EQ(rows, 1); + EXPECT_TRUE(scheduler._pending_predicate_selection.empty()); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { @@ -1192,65 +331,4 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { EXPECT_FALSE(reader.read(1, column, nullptr).ok()); } -TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordReader) { - ParquetColumnSchema schema = int64_schema("bad_leaf"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.leaf_column_id = 3; - schema.type_descriptor.physical_type = ::parquet::Type::INT64; - schema.type_descriptor.doris_type = schema.type; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { - auto schema = struct_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto empty_projection = format::LocalColumnIndex::partial_local(0); - status = factory.create(schema, &empty_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsListProjectionWithoutElement) { - auto schema = list_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto projection = format::LocalColumnIndex::partial_local(0); - const auto status = factory.create(schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsMapInvalidAndKeyOnlyProjection) { - auto schema = map_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(1)); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto key_only_projection = format::LocalColumnIndex::partial_local(0); - key_only_projection.children.push_back(format::LocalColumnIndex::local(0)); - status = factory.create(schema, &key_only_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no value"), std::string::npos); -} - } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 7402595fc37d06..adb3b4b383c9c8 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -35,10 +36,15 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_decimal.h" +#include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -58,6 +64,7 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -66,6 +73,7 @@ #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -142,6 +150,86 @@ class Int32GreaterThanExpr final : public VExpr { const std::string _expr_name = "Int32GreaterThanExpr"; }; +class Int32DictionaryEqualsExpr final : public VExpr { +public: + Int32DictionaryEqualsExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + const auto expected = Field::create_field(_value); + return std::ranges::any_of(dictionary->values, + [&](const Field& value) { return value == expected; }) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "Int32DictionaryEqualsExpr"; +}; + +class DictionaryAcceptAllExpr final : public VExpr { +public: + explicit DictionaryAcceptAllExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + return ctx.slot(_column_id) == nullptr ? ZoneMapFilterResult::kUnsupported + : ZoneMapFilterResult::kMayMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const std::string _expr_name = "DictionaryAcceptAllExpr"; +}; + class Int32SumGreaterThanExpr final : public VExpr { public: Int32SumGreaterThanExpr(int left_column_id, int right_column_id, int32_t value) @@ -376,6 +464,21 @@ VExprContextSPtr create_int32_greater_than_conjunct(int column_id, int32_t value return ctx; } +VExprContextSPtr create_int32_dictionary_equals_conjunct(int column_id, int32_t value) { + auto ctx = VExprContext::create_shared( + std::make_shared(column_id, value)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_dictionary_accept_all_conjunct(int column_id) { + auto ctx = VExprContext::create_shared(std::make_shared(column_id)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + VExprContextSPtr create_int32_sum_greater_than_conjunct(int left_column_id, int right_column_id, int32_t value) { auto ctx = VExprContext::create_shared( @@ -458,6 +561,39 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_nullable_int32_array( + const std::vector>& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(value.has_value() ? builder.Append(*value).ok() : builder.AppendNull().ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_int64_array(const std::vector& values) { + arrow::Int64Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_float_array(const std::vector& values) { + arrow::FloatBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_double_array(const std::vector& values) { + arrow::DoubleBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -466,6 +602,14 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +std::shared_ptr build_binary_array(const std::vector& values) { + arrow::BinaryBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_timestamp_array(const std::shared_ptr& type, const std::vector& values) { arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); @@ -475,6 +619,27 @@ std::shared_ptr build_timestamp_array(const std::shared_ptr build_decimal_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_fixed_binary_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + const int32_t byte_width = + std::static_pointer_cast(type)->byte_width(); + for (const auto& value : values) { + EXPECT_EQ(value.size(), byte_width); + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_struct_array(const std::vector& ids, const std::vector& names) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), @@ -517,6 +682,41 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_unannotated_binary_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("raw_bytes", arrow::binary(), false)}); + auto table = arrow::Table::Make(schema, {build_binary_array({"否", "是", "测试"})}); + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + builder.build())); +} + +void write_decimal_and_fixed_binary_parquet_file(const std::string& file_path) { + auto decimal_type = arrow::decimal128(38, 6); + auto fixed_type = arrow::fixed_size_binary(4); + auto schema = arrow::schema({arrow::field("decimal_value", decimal_type, false), + arrow::field("fixed_value", fixed_type, false)}); + auto table = arrow::Table::Make( + schema, + {build_decimal_array(decimal_type, {1234567, -1, 0, -987654321, 42}), + build_fixed_binary_array(fixed_type, {"ABCD", std::string("\0x\0y", 4), "wxyz", "1234", + std::string("\xff\x00\x7f\x80", 4)})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + std::shared_ptr build_nullable_int_string_map_array() { auto key_builder = std::make_shared(); auto value_builder = std::make_shared(); @@ -686,6 +886,27 @@ void write_nullable_string_struct_parquet_file(const std::string& file_path) { ROW_COUNT, builder.build())); } +void write_nullable_complex_parquet_file(const std::string& file_path) { + auto map_array = build_nullable_int_string_map_array(); + auto list_array = build_nullable_string_list_array(); + auto struct_array = build_nullable_string_struct_array(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("m", map_array->type(), true), + arrow::field("a", list_array->type(), true), + arrow::field("s", struct_array->type(), true)}), + {map_array, list_array, struct_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_nullable_struct_with_list_parquet_file(const std::string& file_path) { auto scalar_first = build_nullable_struct_with_list_array(false); auto list_first = build_nullable_struct_with_list_array(true); @@ -706,6 +927,34 @@ void write_nullable_struct_with_list_parquet_file(const std::string& file_path) ROW_COUNT, builder.build())); } +void write_nested_complex_under_struct_parquet_file(const std::string& file_path) { + auto nested_struct = build_nullable_string_struct_array(); + auto nested_array = build_nullable_string_list_array(); + auto nested_map = build_nullable_int_string_map_array(); + auto marker = build_int32_array({10, 20, 30, 40, 50}); + auto struct_field = arrow::field("nested_struct", nested_struct->type(), true); + auto array_field = arrow::field("nested_array", nested_array->type(), true); + auto map_field = arrow::field("nested_map", nested_map->type(), true); + auto marker_field = arrow::field("marker", arrow::int32(), false); + auto outer_result = + arrow::StructArray::Make({nested_struct, nested_array, nested_map, marker}, + {struct_field, array_field, map_field, marker_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer = *outer_result; + auto table = arrow::Table::Make(arrow::schema({arrow::field("outer", outer->type(), false)}), + {outer}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_int96_timestamp_parquet_file(const std::string& file_path) { auto field = arrow::field("ts_tz", arrow::timestamp(arrow::TimeUnit::MICRO), true); auto array = @@ -792,14 +1041,21 @@ void write_struct_filter_parquet_file(const std::string& file_path) { builder.build())); } -void write_dictionary_filter_parquet_file(const std::string& file_path) { +void write_dictionary_filter_parquet_file( + const std::string& file_path, + ::parquet::Compression::type compression = ::parquet::Compression::UNCOMPRESSED) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + const std::vector values = + compression == ::parquet::Compression::UNCOMPRESSED + ? std::vector {"aa", "az", "lm", "lz", "za", "zz"} + : std::vector {std::string(4096, 'a'), std::string(4096, 'b'), + std::string(4096, 'c'), std::string(4096, 'd'), + std::string(4096, 'e'), std::string(4096, 'f')}; + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_string_array(values)}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -808,7 +1064,7 @@ void write_dictionary_filter_parquet_file(const std::string& file_path) { ::parquet::WriterProperties::Builder builder; builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.compression(compression); builder.enable_dictionary("value"); builder.disable_dictionary("id"); builder.disable_statistics(); @@ -840,16 +1096,14 @@ void write_single_row_group_dictionary_filter_parquet_file(const std::string& fi builder.build())); } -void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { +void write_fixed_width_dictionary_filter_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), - arrow::field("value", arrow::utf8(), false), - arrow::field("payload", arrow::int32(), false), + arrow::field("value", arrow::int32(), true), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), - build_int32_array({10, 20, 30, 40, 50, 60})}); + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_nullable_int32_array({10, 20, std::nullopt, 20, 40, 20})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -861,46 +1115,62 @@ void write_dictionary_filter_with_trailing_column_parquet_file(const std::string builder.compression(::parquet::Compression::UNCOMPRESSED); builder.disable_dictionary("id"); builder.enable_dictionary("value"); - builder.disable_dictionary("payload"); builder.disable_statistics(); PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, builder.build())); } -void write_nested_dictionary_filter_parquet_file(const std::string& file_path) { - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); +void write_all_fixed_width_dictionary_filter_parquet_file(const std::string& file_path) { + auto fixed_type = arrow::fixed_size_binary(4); + auto timestamp_type = arrow::timestamp(arrow::TimeUnit::MICRO); auto schema = arrow::schema({ - arrow::field("s", struct_type, false), + arrow::field("id", arrow::int32(), false), + arrow::field("int64_value", arrow::int64(), false), + arrow::field("float_value", arrow::float32(), false), + arrow::field("double_value", arrow::float64(), false), + arrow::field("fixed_value", fixed_type, false), + arrow::field("int96_value", timestamp_type, false), }); auto table = arrow::Table::Make( - schema, {build_struct_array({1, 2, 3, 4, 5, 6}, {"aa", "az", "lm", "lz", "za", "zz"})}); + schema, + {build_int32_array({1, 2, 3, 4, 5, 6}), build_int64_array({10, 20, 10, 30, 20, 10}), + build_float_array({1.5F, 2.5F, 1.5F, 3.5F, 2.5F, 1.5F}), + build_double_array({10.25, 20.25, 10.25, 30.25, 20.25, 10.25}), + build_fixed_binary_array(fixed_type, {"AAAA", "BBBB", "AAAA", "CCCC", "BBBB", "AAAA"}), + build_timestamp_array(timestamp_type, {1000, 2000, 1000, 3000, 2000, 1000})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); std::shared_ptr out = *file_result; - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("s.name"); - builder.disable_dictionary("s.identifier.field_id"); - builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, - builder.build())); + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + writer_builder.disable_dictionary("id"); + writer_builder.enable_dictionary("int64_value"); + writer_builder.enable_dictionary("float_value"); + writer_builder.enable_dictionary("double_value"); + writer_builder.enable_dictionary("fixed_value"); + writer_builder.enable_dictionary("int96_value"); + writer_builder.disable_statistics(); + ::parquet::ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_force_write_int96_timestamps(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, + writer_builder.build(), + arrow_builder.build())); } -void write_dictionary_edge_parquet_file(const std::string& file_path) { +void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), + arrow::field("payload", arrow::int32(), false), }); - auto table = arrow::Table::Make( - schema, - {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), - build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); + auto table = + arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), + build_int32_array({10, 20, 30, 40, 50, 60})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -910,52 +1180,23 @@ void write_dictionary_edge_parquet_file(const std::string& file_path) { builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("value"); builder.disable_dictionary("id"); + builder.enable_dictionary("value"); + builder.disable_dictionary("payload"); builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, builder.build())); } -void write_nested_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - std::vector names; - names.reserve(ids.size()); - for (const auto id : ids) { - names.push_back("name-" + std::to_string(id)); - } - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make(schema, {build_struct_array(ids, names)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - -void write_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); +void write_dictionary_edge_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + auto table = arrow::Table::Make( + schema, + {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), + build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -965,12 +1206,11 @@ void write_page_index_filter_parquet_file(const std::string& file_path) { builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); + builder.enable_dictionary("value"); + builder.disable_dictionary("id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); } void write_page_index_filter_pair_parquet_file(const std::string& file_path) { @@ -1136,7 +1376,8 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false) const { + bool is_immutable = false, bool enable_mapping_varbinary = false, + std::string fs_name = {}, int64_t mtime = 0) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1145,9 +1386,11 @@ class NewParquetReaderTest : public testing::Test { file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; file_description->is_immutable = is_immutable; + file_description->fs_name = std::move(fs_name); + file_description->mtime = mtime; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); } std::filesystem::path _test_dir; @@ -1172,6 +1415,39 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, RawByteArrayMappingFollowsV2ScanOption) { + write_unannotated_binary_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto string_reader = create_reader(); + const auto string_init_status = string_reader->init(&state); + ASSERT_TRUE(string_init_status.ok()) << string_init_status; + std::vector string_schema; + ASSERT_TRUE(string_reader->get_schema(&string_schema).ok()); + ASSERT_EQ(string_schema.size(), 1); + EXPECT_EQ(remove_nullable(string_schema[0].type)->get_primitive_type(), TYPE_STRING); + + RuntimeProfile binary_profile("raw_binary_mapping_profile"); + auto binary_reader = + create_reader(0, -1, &binary_profile, false, nullptr, std::nullopt, false, true); + const auto binary_init_status = binary_reader->init(&state); + ASSERT_TRUE(binary_init_status.ok()) << binary_init_status; + std::vector binary_schema; + ASSERT_TRUE(binary_reader->get_schema(&binary_schema).ok()); + ASSERT_EQ(binary_schema.size(), 1); + // The explicit VARBINARY mapping must remain available for scans whose table contract asks for it. + EXPECT_EQ(remove_nullable(binary_schema[0].type)->get_primitive_type(), TYPE_VARBINARY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_string_in_conjunct(0, {"是"})); + ASSERT_TRUE(binary_reader->open(request).ok()); + // A table-side STRING comparison may be rewritten through this VARBINARY file slot. Native + // dictionary pruning must leave the row group available for the mapping expression. + EXPECT_EQ(binary_profile.get_counter("RowGroupsFilteredByDictionary")->value(), 0); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. @@ -1203,6 +1479,10 @@ TEST_F(NewParquetReaderTest, CountComplexColumnUsesShapeOnlyPath) { EXPECT_EQ(result.count, 4); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); + ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); + ASSERT_NE(profile.get_counter("ParsePageHeaderNum"), nullptr); + EXPECT_GT(profile.get_counter("ParsePageHeaderNum")->value(), 0); } TEST_F(NewParquetReaderTest, CountArrayColumnUsesLevelsOnlyPath) { @@ -1271,6 +1551,238 @@ TEST_F(NewParquetReaderTest, CountStructWithRepeatedChildUsesTopLevelRowBoundari } } +TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchChanges) { + write_nullable_complex_parquet_file(_file_path); + RuntimeProfile profile("native_complex_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), + field_projection(2)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + int batch = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + if (++batch == 1) { + // Adaptive sizing changes only the logical row cap. The persistent native readers and + // their level/string scratch must continue from the same page cursors. + reader->set_batch_size(3); + } + } + + const auto& nullable_map = assert_cast(*output[0]); + ASSERT_EQ(nullable_map.size(), ROW_COUNT); + EXPECT_FALSE(nullable_map.is_null_at(0)); + EXPECT_TRUE(nullable_map.is_null_at(1)); + EXPECT_FALSE(nullable_map.is_null_at(2)); + const auto& map = assert_cast(nullable_map.get_nested_column()); + EXPECT_EQ(map.get_offsets(), ColumnArray::Offsets64({1, 1, 1, 2, 3})); + const auto& map_keys = assert_cast(map.get_keys()); + const auto& key_values = assert_cast(map_keys.get_nested_column()); + ASSERT_EQ(key_values.size(), 3); + EXPECT_EQ(key_values.get_element(0), 10); + EXPECT_EQ(key_values.get_element(2), 30); + const auto& map_values = assert_cast(map.get_values()); + const auto& value_strings = assert_cast(map_values.get_nested_column()); + EXPECT_EQ(value_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(value_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(map_values.is_null_at(2)); + + const auto& nullable_array = assert_cast(*output[1]); + ASSERT_EQ(nullable_array.size(), ROW_COUNT); + EXPECT_TRUE(nullable_array.is_null_at(1)); + const auto& array = assert_cast(nullable_array.get_nested_column()); + EXPECT_EQ(array.get_offsets(), ColumnArray::Offsets64({2, 2, 2, 3, 4})); + const auto& array_values = assert_cast(array.get_data()); + const auto& array_strings = assert_cast(array_values.get_nested_column()); + EXPECT_EQ(array_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(array_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(array_values.is_null_at(2)); + EXPECT_EQ(array_strings.get_data_at(3).size, 4096); + + const auto& nullable_struct = assert_cast(*output[2]); + ASSERT_EQ(nullable_struct.size(), ROW_COUNT); + EXPECT_TRUE(nullable_struct.is_null_at(1)); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& payload = assert_cast(struct_column.get_column(0)); + const auto& payload_strings = assert_cast(payload.get_nested_column()); + EXPECT_EQ(payload_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(payload_strings.get_data_at(2).size, 4096); + EXPECT_TRUE(payload.is_null_at(3)); + EXPECT_EQ(payload_strings.get_data_at(4).size, 4096); + const auto& ids = assert_cast(struct_column.get_column(1)); + const auto& id_values = assert_cast(ids.get_nested_column()); + EXPECT_EQ(id_values.get_element(0), 1); + EXPECT_EQ(id_values.get_element(4), 4); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + ASSERT_NE(profile.get_counter("NestedBatches"), nullptr); + EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); +} + +TEST_F(NewParquetReaderTest, FullComplexChildUnderPartialParentReadsItsWholeSubtree) { + write_nested_complex_under_struct_parquet_file(_file_path); + for (size_t child_index = 0; child_index < 3; ++child_index) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + ASSERT_EQ(schema[0].children.size(), 4); + auto projection = format::LocalColumnIndex::partial_local(0); + projection.children.push_back( + format::LocalColumnIndex::local(schema[0].children[child_index].file_local_id())); + auto request = std::make_shared(); + request->non_predicate_columns = {projection}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + format::ColumnDefinition projected; + ASSERT_TRUE(format::project_column_definition(schema[0], projection, &projected).ok()); + Block block; + block.insert(ColumnWithTypeAndName(projected.type->create_column(), projected.type, + projected.name)); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_GT(rows, 0); + const auto& outer = nullable_nested_column(block, 0); + ASSERT_EQ(outer.tuple_size(), 1); + const auto& nullable_nested = assert_cast(outer.get_column(0)); + if (child_index == 0) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + ASSERT_EQ(nested.tuple_size(), 2); + const auto& payload = assert_cast(nested.get_column(0)); + EXPECT_EQ(assert_cast(payload.get_nested_column()) + .get_data_at(0) + .to_string(), + "small"); + const auto& ids = assert_cast(nested.get_column(1)); + EXPECT_EQ(ids.get_nested_column().get_int(0), 1); + } else if (child_index == 1) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 2); + EXPECT_EQ(nested.get_data().size(), 4); + } else { + const auto& nested = assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 1); + EXPECT_EQ(nested.get_keys().size(), 3); + EXPECT_EQ(nested.get_values().size(), 3); + } + } +} + +TEST_F(NewParquetReaderTest, NativeNestedMapUsesOuterKeyRepetitionShape) { + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + _file_path = + std::string(source_root) + "/regression-test/data/external_table_p0/tvf/comp.parquet"; + ASSERT_TRUE(std::filesystem::exists(_file_path)); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + for (size_t position = 0; position < schema.size(); ++position) { + request->non_predicate_columns.push_back(field_projection(cast_set(position))); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const Status status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + EXPECT_GT(total_rows, 0); +} + +TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { + write_decimal_and_fixed_binary_parquet_file(_file_path); + RuntimeProfile profile("native_decimal_fixed_binary_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + reader->set_batch_size(3); + } + + const auto& decimals = assert_cast( + assert_cast(*output[0]).get_nested_column()); + ASSERT_EQ(decimals.size(), ROW_COUNT); + EXPECT_EQ(decimals.get_element(0), Decimal128V3(1234567)); + EXPECT_EQ(decimals.get_element(1), Decimal128V3(-1)); + EXPECT_EQ(decimals.get_element(3), Decimal128V3(-987654321)); + + const auto& fixed_values = assert_cast( + assert_cast(*output[1]).get_nested_column()); + ASSERT_EQ(fixed_values.size(), ROW_COUNT); + EXPECT_EQ(fixed_values.get_data_at(0).to_string(), "ABCD"); + EXPECT_EQ(fixed_values.get_data_at(1).to_string(), std::string("\0x\0y", 4)); + EXPECT_EQ(fixed_values.get_data_at(4).to_string(), std::string("\xff\x00\x7f\x80", 4)); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + EXPECT_EQ(profile.get_counter("ConvertTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { write_struct_filter_parquet_file(_file_path); auto reader = create_reader(); @@ -1297,6 +1809,35 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { EXPECT_TRUE(struct_type->get_element(1)->is_nullable()); } +TEST_F(NewParquetReaderTest, ComplexColumnDoesNotMisreportSiblingPagesAsCrossing) { + write_struct_filter_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_complex_page_fragments"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 4); + // Each STRUCT child reads one page per Row Group, but no individual leaf crosses a boundary in + // either batch. The aggregate fragment count remains useful without inflating crossing batches. + EXPECT_EQ(profile.get_counter("NativePageFragments")->value(), 8); + EXPECT_EQ(profile.get_counter("PageCrossingBatches")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaMapsInt96ToTimestampTzWhenTimestampTzMappingEnabled) { write_int96_timestamp_parquet_file(_file_path); auto reader = create_reader(0, -1, nullptr, true); @@ -1532,10 +2073,78 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); ASSERT_NE(profile.get_counter("PageCacheWriteCount"), nullptr); - EXPECT_EQ(profile.get_counter("PageReadCount")->value(), 0); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0); } +TEST_F(NewParquetReaderTest, NativeFooterCacheIdentityIncludesFilesystemAndVersion) { + const auto first = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto other_fs = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-b", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto replaced = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 11, 0, 1024, 1024, false); + EXPECT_FALSE(first.empty()); + EXPECT_NE(first, other_fs); + EXPECT_NE(first, replaced); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotCrossFilesystems) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile first_profile("native_footer_cache_nameservice_a"); + auto first = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-a", 1234567); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + RuntimeProfile second_profile("native_footer_cache_nameservice_b"); + auto second = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-b", 1234567); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheSkipsMutableUnknownVersion) { + EXPECT_TRUE(format::parquet::detail::build_native_file_cache_key("hdfs://nameservice-a", + "/warehouse/mutable.parquet", + 0, 0, 1024, 1024, false) + .empty()); + EXPECT_FALSE( + format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/immutable.parquet", 0, 0, 1024, 1024, true) + .empty()); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotReuseMutableUnknownVersion) { + _file_path = (_test_dir / "mutable_footer_cache.parquet").string(); + write_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("native_footer_cache_mutable_first"); + auto first = create_reader(0, -1, &first_profile); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + write_parquet_file(_file_path); + RuntimeProfile second_profile("native_footer_cache_mutable_second"); + auto second = create_reader(0, -1, &second_profile); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterSizeIsBoundedBeforeMetadataAllocation) { + constexpr size_t file_size = 256UL << 20; + constexpr size_t metadata_limit = 100UL << 20; + const auto status = format::parquet::detail::validate_native_footer_size( + static_cast(metadata_limit + 1), file_size, metadata_limit); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("metadata limit"), std::string::npos); +} + TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { _file_path = (_test_dir / "unknown_mtime_page_cache.parquet").string(); write_parquet_file(_file_path); @@ -1645,24 +2254,42 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterHitCache"), nullptr); ASSERT_GT(profile.get_counter("FileReaderCreateTime")->value(), 0); EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); + TRuntimeProfileTree profile_tree; + profile.to_thrift(&profile_tree); + ASSERT_FALSE(profile_tree.nodes.empty()); + const auto parquet_children = + profile_tree.nodes.front().child_counters_map.find("ParquetReader"); + ASSERT_NE(parquet_children, profile_tree.nodes.front().child_counters_map.end()); + EXPECT_TRUE(parquet_children->second.contains("RowsFilteredByConjunct")); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 1); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); - EXPECT_GT(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value() + + profile.get_counter("FileFooterHitCache")->value(), + 1); rows = 0; eof = false; @@ -1777,14 +2404,61 @@ TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 0); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); } +TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTotalTime) { + RuntimeProfile profile("new_parquet_reader_hierarchy_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + + auto* parquet_total = profile.get_counter("ParquetReader"); + ASSERT_NE(parquet_total, nullptr); + EXPECT_GT(parquet_total->value(), 0); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains(RuntimeProfile::ROOT_COUNTER)); + EXPECT_TRUE(children.at(RuntimeProfile::ROOT_COUNTER).contains("FileScannerV2")); + ASSERT_TRUE(children.contains("FileScannerV2")); + EXPECT_TRUE(children.at("FileScannerV2").contains("TableReader")); + ASSERT_TRUE(children.contains("TableReader")); + EXPECT_TRUE(children.at("TableReader").contains("FileReader")); + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + EXPECT_TRUE(children.at("FileReader").contains("ParquetReader")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowGroupsReadNum")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredRowsByGroup")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredBytes")); + EXPECT_TRUE(children.at("ParquetReader").contains("FileNum")); +} + TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -2010,41 +2684,6 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByStatistics) { TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto value_chunk = row_group->ColumnChunk(1); - ASSERT_NE(value_chunk, nullptr); - ASSERT_TRUE(value_chunk->has_dictionary_page()); - ASSERT_TRUE(value_chunk->statistics() == nullptr || - !value_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 2); - - format::FileScanRequest plan_request; - plan_request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - plan_request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - plan_request, scan_range, false, &plan) - .ok()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, 1); - auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -2080,6 +2719,55 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { EXPECT_EQ(values, std::vector({"lm"})); } +TEST_F(NewParquetReaderTest, DictionaryPruningPublishesColdAndWarmNativePageProfile) { + const double old_cache_threshold = config::parquet_page_cache_decompress_threshold; + config::parquet_page_cache_decompress_threshold = 100.0; + Defer restore_cache_threshold { + [&] { config::parquet_page_cache_decompress_threshold = old_cache_threshold; }}; + write_dictionary_filter_parquet_file(_file_path, ::parquet::Compression::SNAPPY); + + auto open_pruned_reader = [&](RuntimeProfile* profile) { + auto reader = create_reader(0, -1, profile, false, nullptr, std::nullopt, true); + TQueryOptions query_options; + query_options.__set_enable_parquet_file_page_cache(true); + RuntimeState state {query_options, TQueryGlobals()}; + RETURN_IF_ERROR(reader->init(&state)); + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"not-present"})); + use_schema_order_positions(request.get(), schema); + RETURN_IF_ERROR(reader->open(request)); + EXPECT_EQ(profile->get_counter("RowGroupsFilteredByDictionary")->value(), 0); + EXPECT_EQ(profile->get_counter("PageReadCount")->value(), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + // Expensive dictionary probes are current-row-group work now, so advance the scheduler + // once before inspecting page-cache and pruning counters. + return reader->get_block(&block, &rows, &eof); + }; + + RuntimeProfile cold_profile("dictionary_pruning_cold_profile"); + ASSERT_TRUE(open_pruned_reader(&cold_profile).ok()); + for (const auto* counter_name : + {"RowGroupsFilteredByDictionary", "PageReadCount", "PageCacheWriteCount", + "ParsePageHeaderNum", "DecompressCount", "DecodeDictTime"}) { + ASSERT_NE(cold_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(cold_profile.get_counter(counter_name)->value(), 0) << counter_name; + } + + RuntimeProfile warm_profile("dictionary_pruning_warm_profile"); + ASSERT_TRUE(open_pruned_reader(&warm_profile).ok()); + for (const auto* counter_name : {"RowGroupsFilteredByDictionary", "PageReadCount", + "PageCacheHitCount", "ParsePageHeaderNum", "DecodeDictTime"}) { + ASSERT_NE(warm_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(warm_profile.get_counter(counter_name)->value(), 0) << counter_name; + } +} + TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { write_single_row_group_dictionary_filter_parquet_file(_file_path); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); @@ -2135,7 +2823,102 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { EXPECT_GE(profile.get_counter("ReaderSelectRows")->value(), 8); } -TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) { +TEST_F(NewParquetReaderTest, FixedWidthDictionaryPredicateFiltersRowsByDictionaryId) { + write_fixed_width_dictionary_filter_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_fixed_width_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_int32_dictionary_equals_conjunct(1, 20)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({2, 4, 6})); + EXPECT_EQ(values, std::vector({20, 20, 20})); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 3); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); +} + +TEST_F(NewParquetReaderTest, AllFixedWidthDictionaryTypesDecodeThroughDictionaryIds) { + write_all_fixed_width_dictionary_filter_parquet_file(_file_path); + + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + auto row_group = parquet_file_reader->metadata()->RowGroup(0); + ASSERT_NE(row_group, nullptr); + ASSERT_EQ(row_group->num_columns(), 6); + const std::array<::parquet::Type::type, 5> expected_types { + ::parquet::Type::INT64, ::parquet::Type::FLOAT, ::parquet::Type::DOUBLE, + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, ::parquet::Type::INT96}; + for (int column = 1; column < row_group->num_columns(); ++column) { + ASSERT_TRUE(row_group->ColumnChunk(column)->has_dictionary_page()) << column; + EXPECT_EQ(row_group->ColumnChunk(column)->type(), expected_types[column - 1]) << column; + } + + RuntimeProfile profile("new_parquet_reader_all_fixed_width_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 6); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + for (int column = 1; column < 6; ++column) { + request->predicate_columns.push_back(field_projection(column)); + request->conjuncts.push_back(create_dictionary_accept_all_conjunct(column)); + } + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + + EXPECT_EQ(total_rows, 6); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateReaderIsSharedOutsideMergeRangeReader) { write_dictionary_filter_with_trailing_column_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_dictionary_filter_merge_profile"); @@ -2177,8 +2960,14 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) EXPECT_EQ(values, std::vector({"az", "za"})); EXPECT_EQ(payloads, std::vector({20, 50})); EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + // The native dictionary probe keeps its reader and cursor for predicate data pages. Other + // projected chunks still use the row-group merge reader, so probing never duplicates the + // dictionary read or perturbs the merge range's sequential access order. ASSERT_NE(profile.get_counter("MergedIO"), nullptr); ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); + EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); } TEST_F(NewParquetReaderTest, DictionaryPredicateWorksWithoutRuntimeProfile) { @@ -2258,7 +3047,9 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWh // second predicate column is skipped after the selection becomes empty, which verifies the // StarRocks-style round-by-round policy: only rows surviving previous predicates are read. EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 6); - EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6); + // Five dictionary ids are rejected before the residual predicate; the remaining predicate + // reader then skips all six logical rows once the selection is empty. + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 11); } TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) { @@ -2346,208 +3137,10 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateKeepsNestedOrResidualConjunct) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); } -TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) { - write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 2); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByStatistics) { - write_struct_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByDictionary) { - write_nested_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto name_chunk = row_group->ColumnChunk(1); - ASSERT_NE(name_chunk, nullptr); - ASSERT_TRUE(name_chunk->has_dictionary_page()); - ASSERT_TRUE(name_chunk->statistics() == nullptr || !name_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[1]->name, "name"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 6); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) { - write_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - - format::FileScanRequest request; - request.predicate_columns = {field_projection(0)}; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128); - auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0); - EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0); - EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1); - ASSERT_EQ(skip_plan_it->second.skipped_pages.size(), - skip_plan_it->second.skipped_page_compressed_sizes.size()); - int64_t skipped_compressed_bytes = 0; - for (size_t page_idx = 0; page_idx < skip_plan_it->second.skipped_pages.size(); ++page_idx) { - if (skip_plan_it->second.should_skip_page(page_idx)) { - skipped_compressed_bytes += skip_plan_it->second.skipped_page_compressed_size(page_idx); - } - } - EXPECT_GT(skipped_compressed_bytes, 0); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) { - write_nested_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length, - parquet_file_reader->metadata()->RowGroup(0)->num_rows()); - EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - // Scenario: the selected range starts after page-index-pruned rows. The scheduler defers that range // gap for the non-predicate payload reader, then flushes it exactly once before materialization. The -// page skip plan advances the reader without calling Arrow SkipRecords or double-skipping row 64. +// native RowRanges/OffsetIndex plan advances both readers without decoding the rejected pages or +// double-skipping row 64. TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { write_page_index_filter_pair_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_page_skip"); @@ -2571,7 +3164,8 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { bool eof = false; while (!eof) { size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -2589,7 +3183,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); ASSERT_NE(profile.get_counter("RowGroupFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexReadTime"), nullptr); @@ -2599,7 +3193,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 64); EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 0); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); EXPECT_GT(profile.get_counter("RowGroupFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexReadTime")->value(), 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 55387d2c1ed380..9bdb0c7562f0b6 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -21,10 +21,13 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include @@ -43,11 +46,16 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -55,6 +63,8 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/utils.h" +#include "util/coding.h" +#include "util/thrift_util.h" namespace doris { namespace { @@ -70,6 +80,13 @@ const ColumnInt32& int32_data_column(const IColumn& column) { return assert_cast(column); } +const ColumnInt64& int64_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + const ColumnString& string_data_column(const IColumn& column) { if (const auto* nullable_column = check_and_get_column(&column)) { return assert_cast(nullable_column->get_nested_column()); @@ -77,6 +94,64 @@ const ColumnString& string_data_column(const IColumn& column) { return assert_cast(column); } +TEST(ParquetScanMetadataSafetyTest, CheckedChunkRangesDrivePrefetchAndSplitAssignment) { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + metadata.__set_schema({root, leaf}); + + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + column.__set_data_page_offset(100); + column.__set_dictionary_page_offset(-1); + column.__set_total_compressed_size(20); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + metadata.__set_row_groups({row_group}); + + auto schema = std::make_unique(); + schema->local_id = 0; + schema->leaf_column_id = 0; + schema->type = std::make_shared(); + std::vector> file_schema; + file_schema.push_back(std::move(schema)); + std::vector ranges; + ASSERT_TRUE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].offset, 100); + EXPECT_EQ(ranges[0].size, 20); + + std::vector first_rows; + std::vector selected; + format::parquet::ParquetScanRange first_split { + .start_offset = 0, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, first_split, &first_rows, &selected) + .ok()); + EXPECT_TRUE(selected.empty()); + format::parquet::ParquetScanRange second_split { + .start_offset = 100, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, second_split, &first_rows, &selected) + .ok()); + EXPECT_EQ(selected, std::vector({0})); + + metadata.row_groups[0].columns[0].meta_data.__set_data_page_offset(190); + EXPECT_FALSE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); +} + class Int32ZoneMapExpr final : public VExpr { public: enum class Op { GE, GT, LT }; @@ -192,17 +267,223 @@ class Int32PairSumExpr final : public VExpr { const std::string _expr_name = "Int32PairSumExpr"; }; +class Int32DirectGreaterExpr final : public VExpr { +public: + Int32DirectGreaterExpr(int column_id, int32_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int32_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int32_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int32_t _lower_bound; + const std::string _expr_name = "Int32DirectGreaterExpr"; +}; + +class Int64DirectGreaterExpr final : public VExpr { +public: + Int64DirectGreaterExpr(int column_id, int64_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int64_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_BIGINT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + if (value_width != sizeof(int64_t)) { + return Status::Corruption("BIGINT raw predicate received {}-byte values", value_width); + } + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int64_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int64_t _lower_bound; + const std::string _expr_name = "Int64DirectGreaterExpr"; +}; + +class AlwaysTrueSingleColumnExpr final : public VExpr { +public: + explicit AlwaysTrueSingleColumnExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + const std::string _expr_name = "AlwaysTrueSingleColumnExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); } +VExprContextSPtr create_int32_function_conjunct(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto result_type = make_nullable(std::make_shared()); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({nullable_int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(true); + auto root = VectorizedFnCall::create_shared(node); + root->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + root->add_child(VLiteral::create_shared(int_type, Field::create_field(value))); + auto context = VExprContext::create_shared(std::move(root)); + // Direct evaluation does not execute the expression, but a fallback must still fail this test + // instead of silently using an unprepared test-only context. + context->_prepared = true; + context->_opened = true; + return context; +} + +VExprContextSPtr create_int32_mod_greater_than_conjunct(int column_id) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto nullable_bool_type = make_nullable(std::make_shared()); + auto function_node = [](const std::string& name, TExprNodeType::type node_type, + TExprOpcode::type opcode, const DataTypePtr& return_type, + const std::vector& argument_types) { + TFunctionName function_name; + function_name.__set_function_name(name); + TFunction function; + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + std::vector thrift_argument_types; + for (const auto& argument_type : argument_types) { + thrift_argument_types.push_back(argument_type->to_thrift()); + } + function.__set_arg_types(thrift_argument_types); + function.__set_ret_type(return_type->to_thrift()); + function.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(node_type); + node.__set_opcode(opcode); + node.__set_type(return_type->to_thrift()); + node.__set_fn(function); + node.__set_num_children(static_cast(argument_types.size())); + node.__set_is_nullable(return_type->is_nullable()); + return node; + }; + + auto modulo = VectorizedFnCall::create_shared( + function_node("mod", TExprNodeType::ARITHMETIC_EXPR, TExprOpcode::MOD, + nullable_int_type, {nullable_int_type, int_type})); + modulo->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + modulo->add_child(VLiteral::create_shared(int_type, Field::create_field(-1))); + + auto greater_than = VectorizedFnCall::create_shared( + function_node("gt", TExprNodeType::BINARY_PRED, TExprOpcode::GT, nullable_bool_type, + {nullable_int_type, int_type})); + greater_than->add_child(std::move(modulo)); + greater_than->add_child(VLiteral::create_shared(int_type, Field::create_field(0))); + return VExprContext::create_shared(std::move(greater_than)); +} + VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_column_id, int32_t upper_bound) { return VExprContext::create_shared( std::make_shared(left_column_id, right_column_id, upper_bound)); } +VExprContextSPtr create_int32_direct_greater_conjunct(int column_id, int32_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_int64_direct_greater_conjunct(int column_id, int64_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_always_true_single_column_conjunct(int column_id) { + return VExprContext::create_shared(std::make_shared(column_id)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -223,6 +504,14 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_uint32_array(const std::vector& values) { + arrow::UInt32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -278,7 +567,8 @@ std::shared_ptr build_list_array() { void write_table(const std::string& file_path, const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary = false, - bool enable_page_index = false, bool enable_statistics = true) { + bool enable_page_index = false, bool enable_statistics = true, + std::optional<::parquet::Encoding::type> encoding = std::nullopt) { auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); std::shared_ptr out = *file_result; @@ -292,6 +582,9 @@ void write_table(const std::string& file_path, const std::shared_ptr out = *file_result; + + // Arrow intentionally writes time32 as local time (isAdjustedToUTC=false), so use Parquet's + // low-level writer to produce the unsupported required logical type needed by this aggregate + // regression. Required is important: Nereids may push COUNT(col) because NULL filtering cannot + // change its value, which is precisely the path where an empty aggregate request lost `col`. + const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( + "unsupported_time", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto schema_node = ::parquet::schema::GroupNode::Make( + "schema", ::parquet::Repetition::REQUIRED, {adjusted_time}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + auto writer = ::parquet::ParquetFileWriter::Open(out, schema); + auto* row_group = writer->AppendRowGroup(); + auto* time_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + const int32_t values[] = {1000, 2000}; + EXPECT_EQ(time_writer->WriteBatch(2, nullptr, nullptr, values), 2); + time_writer->Close(); + row_group->Close(); + writer->Close(); +} + void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group_size = 2, - bool enable_statistics = true) { + bool enable_statistics = true, + std::optional<::parquet::Encoding::type> encoding = std::nullopt) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("score", arrow::int32(), false), }); auto table = arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_int32_array({10, 20, 30, 40, 50, 60})}); - write_table(file_path, table, row_group_size, false, false, enable_statistics); + write_table(file_path, table, row_group_size, false, false, enable_statistics, encoding); +} + +void write_uint32_pair_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("id", arrow::uint32(), false), + arrow::field("score", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_uint32_array({1, 2147483648U, 4294957294U}), + build_int32_array({10, 20, 30})}); + write_table(file_path, table, 3, false, false, false); +} + +std::vector serialize_test_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +void write_misdeclared_two_page_parquet_file(const std::string& file_path) { + const std::array first_values {1, 2}; + std::vector first_payload(sizeof(first_values)); + memcpy(first_payload.data(), first_values.data(), first_payload.size()); + tparquet::PageHeader first_header; + first_header.type = tparquet::PageType::DATA_PAGE_V2; + first_header.__set_compressed_page_size(first_payload.size()); + first_header.__set_uncompressed_page_size(first_payload.size()); + first_header.__isset.data_page_header_v2 = true; + first_header.data_page_header_v2.__set_num_values(first_values.size()); + first_header.data_page_header_v2.__set_num_nulls(0); + first_header.data_page_header_v2.__set_num_rows(first_values.size()); + first_header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + first_header.data_page_header_v2.__set_definition_levels_byte_length(0); + first_header.data_page_header_v2.__set_repetition_levels_byte_length(0); + first_header.data_page_header_v2.__set_is_compressed(false); + auto column_bytes = serialize_test_page(first_header, first_payload); + + // Bit width 1 followed by an RLE run of two dictionary ids. The payload is structurally valid + // so the raw path reaches its late-encoding guard instead of failing while loading the page. + const std::vector second_payload {1, 4, 0}; + tparquet::PageHeader second_header = first_header; + second_header.__set_compressed_page_size(second_payload.size()); + second_header.__set_uncompressed_page_size(second_payload.size()); + second_header.data_page_header_v2.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + auto second_page = serialize_test_page(second_header, second_payload); + column_bytes.insert(column_bytes.end(), second_page.begin(), second_page.end()); + + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("id"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + // Deliberately omit the second page's unsupported encoding to emulate untrusted footer + // metadata. The reader must reject it before attempting a typed fallback after partial cursor + // progress. + column.__set_encodings({tparquet::Encoding::PLAIN, tparquet::Encoding::RLE}); + column.__set_path_in_schema({"id"}); + column.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column.__set_num_values(4); + column.__set_total_uncompressed_size(column_bytes.size()); + column.__set_total_compressed_size(column_bytes.size()); + column.__set_data_page_offset(4); + tparquet::ColumnChunk chunk; + chunk.__set_file_offset(4); + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(column_bytes.size()); + row_group.__set_num_rows(4); + tparquet::FileMetaData metadata; + metadata.__set_version(2); + metadata.__set_schema({root, leaf}); + metadata.__set_num_rows(4); + metadata.__set_row_groups({row_group}); + + std::vector footer; + ThriftSerializer serializer(/*compact=*/true, 1024); + DORIS_CHECK(serializer.serialize(&metadata, &footer).ok()); + std::ofstream output(file_path, std::ios::binary | std::ios::trunc); + output.write("PAR1", 4); + output.write(reinterpret_cast(column_bytes.data()), column_bytes.size()); + output.write(reinterpret_cast(footer.data()), footer.size()); + std::array footer_size {}; + encode_fixed32_le(footer_size.data(), cast_set(footer.size())); + output.write(reinterpret_cast(footer_size.data()), footer_size.size()); + output.write("PAR1", 4); + output.close(); + DORIS_CHECK(output.good()); +} + +void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) { + std::vector ids(rows); + std::vector first(rows); + std::vector second(rows); + std::iota(ids.begin(), ids.end(), 0); + for (size_t row = 0; row < rows; ++row) { + first[row] = static_cast(row + 10); + second[row] = static_cast(row + 20); + } + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("first", arrow::int32(), false), + arrow::field("second", arrow::int32(), false)}); + auto table = arrow::Table::Make( + schema, {build_int32_array(ids), build_int32_array(first), build_int32_array(second)}); + write_table(file_path, table, rows, false, false, false); } void write_binary_minmax_parquet_file(const std::string& file_path) { @@ -363,15 +792,12 @@ void write_page_index_parquet_file(const std::string& file_path) { write_table(file_path, table, ids.size(), false, true); } -void write_page_index_pair_parquet_file(const std::string& file_path) { - std::vector ids(128); +void write_multi_row_group_page_index_parquet_file(const std::string& file_path) { + std::vector ids(384); std::iota(ids.begin(), ids.end(), 0); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false), - arrow::field("score", arrow::int32(), false), - }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(ids)}); - write_table(file_path, table, ids.size(), false, true); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + write_table(file_path, table, 128, false, true); } int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { @@ -419,24 +845,6 @@ void use_schema_order_positions(format::FileScanRequest* request, } } -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - auto schema_descriptor = reader.metadata()->schema(); - EXPECT_NE(schema_descriptor, nullptr); - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - return file_schema; -} - -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - class ParquetScanTest : public testing::Test { protected: void SetUp() override { @@ -493,87 +901,76 @@ TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) { EXPECT_TRUE(selection.verify(selected_rows, 6).ok()); } -TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2); +TEST(ParquetScanAdaptivePredicateTest, OrdersByObservedCostPerRejectedRow) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map stats; + stats.emplace(0, AdaptivePredicateStats { + .cost_per_input_row_ns = 10, .survival_ratio = 0.8, .samples = 3}); + stats.emplace(1, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.1, .samples = 3}); + stats.emplace(2, AdaptivePredicateStats { + .cost_per_input_row_ns = 50, .survival_ratio = 0.5, .samples = 3}); + + const auto order = format::parquet::detail::order_adaptive_predicates({0, 1, 2}, stats); + EXPECT_EQ(order, std::vector({1, 0, 2})); + const auto prefetched = format::parquet::detail::adaptive_prefetch_prefix(order, stats, 0.25); + EXPECT_EQ(prefetched, std::vector({1})); } -TEST_F(ParquetScanTest, PlanRowGroupsPreservesFirstFileRowAcrossPrunedRowGroups) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); +TEST(ParquetScanAdaptivePredicateTest, SamplesWarmupThenAtLowFrequency) { + using format::parquet::detail::should_sample_adaptive_predicate; + for (size_t samples = 0; samples < 8; ++samples) { + EXPECT_TRUE(should_sample_adaptive_predicate(samples, samples)); + } + EXPECT_FALSE(should_sample_adaptive_predicate(8, 9)); + EXPECT_TRUE(should_sample_adaptive_predicate(8, 16)); + EXPECT_FALSE(should_sample_adaptive_predicate(9, 17)); + EXPECT_TRUE(should_sample_adaptive_predicate(9, 32)); +} - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); +TEST(ParquetScanDeleteConjunctTest, RejectsInputColumnAsEphemeralResult) { + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 0, 2) + .is()); + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 2, 3).ok()); + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 3, 3) + .is()); +} - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.row_groups[0].first_file_row, 4); - EXPECT_EQ(plan.row_groups[0].row_group_rows, 2); - ASSERT_EQ(plan.row_groups[0].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].length, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 2); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 4); -} - -TEST_F(ParquetScanTest, PlanRowGroupsSelectsAllRowGroupsWithoutFilters) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); +TEST(ParquetScanAdaptivePredicateTest, ThrowingNestedFunctionDisablesSelectedRowReordering) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map first_batch_stats; + first_batch_stats.emplace( + 0, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.99, .samples = 1}); + first_batch_stats.emplace( + 1, AdaptivePredicateStats { + .cost_per_input_row_ns = 1, .survival_ratio = 0.1, .samples = 1}); + EXPECT_EQ(format::parquet::detail::order_adaptive_predicates({0, 1}, first_batch_stats), + (std::vector {1, 0})); + + const auto total_comparison = create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0); + const auto throwing_comparison = create_int32_mod_greater_than_conjunct(0); + EXPECT_TRUE(total_comparison->root()->is_safe_to_execute_on_selected_rows()); + // The second-batch order above can reject MIN_INT before mod(MIN_INT, -1) runs. The nested + // partial function must therefore keep the request on full-batch, error-preserving execution. + EXPECT_FALSE(throwing_comparison->root()->is_safe_to_execute_on_selected_rows()); +} - format::FileScanRequest request; - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); +TEST(ParquetScanSmallFileTest, StagesOnlyBoundedHttpObjects) { + using format::parquet::detail::should_stage_small_http_file; + EXPECT_TRUE(should_stage_small_http_file("http://host/tiny.parquet", 512, 1024)); + EXPECT_TRUE(should_stage_small_http_file("https://host/tiny.parquet", 1024, 1024)); + EXPECT_FALSE(should_stage_small_http_file("https://host/large.parquet", 1025, 1024)); + EXPECT_FALSE(should_stage_small_http_file("/tmp/tiny.parquet", 512, 1024)); + EXPECT_FALSE(should_stage_small_http_file("s3://bucket/tiny.parquet", 512, 1024)); +} - ASSERT_EQ(plan.row_groups.size(), 3); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 3); - for (size_t row_group_idx = 0; row_group_idx < plan.row_groups.size(); ++row_group_idx) { - EXPECT_EQ(plan.row_groups[row_group_idx].row_group_id, row_group_idx); - EXPECT_EQ(plan.row_groups[row_group_idx].first_file_row, - static_cast(row_group_idx * 2)); - ASSERT_EQ(plan.row_groups[row_group_idx].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].length, 2); - EXPECT_TRUE(plan.row_groups[row_group_idx].page_skip_plans.empty()); - } +TEST(ParquetScanSelectionTest, NativeLazySkipBitmapIsBounded) { + using format::parquet::detail::MAX_NATIVE_LAZY_SKIP_ROWS; + using format::parquet::detail::bounded_native_lazy_skip_rows; + EXPECT_EQ(bounded_native_lazy_skip_rows(1), 1); + EXPECT_EQ(bounded_native_lazy_skip_rows(MAX_NATIVE_LAZY_SKIP_ROWS + 1), + MAX_NATIVE_LAZY_SKIP_ROWS); } TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater) { @@ -583,7 +980,8 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater .first_file_row = ConditionCacheContext::GRANULE_SIZE, .row_group_rows = ConditionCacheContext::GRANULE_SIZE, .selected_ranges = {{.start = 0, .length = ConditionCacheContext::GRANULE_SIZE}}, - .page_skip_plans = {}}); + .page_skip_plans = {}, + .offset_indexes = {}}); format::parquet::ParquetScanScheduler scheduler; scheduler.set_plan(std::move(plan)); @@ -598,131 +996,6 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater EXPECT_EQ(ctx->base_granule, 0); } -TEST_F(ParquetScanTest, PageIndexIntersectsMultipleFiltersAndBuildsSkipPlan) { - write_page_index_pair_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest single_filter_request; - format::FileScanRequestBuilder single_filter_builder(&single_filter_request); - ASSERT_TRUE(single_filter_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - single_filter_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - format::parquet::RowGroupScanPlan single_filter_plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - single_filter_request, scan_range, false, &single_filter_plan) - .ok()); - ASSERT_EQ(single_filter_plan.row_groups.size(), 1); - const int64_t single_filter_rows = - count_range_rows(single_filter_plan.row_groups[0].selected_ranges); - - format::FileScanRequest intersect_request; - format::FileScanRequestBuilder intersect_builder(&intersect_request); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(1)).ok()); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 96)); - format::parquet::RowGroupScanPlan intersect_plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - intersect_request, scan_range, false, &intersect_plan) - .ok()); - ASSERT_EQ(intersect_plan.row_groups.size(), 1); - ASSERT_FALSE(intersect_plan.row_groups[0].selected_ranges.empty()); - const int64_t intersect_rows = count_range_rows(intersect_plan.row_groups[0].selected_ranges); - EXPECT_GT(single_filter_rows, intersect_rows); - EXPECT_GT(intersect_plan.row_groups[0].selected_ranges.front().start, 0); - const auto& last_range = intersect_plan.row_groups[0].selected_ranges.back(); - EXPECT_LT(last_range.start + last_range.length, 128); - EXPECT_GT(intersect_plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(intersect_plan.pruning_stats.selected_row_ranges, - intersect_plan.row_groups[0].selected_ranges.size()); - - auto id_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(id_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(id_skip_plan->second.leaf_column_id, 0); - EXPECT_FALSE(id_skip_plan->second.empty()); - auto score_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(1); - ASSERT_NE(score_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(score_skip_plan->second.leaf_column_id, 1); - EXPECT_FALSE(score_skip_plan->second.empty()); -} - -TEST_F(ParquetScanTest, PageIndexCanFullyFilterRowGroupAfterRangeIntersection) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::LT, 32)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 1); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 128); -} - -TEST_F(ParquetScanTest, PageIndexFullRangeWhenDisabledOrUnavailable) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); - - const bool old_enable_page_index = config::enable_parquet_page_index; - config::enable_parquet_page_index = false; - std::vector selected_ranges; - std::map page_skip_plans; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - parquet_file_reader.get(), file_schema, request, 0, 128, &selected_ranges, - &page_skip_plans, &pruning_stats) - .ok()); - config::enable_parquet_page_index = old_enable_page_index; - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 128); - EXPECT_TRUE(page_skip_plans.empty()); - EXPECT_EQ(pruning_stats.page_index_read_calls, 0); - - write_int_pair_parquet_file(_file_path, 6); - auto no_index_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto no_index_schema = build_file_schema(*no_index_reader); - format::FileScanRequest no_index_request; - no_index_request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - no_index_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 3)); - selected_ranges.clear(); - page_skip_plans.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - no_index_reader.get(), no_index_schema, no_index_request, 0, 6, - &selected_ranges, &page_skip_plans, &pruning_stats) - .ok()); - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 6); - EXPECT_TRUE(page_skip_plans.empty()); -} - TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -737,6 +1010,15 @@ TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { EXPECT_EQ(count_result.count, 6); EXPECT_TRUE(count_result.columns.empty()); + format::FileAggregateResult required_count_result; + format::FileAggregateRequest required_count_request; + required_count_request.agg_type = TPushAggOp::COUNT; + required_count_request.columns.push_back({.projection = field_projection(0)}); + ASSERT_TRUE(reader->get_aggregate_result(required_count_request, &required_count_result).ok()); + // The required scalar projection is retained for logical-type validation, but after that + // validation COUNT(id) can reuse the same selected-row-group footer count as COUNT(*). + EXPECT_EQ(required_count_result.count, 6); + format::FileAggregateResult minmax_result; format::FileAggregateRequest minmax_request; minmax_request.agg_type = TPushAggOp::MINMAX; @@ -753,6 +1035,96 @@ TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { EXPECT_EQ(minmax_result.columns[1].max_value.get(), 60); } +TEST_F(ParquetScanTest, AggregateCountRejectsRequiredUnsupportedScalarProjection) { + write_required_adjusted_time_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::COUNT; + request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult result; + const auto status = reader->get_aggregate_result(request, &result); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + std::string::npos); +} + +TEST_F(ParquetScanTest, CountStarIgnoresUnsupportedPlannerPlaceholder) { + write_required_adjusted_time_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + // A normal projection still requests the TIME_MILLIS value and must fail at open, before + // row-group pruning or physical INT32 statistics can hide the unsupported logical type. + auto projected_reader = create_reader(); + ASSERT_TRUE(projected_reader->init(&state).ok()); + auto projected_request = std::make_shared(); + format::FileScanRequestBuilder projected_builder(projected_request.get()); + ASSERT_TRUE(projected_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + const auto projected_status = projected_reader->open(projected_request); + EXPECT_TRUE(projected_status.is()) << projected_status; + + // Metadata COUNT(*) carries the same retained scan slot, but its aggregate request has no + // semantic column. The placeholder must not make open fail before footer row counts are used. + auto metadata_count_reader = create_reader(); + ASSERT_TRUE(metadata_count_reader->init(&state).ok()); + auto metadata_count_request = std::make_shared(); + format::FileScanRequestBuilder metadata_count_builder(metadata_count_request.get()); + ASSERT_TRUE(metadata_count_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + metadata_count_request->count_star_placeholder_columns.push_back(format::LocalColumnId(0)); + ASSERT_TRUE(metadata_count_reader->open(metadata_count_request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::COUNT; + format::FileAggregateResult aggregate_result; + ASSERT_TRUE( + metadata_count_reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); + EXPECT_EQ(aggregate_result.count, 2); + + // The marker is independent of aggregate eligibility. Simulate a position delete, + // which disables metadata COUNT and requires the reader to produce surviving rows. Parquet + // reads only the virtual row position, filters row 1, and fills a default placeholder for row 0 + // without validating or decoding either unsupported TIME_MILLIS value. + auto count_star_reader = create_reader(); + ASSERT_TRUE(count_star_reader->init(&state).ok()); + auto count_star_request = std::make_shared(); + format::FileScanRequestBuilder count_star_builder(count_star_request.get()); + ASSERT_TRUE(count_star_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(count_star_builder + .add_predicate_column(format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) + .ok()); + count_star_request->count_star_placeholder_columns.push_back(format::LocalColumnId(0)); + + const std::vector deleted_rows {1}; + auto delete_predicate = std::make_shared(deleted_rows); + const auto row_position = count_star_request->local_positions.at( + format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)); + delete_predicate->add_child(VSlotRef::create_shared( + cast_set(row_position.value()), cast_set(row_position.value()), -1, + std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + auto delete_context = VExprContext::create_shared(std::move(delete_predicate)); + ASSERT_TRUE(delete_context->prepare(&state, RowDescriptor()).ok()); + ASSERT_TRUE(delete_context->open(&state).ok()); + count_star_request->delete_conjuncts.push_back(std::move(delete_context)); + ASSERT_TRUE(count_star_reader->open(count_star_request).ok()); + + std::vector file_schema; + ASSERT_TRUE(count_star_reader->get_schema(&file_schema).ok()); + file_schema.push_back(format::row_position_column_definition()); + auto block = build_file_block(file_schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(count_star_reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 1); + EXPECT_EQ(block.get_by_position(0).column->size(), 1); + const auto& positions = + assert_cast(*block.get_by_position(row_position.value()).column); + ASSERT_EQ(positions.size(), 1); + EXPECT_EQ(positions.get_element(0), 0); +} + TEST_F(ParquetScanTest, AggregateMinMaxRejectsInexactBinaryStatistics) { write_binary_minmax_parquet_file(_file_path); auto reader = create_reader(); @@ -953,6 +1325,37 @@ TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { EXPECT_EQ(row_ids, std::vector({2, 3})); } +TEST_F(ParquetScanTest, PredicateOnlyGlobalRowIdKeepsSignedFileLocalId) { + write_int_pair_parquet_file(_file_path, 6, false); + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(0, -1, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + const auto global_rowid = format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(global_rowid).ok()); + request->predicate_only_columns.push_back(global_rowid); + const auto global_rowid_position = request->local_positions.at(global_rowid); + request->conjuncts.push_back(create_always_true_single_column_conjunct( + cast_set(global_rowid_position.value()))); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows, 6); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {1, 2, 3, 4, 5, 6})); +} + TEST_F(ParquetScanTest, EmptyScanPlanReturnsEofWithoutReadingColumns) { write_int_pair_parquet_file(_file_path, 2); auto reader = create_reader(); @@ -1018,7 +1421,8 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { while (!eof) { Block block = build_file_block(schema); size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -1037,6 +1441,9 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); EXPECT_EQ(counter_value(profile, "SelectedRows"), 2); EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 4); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 1); + EXPECT_GT(counter_value(profile, "PredicateCompactionBytes"), 0); + ASSERT_NE(profile.get_counter("PredicateCompactionTime"), nullptr); EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 10); EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 2); @@ -1078,6 +1485,286 @@ TEST_F(ParquetScanTest, PredicateColumnsSkipUnreadColumnsWhenFirstPredicateFilte EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 6); } +TEST_F(ParquetScanTest, PredicateOnlyColumnDropsPayloadAfterFiltering) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedPlainComparisonUsesPhysicalFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedByteStreamSplitUsesFixedWidthFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false, ::parquet::Encoding::BYTE_STREAM_SPLIT); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedDeltaBinaryPackedUsesFixedWidthFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false, ::parquet::Encoding::DELTA_BINARY_PACKED); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PlainPredicateDirectPathCrossesScratchProbeCadence) { + constexpr int64_t ROWS = 32; + std::vector ids(ROWS); + std::vector scores(ROWS); + std::iota(ids.begin(), ids.end(), 1); + std::iota(scores.begin(), scores.end(), 10); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(scores)}); + write_table(_file_path, table, ROWS, false, false, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector file_schema; + ASSERT_TRUE(reader->get_schema(&file_schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(file_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, ROWS); + // Direct predicate evaluation must survive multiple 16-batch scratch probes; otherwise this + // path can retain a previous outlier for the whole row group without ever aging its capacity. + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), ROWS); +} + +TEST_F(ParquetScanTest, PredicateOnlyUint32FallsBackBeforeRawPlainDecode) { + write_uint32_pair_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_BIGINT); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back( + create_int64_direct_greater_conjunct(0, std::numeric_limits::max())); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows, 2); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {20, 30})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 0); +} + +TEST_F(ParquetScanTest, FixedWidthPredicateReportsUnsupportedEncodingAfterProgress) { + write_misdeclared_two_page_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("encoding"), std::string::npos) << status; +} + +TEST_F(ParquetScanTest, PlainDirectPathKeepsPayloadForMultiColumnResidual) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 42)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + // A residual expression still consumes id, so raw filtering must leave its payload available. + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 0); +} + // Scenario: every physical batch in every row group is rejected. Predicate readers reach each row // group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that // reader and its pending lag instead of issuing SkipRecords for values that can never be observed. @@ -1108,10 +1795,12 @@ TEST_F(ParquetScanTest, FullyFilteredRowGroupsDropPendingLazyReaders) { } EXPECT_EQ(total_rows, 0); - EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 6); + // The first one-row probe grows after rejection and remains grown across consecutive empty + // row groups, so later two-row groups are consumed in one predicate batch each. + EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); } // Scenario: row group 0 is fully filtered and leaves two pending lazy rows. Reset must discard that @@ -1153,6 +1842,85 @@ TEST_F(ParquetScanTest, PendingLazySkipDoesNotCrossRowGroupReset) { EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); } +TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunks) { + constexpr size_t ROWS = 70000; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, ROWS - 1)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_element(0), ROWS - 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_element(0), ROWS + 9); + EXPECT_EQ(int32_data_column(*block.get_by_position(2).column).get_element(0), ROWS + 19); + EXPECT_LT(counter_value(profile, "TotalBatches"), 32); +} + +TEST_F(ParquetScanTest, EmptyPrefixNeverWidensReturnedBatchPastCallerCap) { + constexpr size_t ROWS = 300; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + int32_t expected_id = 32; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + // The first 32-row probe is empty. Its feedback may accelerate later predicate work, but + // each pending-output slice must preserve the caller's memory-derived row cap. + ASSERT_LE(rows, 32); + const auto& ids = int32_data_column(*block.get_by_position(0).column); + const auto& first_lazy = int32_data_column(*block.get_by_position(1).column); + const auto& second_lazy = int32_data_column(*block.get_by_position(2).column); + ASSERT_EQ(first_lazy.size(), rows); + ASSERT_EQ(second_lazy.size(), rows); + for (size_t row = 0; row < rows; ++row, ++expected_id) { + EXPECT_EQ(ids.get_element(row), expected_id); + EXPECT_EQ(first_lazy.get_element(row), expected_id + 10); + EXPECT_EQ(second_lazy.get_element(row), expected_id + 20); + } + total_rows += rows; + } + EXPECT_EQ(total_rows, ROWS - 32); + EXPECT_EQ(expected_id, ROWS); +} + // Scenario: a nested lazy column stays behind while id=1 is rejected. Flushing skip(1) must consume // the complete repetition/definition-level span for the first list, then materialize the remaining // two parent rows without corrupting their child boundaries. @@ -1287,5 +2055,30 @@ TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); } +TEST_F(ParquetScanTest, OpenDefersPageIndexProbeToCurrentRowGroup) { + write_multi_row_group_page_index_parquet_file(_file_path); + RuntimeProfile profile("lazy_page_index_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, -1)); + ASSERT_TRUE(reader->open(request).ok()); + + // LIMIT can stop after the first block, so open must not pay remote index I/O for later groups. + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 128); + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 1); +} + } // namespace } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index e620ed718efbf2..74781c0e532b31 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -16,512 +16,706 @@ // under the License. #include -#include +#include #include +#include #include #include "core/assert_cast.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" namespace doris::format::parquet { -namespace { - -std::vector> build_fields( - const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - EXPECT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - return fields; +TEST(ParquetSchemaTest, NativeMetadataAcceptsRequiredRootWithoutColumns) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(0); + root.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root}).ok()); + EXPECT_EQ(descriptor.size(), 0); + + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root}); + thrift_metadata.__set_num_rows(0); + NativeParquetMetadata metadata(std::move(thrift_metadata), 0); + // An empty physical tree remains useful for metadata-only COUNT(*); rejecting it here changes + // the compatibility contract before request planning can select that path. + EXPECT_TRUE(metadata.init_schema(false, false).ok()); + EXPECT_EQ(metadata.schema().size(), 0); } -Status build_status(const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - return build_parquet_column_schema(descriptor, &fields); -} +TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); -} // namespace - -TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("required_i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("optional_i64", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT64, - ::parquet::ConvertedType::NONE, -1, -1, -1, 42), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->local_id, 0); - EXPECT_EQ(fields[0]->name, "required_i32"); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->leaf_column_id, 0); - EXPECT_EQ(fields[0]->nullable_definition_level, 0); - EXPECT_FALSE(fields[0]->type->is_nullable()); - - EXPECT_EQ(fields[1]->local_id, 1); - EXPECT_EQ(fields[1]->parquet_field_id, 42); - EXPECT_EQ(fields[1]->leaf_column_id, 1); - EXPECT_EQ(fields[1]->nullable_definition_level, 1); - EXPECT_TRUE(fields[1]->type->is_nullable()); -} + tparquet::SchemaElement protocol; + protocol.__set_name("protocol"); + protocol.__set_num_children(2); + protocol.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + protocol.__set_field_id(10); -TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(false, - ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64), - ::parquet::schema::PrimitiveNode::Make("i8", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8), - ::parquet::schema::PrimitiveNode::Make("plain", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - }); - - ASSERT_EQ(fields.size(), 3); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(fields[0]->type_descriptor.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - EXPECT_TRUE(fields[0]->type_descriptor.is_timestamp); - EXPECT_FALSE(fields[0]->type_descriptor.timestamp_is_adjusted_to_utc); - - EXPECT_EQ(remove_nullable(fields[1]->type)->get_primitive_type(), TYPE_TINYINT); - EXPECT_EQ(fields[1]->type_descriptor.integer_bit_width, 8); - EXPECT_FALSE(fields[1]->type_descriptor.is_unsigned_integer); - - EXPECT_EQ(remove_nullable(fields[2]->type)->get_primitive_type(), TYPE_DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.physical_type, ::parquet::Type::DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::NONE); -} + tparquet::SchemaElement min_reader; + min_reader.__set_name("minReaderVersion"); + min_reader.__set_type(tparquet::Type::INT32); + min_reader.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + min_reader.__set_field_id(11); + + tparquet::SchemaElement min_writer = min_reader; + min_writer.__set_name("minWriterVersion"); + min_writer.__set_field_id(12); -TEST(ParquetSchemaTest, StructMakesDataTypeChildrenNullableAndPropagatesLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::PrimitiveNode::Make("a", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - })}); + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, protocol, min_reader, min_writer}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& struct_schema = *fields[0]; - EXPECT_EQ(struct_schema.kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(struct_schema.nullable_definition_level, 1); - ASSERT_EQ(struct_schema.children.size(), 2); - EXPECT_EQ(struct_schema.children[0]->definition_level, 1); - EXPECT_EQ(struct_schema.children[1]->definition_level, 2); - EXPECT_EQ(struct_schema.max_definition_level, 2); - - const auto& struct_type = - assert_cast(*remove_nullable(struct_schema.type)); - ASSERT_EQ(struct_type.get_elements().size(), 2); - EXPECT_TRUE(struct_type.get_elements()[0]->is_nullable()); - EXPECT_TRUE(struct_type.get_elements()[1]->is_nullable()); + EXPECT_EQ(fields[0]->name, "protocol"); + EXPECT_EQ(fields[0]->parquet_field_id, 10); + ASSERT_EQ(fields[0]->children.size(), 2); + EXPECT_EQ(fields[0]->children[0]->name, "minReaderVersion"); + EXPECT_EQ(fields[0]->children[0]->leaf_column_id, 0); + EXPECT_EQ(fields[0]->children[1]->name, "minWriterVersion"); + EXPECT_EQ(fields[0]->children[1]->leaf_column_id, 1); + + std::shared_ptr mapping; + ASSERT_TRUE(build_native_schema_node(fields[0]->type, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("minReaderVersion")); + EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); + ASSERT_NE(mapping->child("minReaderVersion"), nullptr); } - -TEST(ParquetSchemaTest, ListCompatibilityRulesAndLevels) { - const auto standard_list = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - const auto structural_array = ::parquet::schema::GroupNode::Make( - "ys", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "array", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({standard_list, structural_array}); - ASSERT_EQ(fields.size(), 2); - - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(xs.definition_level, 2); - EXPECT_EQ(xs.repetition_level, 1); - ASSERT_EQ(xs.children.size(), 1); - EXPECT_EQ(xs.children[0]->name, "element"); - EXPECT_EQ(xs.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_TRUE(xs.children[0]->type->is_nullable()); - const auto& xs_type = assert_cast(*remove_nullable(xs.type)); - EXPECT_TRUE(xs_type.get_nested_type()->is_nullable()); - - const auto& ys = *fields[1]; - EXPECT_EQ(ys.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(ys.children.size(), 1); - EXPECT_EQ(ys.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(remove_nullable(ys.children[0]->type)->get_primitive_type(), TYPE_STRUCT); +TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement adjusted_time; + adjusted_time.__set_name("time_ms"); + adjusted_time.__set_type(tparquet::Type::INT32); + adjusted_time.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + adjusted_time.__set_logicalType(tparquet::LogicalType()); + adjusted_time.logicalType.__set_TIME(tparquet::TimeType()); + adjusted_time.logicalType.TIME.__set_isAdjustedToUTC(true); + adjusted_time.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + adjusted_time.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, adjusted_time}).ok()); + native_schema.assign_ids(); + std::vector> fields; + const auto status = build_parquet_column_schema(native_schema, &fields); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(fields.size(), 1); + // Preserve the unsupported marker in metadata; request-level validation decides whether the + // leaf is a real projection or an ignorable COUNT(*) placeholder. + EXPECT_EQ(fields[0]->type_descriptor.unsupported_reason, + "Parquet TIME with isAdjustedToUTC=true is not supported"); } -TEST(ParquetSchemaTest, LegacyListElementResolutionRulesArePreserved) { - const auto two_level_list = ::parquet::schema::GroupNode::Make( - "two_level", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - const auto tuple_list = ::parquet::schema::GroupNode::Make( - "tuple_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "tuple_list_tuple", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - const auto multi_field_list = ::parquet::schema::GroupNode::Make( - "records", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("name", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::LIST); - const auto fields = build_fields({two_level_list, tuple_list, multi_field_list}); - ASSERT_EQ(fields.size(), 3); - - const auto& two_level = *fields[0]; - EXPECT_EQ(two_level.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(two_level.definition_level, 2); - EXPECT_EQ(two_level.repetition_level, 1); - ASSERT_EQ(two_level.children.size(), 1); - EXPECT_EQ(two_level.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(two_level.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(two_level.children[0]->type)->get_primitive_type(), TYPE_INT); - - const auto& tuple = *fields[1]; - ASSERT_EQ(tuple.children.size(), 1); - EXPECT_EQ(tuple.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(tuple.children[0]->name, "element"); - ASSERT_EQ(tuple.children[0]->children.size(), 1); - EXPECT_EQ(tuple.children[0]->children[0]->name, "value"); - - const auto& multi_field = *fields[2]; - ASSERT_EQ(multi_field.children.size(), 1); - EXPECT_EQ(multi_field.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(multi_field.children[0]->children.size(), 2); - EXPECT_EQ(multi_field.children[0]->children[0]->name, "id"); - EXPECT_EQ(multi_field.children[0]->children[1]->name, "name"); +TEST(ParquetSchemaTest, NativeOversizedByteArrayDecimalIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement decimal; + decimal.__set_name("decimal_77"); + decimal.__set_type(tparquet::Type::BYTE_ARRAY); + decimal.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + decimal.__set_logicalType(tparquet::LogicalType()); + decimal.logicalType.__set_DECIMAL(tparquet::DecimalType()); + decimal.logicalType.DECIMAL.__set_precision(77); + decimal.logicalType.DECIMAL.__set_scale(2); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, decimal}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); + ASSERT_EQ(fields.size(), 1); + EXPECT_TRUE(fields[0]->type_descriptor.is_decimal); + EXPECT_FALSE(fields[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find("precision 77"), + std::string::npos); } -TEST(ParquetSchemaTest, NestedRepeatedInsideListElementIsWrappedOnce) { - const auto list_with_repeated_child = ::parquet::schema::GroupNode::Make( - "outer", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({list_with_repeated_child}); +TEST(ParquetSchemaTest, NativeUnknownLogicalTypeRetainsPhysicalFallback) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement interval; + interval.__set_name("duration"); + interval.__set_type(tparquet::Type::FIXED_LEN_BYTE_ARRAY); + interval.__set_type_length(12); + interval.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + interval.__set_logicalType(tparquet::LogicalType()); + interval.__set_converted_type(tparquet::ConvertedType::INTERVAL); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, interval}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& outer = *fields[0]; - EXPECT_EQ(outer.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(outer.children.size(), 1); - const auto& element = *outer.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.children[0]->name, "items"); - ASSERT_EQ(element.children[0]->children.size(), 1); - EXPECT_EQ(element.children[0]->children[0]->name, "element"); + EXPECT_TRUE(fields[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_EQ(fields[0]->type_descriptor.doris_type->get_primitive_type(), TYPE_STRING); } -TEST(ParquetSchemaTest, ListWrapperWithLogicalAnnotationIsPreservedAsElement) { - const auto annotated_repeated_group = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - EXPECT_FALSE(build_status({annotated_repeated_group}).ok()); - - const auto nested_list_wrapper = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({nested_list_wrapper}); - ASSERT_EQ(fields.size(), 1); - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(xs.children.size(), 1); - const auto& element = *xs.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.name, "element"); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(element.children[0]->type)->get_primitive_type(), TYPE_INT); +TEST(ParquetSchemaTest, NativeGroupPrimitiveLogicalTypesAreRejected) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement child; + child.__set_name("value"); + child.__set_type(tparquet::Type::BYTE_ARRAY); + child.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + std::vector> logical_types; + tparquet::LogicalType string_type; + string_type.__set_STRING(tparquet::StringType()); + logical_types.emplace_back("STRING", string_type); + tparquet::LogicalType enum_type; + enum_type.__set_ENUM(tparquet::EnumType()); + logical_types.emplace_back("ENUM", enum_type); + tparquet::LogicalType decimal_type; + decimal_type.__set_DECIMAL(tparquet::DecimalType()); + logical_types.emplace_back("DECIMAL", decimal_type); + tparquet::LogicalType date_type; + date_type.__set_DATE(tparquet::DateType()); + logical_types.emplace_back("DATE", date_type); + tparquet::LogicalType time_type; + time_type.__set_TIME(tparquet::TimeType()); + logical_types.emplace_back("TIME", time_type); + tparquet::LogicalType timestamp_type; + timestamp_type.__set_TIMESTAMP(tparquet::TimestampType()); + logical_types.emplace_back("TIMESTAMP", timestamp_type); + tparquet::LogicalType integer_type; + integer_type.__set_INTEGER(tparquet::IntType()); + logical_types.emplace_back("INTEGER", integer_type); + tparquet::LogicalType uuid_type; + uuid_type.__set_UUID(tparquet::UUIDType()); + logical_types.emplace_back("UUID", uuid_type); + tparquet::LogicalType float16_type; + float16_type.__set_FLOAT16(tparquet::Float16Type()); + logical_types.emplace_back("FLOAT16", float16_type); + + for (const auto& [name, logical_type] : logical_types) { + SCOPED_TRACE(name); + tparquet::SchemaElement group; + group.__set_name("bad_" + name + "_group"); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + group.__set_logicalType(logical_type); + NativeFieldDescriptor native_schema; + const auto status = native_schema.parse_from_thrift({root, group, child}); + EXPECT_FALSE(status.ok()); + if (name == "ENUM") { + EXPECT_NE(status.to_string().find("Logical type Enum cannot be applied to group node"), + std::string::npos); + } + } + + for (const auto converted_type : + {tparquet::ConvertedType::UTF8, tparquet::ConvertedType::ENUM, + tparquet::ConvertedType::DECIMAL, tparquet::ConvertedType::DATE, + tparquet::ConvertedType::TIME_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS, + tparquet::ConvertedType::INT_32, tparquet::ConvertedType::JSON, + tparquet::ConvertedType::BSON}) { + SCOPED_TRACE(tparquet::to_string(converted_type)); + tparquet::SchemaElement group; + group.__set_name("bad_converted_group"); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + group.__set_converted_type(converted_type); + NativeFieldDescriptor native_schema; + EXPECT_FALSE(native_schema.parse_from_thrift({root, group, child}).ok()); + } } -TEST(ParquetSchemaTest, MapWrapperIsFoldedAndOptionalKeyIsAllowed) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); +TEST(ParquetSchemaTest, NativeFlatLeafValueCountMustMatchRowCount) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + tparquet::Statistics statistics; + statistics.__set_null_count(2); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(2); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(1); + thrift_metadata.__set_row_groups({row_group}); + + NativeParquetMetadata metadata(std::move(thrift_metadata), 0); + const auto status = metadata.init_schema(false, false); + EXPECT_TRUE(status.is()) << status; +} - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_EQ(map_schema.kind, ParquetColumnSchemaKind::MAP); - EXPECT_EQ(map_schema.definition_level, 2); - EXPECT_EQ(map_schema.repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->name, "key"); - EXPECT_EQ(map_schema.children[1]->name, "value"); - EXPECT_TRUE(map_schema.children[0]->type->is_nullable()); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeStringAnnotationsAndTimeUnitsPreserveLogicalTypes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(8); + + auto binary_leaf = [](const std::string& name) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return leaf; + }; + auto logical_enum = binary_leaf("logical_enum"); + logical_enum.__set_logicalType(tparquet::LogicalType()); + logical_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto logical_bson = binary_leaf("logical_bson"); + logical_bson.__set_logicalType(tparquet::LogicalType()); + logical_bson.logicalType.__set_BSON(tparquet::BsonType()); + auto converted_enum = binary_leaf("converted_enum"); + converted_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto converted_bson = binary_leaf("converted_bson"); + converted_bson.__set_converted_type(tparquet::ConvertedType::BSON); + + auto logical_time = [](const std::string& name, bool millis) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(millis ? tparquet::Type::INT32 : tparquet::Type::INT64); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + leaf.__set_logicalType(tparquet::LogicalType()); + leaf.logicalType.__set_TIME(tparquet::TimeType()); + leaf.logicalType.TIME.__set_isAdjustedToUTC(false); + leaf.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + if (millis) { + leaf.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + } else { + leaf.logicalType.TIME.unit.__set_MICROS(tparquet::MicroSeconds()); + } + return leaf; + }; + auto logical_millis = logical_time("logical_millis", true); + auto logical_micros = logical_time("logical_micros", false); + auto converted_millis = logical_time("converted_millis", true); + converted_millis.__isset.logicalType = false; + converted_millis.__set_converted_type(tparquet::ConvertedType::TIME_MILLIS); + auto converted_micros = logical_time("converted_micros", false); + converted_micros.__isset.logicalType = false; + converted_micros.__set_converted_type(tparquet::ConvertedType::TIME_MICROS); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift({root, logical_enum, logical_bson, converted_enum, + converted_bson, logical_millis, logical_micros, + converted_millis, converted_micros}) + .ok()); + for (size_t field = 0; field < 4; ++field) { + EXPECT_EQ(remove_nullable(descriptor.get_column(field)->data_type)->get_primitive_type(), + TYPE_STRING); + } + EXPECT_EQ(remove_nullable(descriptor.get_column(4)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(5)->data_type)->get_scale(), 6); + EXPECT_EQ(remove_nullable(descriptor.get_column(6)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(7)->data_type)->get_scale(), 6); } -TEST(ParquetSchemaTest, StandardMapLevelsAndDataTypesAreBuiltFromEntryContext) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::REQUIRED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); +TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { + auto valid_root = []() { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + return root; + }; + auto valid_leaf = []() { + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + return leaf; + }; + + NativeFieldDescriptor descriptor; + auto primitive_root = valid_root(); + primitive_root.__set_type(tparquet::Type::INT32); + EXPECT_FALSE(descriptor.parse_from_thrift({primitive_root, valid_leaf()}).ok()); + + auto repeated_root = valid_root(); + repeated_root.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + EXPECT_FALSE(descriptor.parse_from_thrift({repeated_root, valid_leaf()}).ok()); + + auto missing_repetition = valid_leaf(); + missing_repetition.__isset.repetition_type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_repetition}).ok()); + + auto dual_kind = valid_leaf(); + dual_kind.__set_num_children(1); + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), dual_kind}).ok()); + + auto legacy_zero_children = valid_leaf(); + legacy_zero_children.__set_num_children(0); + EXPECT_TRUE(descriptor.parse_from_thrift({valid_root(), legacy_zero_children}).ok()); + + auto missing_kind = valid_leaf(); + missing_kind.__isset.type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_kind}).ok()); +} - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_FALSE(map_schema.type->is_nullable()); - EXPECT_EQ(map_schema.definition_level, 1); - EXPECT_EQ(map_schema.repetition_level, 1); - EXPECT_EQ(map_schema.repeated_repetition_level, 1); - EXPECT_EQ(map_schema.max_definition_level, 2); - EXPECT_EQ(map_schema.max_repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->definition_level, 1); - EXPECT_EQ(map_schema.children[0]->repetition_level, 1); - EXPECT_EQ(map_schema.children[1]->definition_level, 2); - EXPECT_EQ(map_schema.children[1]->nullable_definition_level, 2); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeSchemaRejectsUnboundedChildCountsBeforeAllocation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(std::numeric_limits::max()); + NativeFieldDescriptor descriptor; + EXPECT_FALSE(descriptor.parse_from_thrift({root}).ok()); + + root.__set_num_children(1); + tparquet::SchemaElement nested; + nested.__set_name("nested"); + nested.__set_num_children(std::numeric_limits::max()); + nested.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + EXPECT_FALSE(descriptor.parse_from_thrift({root, nested}).ok()); } -TEST(ParquetSchemaTest, BareRepeatedFieldsAreWrappedAsLists) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("items", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32), - ::parquet::schema::GroupNode::Make( - "links", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("url", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("rank", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[0]->children.size(), 1); - EXPECT_EQ(fields[0]->children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->children[0]->name, "element"); - - EXPECT_EQ(fields[1]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[1]->children.size(), 1); - EXPECT_EQ(fields[1]->children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(fields[1]->children[0]->name, "element"); +std::vector nested_native_schema(size_t depth) { + std::vector schema; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + schema.push_back(root); + for (size_t level = 0; level < depth; ++level) { + tparquet::SchemaElement group; + group.__set_name("g" + std::to_string(level)); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(group); + } + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(leaf); + return schema; } -TEST(ParquetSchemaTest, DeepLevelChainPropagatesDefinitionAndRepetitionLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "inner", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})})}); +TEST(ParquetSchemaTest, NativeSchemaBoundsRecursiveDepth) { + NativeFieldDescriptor accepted; + EXPECT_TRUE(accepted.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH)).ok()); + NativeFieldDescriptor rejected; + EXPECT_FALSE( + rejected.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH + 1)).ok()); +} - ASSERT_EQ(fields.size(), 1); - const auto& s = *fields[0]; - EXPECT_EQ(s.definition_level, 1); - EXPECT_EQ(s.nullable_definition_level, 1); - ASSERT_EQ(s.children.size(), 1); - const auto& inner = *s.children[0]; - EXPECT_EQ(inner.definition_level, 2); - EXPECT_EQ(inner.nullable_definition_level, 2); - ASSERT_EQ(inner.children.size(), 1); - const auto& items = *inner.children[0]; - EXPECT_EQ(items.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(items.definition_level, 3); - EXPECT_EQ(items.repetition_level, 1); - EXPECT_EQ(items.repeated_ancestor_definition_level, 3); - EXPECT_EQ(items.repeated_repetition_level, 1); - EXPECT_EQ(items.max_definition_level, 3); - EXPECT_EQ(items.max_repetition_level, 1); - ASSERT_EQ(items.children.size(), 1); - EXPECT_EQ(items.children[0]->definition_level, 3); - EXPECT_EQ(items.children[0]->repetition_level, 1); +TEST(ParquetSchemaTest, NativeListTupleCompatibilityRequiresEnclosingListName) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto list = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema.__set_converted_type(tparquet::ConvertedType::LIST); + return schema; + }; + auto wrapper = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + return schema; + }; + auto item = []() { + tparquet::SchemaElement schema; + schema.__set_name("item"); + schema.__set_type(tparquet::Type::INT32); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return schema; + }; + + NativeFieldDescriptor mismatched; + ASSERT_TRUE(mismatched.parse_from_thrift({root(), list("xs"), wrapper("other_tuple"), item()}) + .ok()); + const auto* mismatched_element = &mismatched.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(mismatched_element->data_type)->get_primitive_type(), TYPE_INT); + + NativeFieldDescriptor matching; + ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}).ok()); + const auto* matching_element = &matching.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(matching_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(matching_element->children.size(), 1); + EXPECT_EQ(matching_element->children[0].name, "item"); } -TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { - auto empty_root = ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, - ::parquet::schema::NodeVector {}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(empty_root); +TEST(ParquetSchemaTest, NativeListPreservesRepeatedAndAnnotatedElementWrappers) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto group = [](const std::string& name, tparquet::FieldRepetitionType::type repetition) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(repetition); + return schema; + }; + auto outer_list = group("outer", tparquet::FieldRepetitionType::OPTIONAL); + outer_list.__set_converted_type(tparquet::ConvertedType::LIST); + + auto repeated_wrapper = group("list", tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement repeated_items; + repeated_items.__set_name("items"); + repeated_items.__set_type(tparquet::Type::INT32); + repeated_items.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + NativeFieldDescriptor repeated; + ASSERT_TRUE(repeated.parse_from_thrift({root(), outer_list, repeated_wrapper, repeated_items}) + .ok()); + const auto* repeated_element = &repeated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(repeated_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(repeated_element->children.size(), 1); + EXPECT_EQ(repeated_element->children[0].name, "items"); + EXPECT_EQ(remove_nullable(repeated_element->children[0].data_type)->get_primitive_type(), + TYPE_ARRAY); + + auto annotated_wrapper = repeated_wrapper; + annotated_wrapper.__set_converted_type(tparquet::ConvertedType::LIST); + auto nested_wrapper = repeated_wrapper; + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT32); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + NativeFieldDescriptor annotated; + ASSERT_TRUE(annotated + .parse_from_thrift( + {root(), outer_list, annotated_wrapper, nested_wrapper, value}) + .ok()); + const auto* annotated_element = &annotated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(annotated_element->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(annotated_element->children.size(), 1); + EXPECT_EQ(remove_nullable(annotated_element->children[0].data_type)->get_primitive_type(), + TYPE_INT); +} + +TEST(ParquetSchemaTest, NativeSchemaRecognizesLogicalTypeOnlyListAndMap) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + + tparquet::SchemaElement list; + list.__set_name("items"); + list.__set_num_children(1); + list.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + list.__set_logicalType(tparquet::LogicalType()); + list.logicalType.__set_LIST(tparquet::ListType()); + tparquet::SchemaElement list_wrapper; + list_wrapper.__set_name("list"); + list_wrapper.__set_num_children(1); + list_wrapper.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement element; + element.__set_name("element"); + element.__set_type(tparquet::Type::INT32); + element.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + tparquet::SchemaElement map; + map.__set_name("attributes"); + map.__set_num_children(1); + map.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + map.__set_logicalType(tparquet::LogicalType()); + map.logicalType.__set_MAP(tparquet::MapType()); + tparquet::SchemaElement key_value; + key_value.__set_name("key_value"); + key_value.__set_num_children(2); + key_value.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::BYTE_ARRAY); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT64); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift( + {root, list, list_wrapper, element, map, key_value, key, value}) + .ok()); + ASSERT_EQ(descriptor.size(), 2); + EXPECT_EQ(remove_nullable(descriptor.get_column(0)->data_type)->get_primitive_type(), + TYPE_ARRAY); + EXPECT_EQ(remove_nullable(descriptor.get_column(1)->data_type)->get_primitive_type(), TYPE_MAP); +} - EXPECT_FALSE(build_parquet_column_schema(descriptor, nullptr).ok()); +TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingMetadata) { + auto make_metadata = []() { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + tparquet::SchemaElement first; + first.__set_name("a"); + first.__set_type(tparquet::Type::INT32); + first.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement second = first; + second.__set_name("b"); + metadata.__set_schema({root, first, second}); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + return metadata; + }; + + { + NativeParquetMetadata metadata(make_metadata(), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } + { + auto thrift = make_metadata(); + thrift.row_groups[0].columns.resize(2); + tparquet::ColumnMetaData first_meta; + first_meta.__set_type(tparquet::Type::INT32); + thrift.row_groups[0].columns[0].__set_meta_data(first_meta); + NativeParquetMetadata metadata(std::move(thrift), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } +} +TEST(ParquetSchemaTest, NativeProjectionUsesResolvedIdentityBeforeCaseInsensitiveFallback) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement group; + group.__set_name("s"); + group.__set_num_children(3); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + tparquet::SchemaElement upper; + upper.__set_name("Value"); + upper.__set_type(tparquet::Type::INT32); + upper.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + upper.__set_field_id(1); + tparquet::SchemaElement lower = upper; + lower.__set_name("value"); + lower.__set_field_id(2); + tparquet::SchemaElement other = upper; + other.__set_name("other"); + other.__set_field_id(3); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, group, upper, lower, other}).ok()); + descriptor.assign_ids(); std::vector> fields; ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - EXPECT_TRUE(fields.empty()); -} -TEST(ParquetSchemaTest, RejectInvalidListMapAndUnsupportedTime) { - const auto bad_list = ::parquet::schema::GroupNode::Make( - "bad_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({bad_list}).ok()); - - const auto bad_map = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("entry", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({bad_map}).ok()); - - const auto converted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS); - const auto status = build_status({converted_time}); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); + const auto int_type = make_nullable(std::make_shared()); + std::shared_ptr mapping; + auto other_only = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"other"})); + ASSERT_TRUE(build_native_schema_node(other_only, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("other")); + + auto exact_case = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"Value"})); + ASSERT_TRUE(build_native_schema_node(exact_case, *fields[0], &mapping).ok()); + EXPECT_EQ(mapping->file_child_name("Value"), "Value"); + + auto ambiguous_fallback = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"VALUE"})); + const auto status = build_native_schema_node(ambiguous_fallback, *fields[0], &mapping); + EXPECT_TRUE(status.is()) << status; } -TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { - const auto zero_child_list = ::parquet::schema::GroupNode::Make( - "zero_child_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make("list", ::parquet::Repetition::REPEATED, - ::parquet::schema::NodeVector {})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({zero_child_list}).ok()); - - const auto repeated_list = ::parquet::schema::GroupNode::Make( - "repeated_list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({repeated_list}).ok()); - - const auto map_with_two_fields = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::GroupNode::Make( - "entry1", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - ::parquet::schema::GroupNode::Make( - "entry2", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_with_two_fields}).ok()); - - const auto non_repeated_map_entry = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({non_repeated_map_entry}).ok()); - - const auto map_entry_with_one_child = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_entry_with_one_child}).ok()); - - const auto repeated_map = ::parquet::schema::GroupNode::Make( - "repeated_map", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({repeated_map}).ok()); +TEST(ParquetSchemaTest, NativeSetMapKeyValueWrapperRemainsSingleListElement) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement set; + set.__set_name("tags"); + set.__set_num_children(1); + set.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + set.__set_converted_type(tparquet::ConvertedType::MAP); + tparquet::SchemaElement wrapper; + wrapper.__set_name("key_value"); + wrapper.__set_num_children(1); + wrapper.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + wrapper.__set_converted_type(tparquet::ConvertedType::MAP_KEY_VALUE); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::INT32); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, set, wrapper, key}).ok()); + const auto* column = descriptor.get_column(0); + EXPECT_EQ(remove_nullable(column->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(column->children.size(), 1); + EXPECT_EQ(remove_nullable(column->children[0].data_type)->get_primitive_type(), TYPE_INT); } -TEST(ParquetSchemaTest, LogicalUtcTimeIsRejected) { - const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32); - const auto status = build_status({adjusted_time}); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); +TEST(ParquetSchemaTest, NativeListPreservesLegacyMapKeyValueElement) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement list; + list.__set_name("entries"); + list.__set_num_children(1); + list.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + list.__set_converted_type(tparquet::ConvertedType::LIST); + tparquet::SchemaElement element; + element.__set_name("element"); + element.__set_num_children(1); + element.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + element.__set_converted_type(tparquet::ConvertedType::MAP_KEY_VALUE); + tparquet::SchemaElement map; + map.__set_name("map"); + map.__set_num_children(2); + map.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::INT32); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::BYTE_ARRAY); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, list, element, map, key, value}).ok()); + const auto* column = descriptor.get_column(0); + EXPECT_EQ(remove_nullable(column->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(column->children.size(), 1); + EXPECT_EQ(remove_nullable(column->children[0].data_type)->get_primitive_type(), TYPE_MAP); + ASSERT_EQ(column->children[0].children.size(), 2); + EXPECT_EQ(remove_nullable(column->children[0].children[0].data_type)->get_primitive_type(), + TYPE_INT); + EXPECT_EQ(remove_nullable(column->children[0].children[1].data_type)->get_primitive_type(), + TYPE_STRING); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp deleted file mode 100644 index c35138e3263723..00000000000000 --- a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp +++ /dev/null @@ -1,459 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_decimal.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/types.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -class ParquetSerdeReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_serde_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "serde.parquet").string(); - write_parquet_file(); - open_file(_file_path); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(0x0000).ok()); - EXPECT_TRUE(builder.Append(0x8000).ok()); - EXPECT_TRUE(builder.Append(0x3E00).ok()); - EXPECT_TRUE(builder.Append(0x7E00).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_binary_array(const std::vector& values) { - arrow::BinaryBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data()), - static_cast(value.size())) - .ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_fixed_binary_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_timestamp_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_decimal_array(const std::shared_ptr& type, - const std::vector& values) { - arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); - } - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, - std::shared_ptr array) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - } - - void write_table(const std::string& file_path, const std::shared_ptr& table, - std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = nullptr) { - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - ::parquet::WriterProperties::Builder writer_builder; - writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - writer_builder.compression(::parquet::Compression::UNCOMPRESSED); - if (arrow_properties == nullptr) { - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_properties = arrow_builder.build(); - } - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), *file_result, ROW_COUNT, - writer_builder.build(), std::move(arrow_properties))); - } - - void write_parquet_file() { - add_field(arrow::field("bool_col", arrow::boolean(), false), - build_required_array( - {true, false, true, false, true})); - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50})); - add_field(arrow::field("int64_col", arrow::int64(), false), - build_required_array( - {10000000000L, -9L, 42L, 77L, 123L})); - add_field(arrow::field("uint32_col", arrow::uint32(), false), - build_required_array( - {0U, 1U, 1U << 31, std::numeric_limits::max(), 42U})); - add_field(arrow::field("uint64_col", arrow::uint64(), false), - build_required_array( - {0ULL, 1ULL, 1ULL << 63, std::numeric_limits::max(), 42ULL})); - add_field(arrow::field("float_col", arrow::float32(), false), - build_required_array( - {1.5F, -2.25F, 3.0F, 4.5F, 5.75F})); - add_field(arrow::field("double_col", arrow::float64(), false), - build_required_array({3.5, -4.75, 6.0, 7.25, 8.5})); - add_field(arrow::field("nullable_float16_col", arrow::float16(), true), - build_nullable_float16_array()); - add_field(arrow::field("binary_col", arrow::binary(), false), - build_binary_array({"bin_a", "bin_b", "bin_c", "bin_d", "bin_e"})); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"})); - add_field(arrow::field("fixed_binary_col", arrow::fixed_size_binary(4), false), - build_fixed_binary_array(arrow::fixed_size_binary(4), - {"aaaa", "bbbb", "cccc", "dddd", "eeee"})); - add_field(arrow::field("date_col", arrow::date32(), false), - build_required_array({0, 1, 18628, 18629, 18630})); - add_field(arrow::field("timestamp_millis_col", arrow::timestamp(arrow::TimeUnit::MILLI), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MILLI), - {0, 1234, 1609459200000, 1609459201000, -1})); - add_field(arrow::field("timestamp_micros_col", arrow::timestamp(arrow::TimeUnit::MICRO), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("timestamp_micros_utc_col", - arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("decimal_fixed_binary_9_2_col", arrow::decimal128(9, 2), false), - build_decimal_array(arrow::decimal128(9, 2), {12345, -67, 0, 987, 1000})); - add_field(arrow::field("decimal_fixed_binary_18_6_col", arrow::decimal128(18, 6), false), - build_decimal_array(arrow::decimal128(18, 6), - {1234567, -670000, 0, 9870000, 1000000})); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array()); - - write_table(_file_path, arrow::Table::Make(arrow::schema(_arrow_fields), _arrays)); - } - - void open_file(const std::string& file_path) { - _file_reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); - ASSERT_NE(_file_reader, nullptr); - ASSERT_EQ(_file_reader->metadata()->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet serde test field " << name; - return _fields.size(); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - template - void read_and_validate(const std::string& name, Validator validator) const { - const auto field_idx = find_field_idx(name); - ASSERT_TRUE(supports_record_reader(_fields[field_idx]->type_descriptor)); - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - validator(*_fields[field_idx], *column); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; -}; - -TEST_F(ParquetSerdeReaderTest, ReadAllSupportedPhysicalAndLogicalTypes) { - read_and_validate("bool_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BOOLEAN); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 0); - EXPECT_EQ(values.get_element(4), 1); - }); - read_and_validate("int32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - read_and_validate("int64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10000000000L); - EXPECT_EQ(values.get_element(1), -9L); - }); - read_and_validate("uint32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_BIGINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), 2147483648L); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("uint64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_LARGEINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), static_cast(1) << 63); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("float_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FLOAT); - const auto& values = assert_cast(column); - EXPECT_FLOAT_EQ(values.get_element(0), 1.5F); - EXPECT_FLOAT_EQ(values.get_element(1), -2.25F); - }); - read_and_validate("double_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::DOUBLE); - const auto& values = assert_cast(column); - EXPECT_DOUBLE_EQ(values.get_element(0), 3.5); - EXPECT_DOUBLE_EQ(values.get_element(1), -4.75); - }); - read_and_validate("nullable_float16_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 2); - EXPECT_EQ(schema.type_descriptor.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(schema.type_descriptor.is_string_like); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_FLOAT); - const auto& nullable_column = assert_cast(column); - const auto& values = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FLOAT_EQ(values.get_element(1), 0.0F); - EXPECT_FALSE(std::signbit(values.get_element(1))); - EXPECT_FLOAT_EQ(values.get_element(2), -0.0F); - EXPECT_TRUE(std::signbit(values.get_element(2))); - EXPECT_FLOAT_EQ(values.get_element(3), 1.5F); - EXPECT_TRUE(std::isnan(values.get_element(4))); - }); - read_and_validate("binary_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BYTE_ARRAY); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "bin_a"); - EXPECT_EQ(values.get_data_at(3).to_string(), "bin_d"); - }); - read_and_validate("string_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - read_and_validate("fixed_binary_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 4); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "aaaa"); - EXPECT_EQ(values.get_data_at(2).to_string(), "cccc"); - }); - read_and_validate("date_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATEV2); - EXPECT_EQ(schema.type->to_string(column, 0), "1970-01-01"); - EXPECT_EQ(schema.type->to_string(column, 2), "2021-01-01"); - }); - read_and_validate( - "timestamp_millis_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999"); - }); - read_and_validate( - "timestamp_micros_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("timestamp_micros_utc_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.timestamp_is_adjusted_to_utc); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("decimal_fixed_binary_9_2_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal32(12345)); - EXPECT_EQ(schema.type->to_string(column, 0), "123.45"); - }); - read_and_validate("decimal_fixed_binary_18_6_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal64(1234567)); - EXPECT_EQ(schema.type->to_string(column, 0), "1.234567"); - }); - read_and_validate( - "nullable_int_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); -} - -TEST_F(ParquetSerdeReaderTest, ReadInt96TimestampAsDateTimeV2) { - const auto file_path = (_test_dir / "int96_timestamp.parquet").string(); - auto field = arrow::field("col_datetime", arrow::timestamp(arrow::TimeUnit::MICRO), false); - auto array = build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1}); - auto table = arrow::Table::Make(arrow::schema({field}), {array}); - - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_builder.enable_force_write_int96_timestamps(); - _fields.clear(); - _file_reader.reset(); - _row_group.reset(); - write_table(file_path, table, arrow_builder.build()); - open_file(file_path); - - ASSERT_EQ(_fields.size(), 1); - EXPECT_EQ(_fields[0]->type_descriptor.physical_type, ::parquet::Type::INT96); - EXPECT_EQ(_fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - ASSERT_TRUE(supports_record_reader(_fields[0]->type_descriptor)); - ASSERT_EQ(remove_nullable(_fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - - auto reader = create_reader(0); - ASSERT_NE(reader, nullptr); - auto column = _fields[0]->type->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(ROW_COUNT, column, &rows_read).ok()); - ASSERT_EQ(rows_read, ROW_COUNT); - EXPECT_EQ(_fields[0]->type->to_string(*column, 0), "1970-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 2), "2021-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 4), "1969-12-31 23:59:59.999999"); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index dd2138279b11d1..e69f1311e4ab2b 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -17,29 +17,26 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include #include -#include -#include -#include -#include -#include #include #include #include +#include #include #include #include #include +#include #include #include +#include "core/data_type/data_type_date.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_time.h" #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr.h" @@ -47,293 +44,157 @@ #include "exprs/vslot_ref.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "storage/index/bloom_filter/block_split_bloom_filter.h" -#include "storage/index/zone_map/zonemap_eval_context.h" -#include "storage/index/zone_map/zonemap_filter_result.h" - +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "io/fs/file_reader.h" +#include "util/thrift_util.h" namespace doris { namespace { -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -std::shared_ptr int32_array(const std::vector>& values) { - arrow::Int32Builder builder; - for (const auto& value : values) { - if (value.has_value()) { - EXPECT_TRUE(builder.Append(*value).ok()); - } else { - EXPECT_TRUE(builder.AppendNull().ok()); - } - } - return finish_array(&builder); -} - -std::shared_ptr uint32_array(const std::vector& values) { - arrow::UInt32Builder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr float_array(const std::vector& values) { - arrow::FloatBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr double_array(const std::vector& values) { - arrow::DoubleBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -template -std::string encoded_value(const NativeType& value) { - return {reinterpret_cast(&value), sizeof(value)}; -} - -std::shared_ptr string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr timestamp_array(const std::vector& values) { - arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::unique_ptr<::parquet::ParquetFileReader> make_reader( - const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary, - bool enable_statistics) { - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.compression(::parquet::Compression::UNCOMPRESSED); - if (enable_dictionary) { - builder.enable_dictionary(); - } else { - builder.disable_dictionary(); - } - if (!enable_statistics) { - builder.disable_statistics(); - } - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - row_group_size, builder.build()) - .ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - return ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); -} - -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*reader.metadata()->schema(), &file_schema) - .ok()); - return file_schema; -} - -template -class TestColumnIndex final : public ::parquet::TypedColumnIndex { +class StatisticsMemoryFileReader final : public io::FileReader { public: - using NativeType = typename ParquetDType::c_type; - - TestColumnIndex(NativeType min_value, NativeType max_value) - : TestColumnIndex(std::vector {min_value}, - std::vector {max_value}) {} - - TestColumnIndex(std::vector min_values, std::vector max_values) - : _null_pages(min_values.size(), false), - _null_counts(min_values.size(), 0), - _min_values(std::move(min_values)), - _max_values(std::move(max_values)) { - EXPECT_EQ(_min_values.size(), _max_values.size()); - for (size_t page_idx = 0; page_idx < _min_values.size(); ++page_idx) { - _non_null_page_indices.push_back(static_cast(page_idx)); - } - } + explicit StatisticsMemoryFileReader(std::vector bytes) + : _bytes(std::move(bytes)), _path("native-bloom-filter.parquet") {} - const std::vector& null_pages() const override { return _null_pages; } - const std::vector& encoded_min_values() const override { return _encoded_values; } - const std::vector& encoded_max_values() const override { return _encoded_values; } - ::parquet::BoundaryOrder::type boundary_order() const override { - return ::parquet::BoundaryOrder::Unordered; + Status close() override { + _closed = true; + return Status::OK(); } - bool has_null_counts() const override { return true; } - const std::vector& null_counts() const override { return _null_counts; } - const std::vector& non_null_page_indices() const override { - return _non_null_page_indices; + const io::Path& path() const override { return _path; } + size_t size() const override { return _bytes.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _bytes.size() || result.size > _bytes.size() - offset) { + return Status::IOError("native Bloom test read exceeds memory file"); + } + memcpy(result.data, _bytes.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); } - const std::vector& min_values() const override { return _min_values; } - const std::vector& max_values() const override { return _max_values; } private: - const std::vector _null_pages; - const std::vector _encoded_values; - const std::vector _null_counts; - std::vector _non_null_page_indices; - const std::vector _min_values; - const std::vector _max_values; + std::vector _bytes; + io::Path _path; + bool _closed = false; }; - -class Int32ZoneMapExpr final : public VExpr { +class BloomInExpr final : public VExpr { public: - enum class Op { GE, GT, IS_NULL, IS_NOT_NULL }; - - Int32ZoneMapExpr(int column_id, Op op, int32_t value = 0) + BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) : VExpr(std::make_shared(), false), - _column_id(column_id), - _op(op), - _value(value) {} + _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), + _values(std::move(values)) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("Int32ZoneMapExpr is only used by parquet statistics tests"); + return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); } - bool can_evaluate_zonemap_filter() const override { return true; } + bool can_evaluate_bloom_filter() const override { return true; } - void collect_slot_column_ids(std::set& column_ids) const override { - column_ids.insert(_column_id); + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { + return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); } - ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { - auto zone_map = ctx.zone_map(_column_id); - if (zone_map == nullptr) { - return unsupported_zonemap_filter(ctx); - } - if (_op == Op::IS_NULL) { - return zone_map->has_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (_op == Op::IS_NOT_NULL) { - return zone_map->has_not_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (!zone_map->has_not_null) { - return ZoneMapFilterResult::kNoMatch; - } - const auto literal = Field::create_field(_value); - if (_op == Op::GE) { - return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; - } - return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; + void collect_slot_column_ids(std::set& column_ids) const override { + _slot->collect_slot_column_ids(column_ids); } private: - int _column_id; - Op _op; - int32_t _value; - const std::string _expr_name = "Int32ZoneMapExpr"; + VExprSPtr _slot; + std::vector _values; + const std::string _expr_name = "BloomInExpr"; }; -class StringDictionaryInExpr final : public VExpr { +class DictionaryStringInExpr final : public VExpr { public: - StringDictionaryInExpr(int column_id, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::make_shared(), - "c0")) { - _values.reserve(values.size()); - for (auto& value : values) { - _values.emplace_back(Field::create_field(std::move(value))); - } - } + DictionaryStringInExpr() : VExpr(std::make_shared(), false) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError( - "StringDictionaryInExpr is only used by parquet statistics tests"); + return Status::InternalError("DictionaryStringInExpr is metadata-only"); } bool can_evaluate_dictionary_filter() const override { return true; } - ZoneMapFilterResult evaluate_dictionary_filter( - const DictionaryEvalContext& ctx) const override { - return expr_zonemap::eval_in_dictionary(ctx, _slot, false, _values); + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext&) const override { + return ZoneMapFilterResult::kNoMatch; } - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); - } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "StringDictionaryInExpr"; + const std::string _expr_name = "DictionaryStringInExpr"; }; -class BloomInExpr final : public VExpr { +class MetadataInt32GreaterThanExpr final : public VExpr { public: - BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), - _values(std::move(values)) {} + explicit MetadataInt32GreaterThanExpr(int32_t value) + : VExpr(std::make_shared(), false), _value(value) {} const std::string& expr_name() const override { return _expr_name; } - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); - } - - bool can_evaluate_bloom_filter() const override { return true; } - - ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { - return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); + return Status::InternalError("MetadataInt32GreaterThanExpr is metadata-only"); } - - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + return zone_map->max_value <= Field::create_field(_value) + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "BloomInExpr"; + int32_t _value; + const std::string _expr_name = "MetadataInt32GreaterThanExpr"; }; -format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); - return request; -} +class MetadataBoundsProbeExpr final : public VExpr { +public: + explicit MetadataBoundsProbeExpr(bool require_false_boolean = false) + : VExpr(std::make_shared(), false), + _require_false_boolean(require_false_boolean) {} -format::FileScanRequest request_with_dictionary_conjunct(std::vector values) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared( - std::make_shared(0, std::move(values)))); - return request; -} + const std::string& expr_name() const override { return _expr_name; } + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("MetadataBoundsProbeExpr is metadata-only"); + } + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr || !zone_map->has_not_null || zone_map->min_value.is_null() || + zone_map->max_value.is_null()) { + return ZoneMapFilterResult::kMayMatch; + } + if (_require_false_boolean && + zone_map->min_value == Field::create_field(false) && + zone_map->max_value == Field::create_field(false)) { + return ZoneMapFilterResult::kMayMatch; + } + return ZoneMapFilterResult::kNoMatch; + } +private: + bool _require_false_boolean; + const std::string _expr_name = "MetadataBoundsProbeExpr"; +}; VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -346,690 +207,514 @@ format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type, request.conjuncts = bloom_conjuncts(std::move(data_type), std::move(values)); return request; } - -void add_bloom_field(segment_v2::BlockSplitBloomFilter* bloom_filter, const Field& value, - PrimitiveType type) { - DORIS_CHECK(bloom_filter != nullptr); - switch (type) { - case TYPE_BOOLEAN: { - const bool typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_INT: { - const int32_t typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_STRING: { - const auto& typed_value = value.get(); - bloom_filter->add_bytes(typed_value.data(), typed_value.size()); - break; - } - default: - DORIS_CHECK(false); - } -} - -std::unique_ptr bloom_filter_for_fields( - const std::vector& values, PrimitiveType type) { - auto bloom_filter = std::make_unique(); - EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); - for (const auto& value : values) { - add_bloom_field(bloom_filter.get(), value, type); - } - return bloom_filter; -} - -BloomFilterEvalContext bloom_context(const DataTypePtr& data_type, - const segment_v2::BloomFilter* bloom_filter) { - BloomFilterEvalContext ctx; - ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter {.data_type = data_type, - .bloom_filter = bloom_filter}); - return ctx; -} - -std::unique_ptr<::parquet::BlockSplitBloomFilter> parquet_bloom_filter() { - auto bloom_filter = std::make_unique<::parquet::BlockSplitBloomFilter>(); - bloom_filter->Init(::parquet::BlockSplitBloomFilter::kMinimumBloomFilterBytes); - return bloom_filter; -} - format::parquet::ParquetColumnSchema uint32_parquet_bloom_schema() { format::parquet::ParquetColumnSchema column_schema; column_schema.type = std::make_shared(); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::INT32; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; column_schema.type_descriptor.integer_bit_width = 32; column_schema.type_descriptor.is_unsigned_integer = true; return column_schema; } -format::parquet::ParquetColumnSchema fixed_len_string_parquet_bloom_schema(int fixed_length) { +TEST(NativeParquetStatisticsTest, InvalidNullableDateBoundsDisableMinMax) { format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); + column_schema.type = make_nullable(std::make_shared()); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = fixed_length; - column_schema.type_descriptor.is_string_like = true; - return column_schema; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + + const int32_t invalid_date = std::numeric_limits::min(); + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_date), sizeof(invalid_date))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_date), sizeof(invalid_date))); + + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -format::parquet::ParquetColumnSchema float16_parquet_bloom_schema() { +TEST(NativeParquetStatisticsTest, InvalidNullableDecimalBoundsDisableMinMax) { format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); + column_schema.type = make_nullable(std::make_shared(2, 0)); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = 2; - column_schema.type_descriptor.extra_type_info = format::parquet::ParquetExtraTypeInfo::FLOAT16; - return column_schema; -} - -TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTimestamp) { - auto table = arrow::Table::Make( - arrow::schema({ - arrow::field("i", arrow::int32(), true), - arrow::field("u", arrow::uint32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - }), - {int32_array({1, std::nullopt, 5}), uint32_array({7, 9, 11}), - string_array({"alpha", "beta", "omega"}), timestamp_array({1000, 2000, 3000})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto row_group = reader->metadata()->RowGroup(0); - - const auto int_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], row_group->ColumnChunk(0)->statistics()); - EXPECT_TRUE(int_stats.has_min_max); - EXPECT_TRUE(int_stats.has_null_count); - EXPECT_TRUE(int_stats.has_null); - EXPECT_TRUE(int_stats.has_not_null); - EXPECT_EQ(int_stats.min_value.get(), 1); - EXPECT_EQ(int_stats.max_value.get(), 5); - - const auto uint_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[1], row_group->ColumnChunk(1)->statistics()); - EXPECT_TRUE(uint_stats.has_min_max); - EXPECT_EQ(uint_stats.min_value.get(), 7); - EXPECT_EQ(uint_stats.max_value.get(), 11); - - const auto string_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], row_group->ColumnChunk(2)->statistics()); - EXPECT_TRUE(string_stats.has_min_max); - EXPECT_EQ(string_stats.min_value.get(), "alpha"); - EXPECT_EQ(string_stats.max_value.get(), "omega"); - - auto utc = cctz::utc_time_zone(); - const auto timestamp_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[3], row_group->ColumnChunk(3)->statistics(), &utc); - EXPECT_TRUE(timestamp_stats.has_min_max); - EXPECT_EQ(timestamp_stats.min_value.get_type(), TYPE_DATETIMEV2); - EXPECT_EQ(timestamp_stats.max_value.get_type(), TYPE_DATETIMEV2); - EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); -} + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + column_schema.type_descriptor.is_decimal = true; + column_schema.type_descriptor.decimal_precision = 2; + column_schema.type_descriptor.decimal_scale = 0; -TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. - // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover - // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, - 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(local_stats.has_not_null); - EXPECT_FALSE(local_stats.has_min_max); - - auto utc = cctz::utc_time_zone(); - const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &utc); - EXPECT_TRUE(utc_stats.has_min_max); - EXPECT_LT(utc_stats.min_value, utc_stats.max_value); -} + const int32_t invalid_decimal = 1000; + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_decimal), sizeof(invalid_decimal))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_decimal), sizeof(invalid_decimal))); -TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp - // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. - schema[0]->type = std::make_shared(6); - schema[0]->type_descriptor.doris_type = schema[0]->type; - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto timestamp_tz_stats = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(timestamp_tz_stats.has_min_max); - EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { - auto no_stats_table = arrow::Table::Make( - arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); - auto no_stats_reader = make_reader(no_stats_table, 3, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - auto no_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *no_stats_schema[0], - no_stats_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_FALSE(no_stats.has_min_max); - - auto all_null_table = - arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt})}); - auto all_null_reader = make_reader(all_null_table, 2, false, true); - auto all_null_schema = build_file_schema(*all_null_reader); - auto all_null_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *all_null_schema[0], - all_null_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_TRUE(all_null_stats.has_null_count); - EXPECT_TRUE(all_null_stats.has_null); - EXPECT_FALSE(all_null_stats.has_not_null); - EXPECT_FALSE(all_null_stats.has_min_max); -} +TEST(NativeParquetStatisticsTest, InvalidTimeBoundsDisableFooterMinMax) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(3); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + column_schema.type_descriptor.time_unit = format::parquet::ParquetTimeUnit::MILLIS; -TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({1, std::nullopt, 3})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( - reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), - file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); - - const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics_without_null_count); - EXPECT_FALSE(statistics.has_null_count); - EXPECT_TRUE(statistics.has_null); - EXPECT_TRUE(statistics.has_not_null); - EXPECT_TRUE(statistics.has_min_max); - EXPECT_EQ(statistics.min_value.get(), 1); - EXPECT_EQ(statistics.max_value.get(), 3); -} + const int32_t invalid_time = 90000000; + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_time), sizeof(invalid_time))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_time), sizeof(invalid_time))); -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const float float_nan = std::numeric_limits::quiet_NaN(); - const float float_max = 2.0F; - auto float_stats = ::parquet::MakeStatistics<::parquet::FloatType>( - schema[0]->descriptor, encoded_value(float_nan), encoded_value(float_max), 2, 0, 0, - true, true, false); - const auto converted_float = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], float_stats); - EXPECT_FALSE(converted_float.has_min_max); - EXPECT_TRUE(converted_float.has_not_null); - - const double double_nan = std::numeric_limits::quiet_NaN(); - const double double_min = 1.0; - auto double_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_nan), 2, 0, 0, - true, true, false); - const auto converted_double = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - double_stats); - EXPECT_FALSE(converted_double.has_min_max); - EXPECT_TRUE(converted_double.has_not_null); - - const double double_max = 2.0; - auto finite_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_max), 2, 0, 0, - true, true, false); - const auto converted_finite = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - finite_stats); - EXPECT_TRUE(converted_finite.has_min_max); + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleColumnIndexMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - auto float_index = std::make_shared>( - 1.0F, std::numeric_limits::quiet_NaN()); - format::parquet::ParquetColumnStatistics float_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - float_index, *schema[0], 0, &float_page_stats)); - EXPECT_FALSE(float_page_stats.has_min_max); - EXPECT_TRUE(float_page_stats.has_not_null); - - auto double_index = std::make_shared>( - std::numeric_limits::quiet_NaN(), 2.0); - format::parquet::ParquetColumnStatistics double_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - double_index, *schema[1], 0, &double_page_stats)); - EXPECT_FALSE(double_page_stats.has_min_max); - EXPECT_TRUE(double_page_stats.has_not_null); - - auto finite_index = std::make_shared>(1.0, 2.0); - format::parquet::ParquetColumnStatistics finite_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - finite_index, *schema[1], 0, &finite_page_stats)); - EXPECT_TRUE(finite_page_stats.has_min_max); - - auto mixed_index = std::make_shared>( - std::vector {std::numeric_limits::quiet_NaN(), 1.0}, - std::vector {std::numeric_limits::quiet_NaN(), 2.0}); - format::parquet::ParquetColumnStatistics nan_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 0, &nan_page_stats)); - EXPECT_FALSE(nan_page_stats.has_min_max); - - format::parquet::ParquetColumnStatistics following_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 1, &following_page_stats)); - EXPECT_TRUE(following_page_stats.has_min_max); - EXPECT_EQ(following_page_stats.min_value.get(), 1.0); - EXPECT_EQ(following_page_stats.max_value.get(), 2.0); -} +TEST(NativeParquetStatisticsTest, BooleanFooterBoundsDecodeOnlyTheValueBit) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = tparquet::Type::BOOLEAN; + + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value(std::string(1, '\x02')); + statistics.__set_max_value(std::string(1, '\x02')); + + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + ASSERT_TRUE(result.has_min_max); + EXPECT_EQ(result.min_value, Field::create_field(false)); + EXPECT_EQ(result.max_value, Field::create_field(false)); +} + +TEST(NativeParquetStatisticsTest, InvalidTimeAndPaddedBooleanPageBoundsCannotPrune) { + auto run_page_index = [](std::unique_ptr column_schema, + std::string min_value, std::string max_value, + bool require_false_boolean) { + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + tparquet::FileMetaData metadata; + metadata.__set_column_orders({order}); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared( + std::make_shared(require_false_boolean))}; + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({std::move(min_value)}); + page_index.column_index.__set_max_values({std::move(max_value)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, std::move(page_index)); + std::vector selected_ranges; + std::map skip_plans; + EXPECT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, + &skip_plans, nullptr) + .ok()); + return selected_ranges; + }; + + auto time_schema = std::make_unique(); + time_schema->type = std::make_shared(3); + time_schema->type_descriptor.doris_type = time_schema->type; + time_schema->type_descriptor.physical_type = tparquet::Type::INT32; + time_schema->type_descriptor.time_unit = format::parquet::ParquetTimeUnit::MILLIS; + const int32_t invalid_time = 90000000; + const std::string invalid_time_bytes(reinterpret_cast(&invalid_time), + sizeof(invalid_time)); + const auto time_ranges = + run_page_index(std::move(time_schema), invalid_time_bytes, invalid_time_bytes, false); + ASSERT_EQ(time_ranges.size(), 1); + EXPECT_EQ(time_ranges[0].start, 0); + EXPECT_EQ(time_ranges[0].length, 1); + + auto bool_schema = std::make_unique(); + bool_schema->type = std::make_shared(); + bool_schema->type_descriptor.doris_type = bool_schema->type; + bool_schema->type_descriptor.physical_type = tparquet::Type::BOOLEAN; + const auto bool_ranges = run_page_index(std::move(bool_schema), std::string(1, '\x02'), + std::string(1, '\x02'), true); + ASSERT_EQ(bool_ranges.size(), 1); + EXPECT_EQ(bool_ranges[0].start, 0); + EXPECT_EQ(bool_ranges[0].length, 1); +} +TEST(ParquetBloomFilterPruningTest, NativeUint32BloomUsesPhysicalInt32Hash) { + const auto column_schema = uint32_parquet_bloom_schema(); + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) + .ok()); -TEST(ParquetStatisticsTransformTest, IgnoresInvertedFooterAndColumnIndexMinMax) { - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("i", arrow::int32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {int32_array({1, 2}), string_array({"a", "z"}), timestamp_array({1000000, 2000000})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const int32_t inverted_min = 10; - const int32_t inverted_max = 1; - auto int_stats = ::parquet::MakeStatistics<::parquet::Int32Type>( - schema[0]->descriptor, encoded_value(inverted_min), encoded_value(inverted_max), 2, 0, - 0, true, true, false); - const auto converted_int = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], int_stats); - EXPECT_TRUE(converted_int.has_not_null); - EXPECT_FALSE(converted_int.has_min_max); - - const std::string inverted_string_min = "z"; - const std::string inverted_string_max = "a"; - auto string_stats = ::parquet::MakeStatistics<::parquet::ByteArrayType>( - schema[1]->descriptor, inverted_string_min, inverted_string_max, 2, 0, 0, true, true, - false); - const auto converted_string = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - string_stats); - EXPECT_TRUE(converted_string.has_not_null); - EXPECT_FALSE(converted_string.has_min_max); - - auto int_index = - std::make_shared>(inverted_min, inverted_max); - format::parquet::ParquetColumnStatistics page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - int_index, *schema[0], 0, &page_stats)); - EXPECT_TRUE(page_stats.has_not_null); - EXPECT_FALSE(page_stats.has_min_max); - - // These endpoints are inverted within one second. Whole-second validation alone would miss - // the corruption, and TIMESTAMPTZ must reject the same raw inversion before its UTC shortcut. - constexpr int64_t timestamp_min = 1500000; - constexpr int64_t timestamp_max = 1000000; - auto timestamp_stats = ::parquet::MakeStatistics<::parquet::Int64Type>( - schema[2]->descriptor, encoded_value(timestamp_min), encoded_value(timestamp_max), 2, 0, - 0, true, true, false); - auto utc = cctz::utc_time_zone(); - const auto converted_timestamp = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp.has_min_max); - - schema[2]->type = std::make_shared(6); - schema[2]->type_descriptor.doris_type = schema[2]->type; - const auto converted_timestamp_tz = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp_tz.has_min_max); -} + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); -TEST(ParquetStatisticsTransformTest, PreservesNullCountWhenNaNInvalidatesMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float64(), false)}), - {double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const double nan = std::numeric_limits::quiet_NaN(); - const double max_value = 2.0; - auto footer_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[0]->descriptor, encoded_value(nan), encoded_value(max_value), 2, 0, 0, true, - true, false); - const auto converted_footer = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[0], - footer_stats); - auto footer_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(converted_footer); - ASSERT_NE(footer_zone_map, nullptr); - EXPECT_TRUE(footer_zone_map->pass_all); - EXPECT_FALSE(footer_zone_map->has_null); - EXPECT_TRUE(footer_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*footer_zone_map, schema[0]->type)); - - ZoneMapEvalContext footer_ctx; - footer_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = footer_zone_map}); - Int32ZoneMapExpr is_null_expr(0, Int32ZoneMapExpr::Op::IS_NULL); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(footer_ctx), ZoneMapFilterResult::kNoMatch); - - auto column_index = std::make_shared>(nan, max_value); - format::parquet::ParquetColumnStatistics page_stats; - ASSERT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *schema[0], 0, &page_stats)); - auto page_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(page_stats); - ASSERT_NE(page_zone_map, nullptr); - EXPECT_TRUE(page_zone_map->pass_all); - EXPECT_FALSE(page_zone_map->has_null); - EXPECT_TRUE(page_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*page_zone_map, schema[0]->type)); - - ZoneMapEvalContext page_ctx; - page_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = page_zone_map}); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(page_ctx), ZoneMapFilterResult::kNoMatch); + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field( + static_cast(present_value))}), + bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), + bloom_filter)); } -TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); +TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Max) { + auto column_schema = + std::make_unique(uint32_parquet_bloom_schema()); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct( - std::make_shared(0, Int32ZoneMapExpr::Op::GE, 5)), - nullptr, &selected, false, &pruning_stats) + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) .ok()); - EXPECT_EQ(selected, std::vector({2})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); - - selected.clear(); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NOT_NULL)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1, 2})); - - selected.clear(); + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); + + tparquet::BloomFilterAlgorithm algorithm; + algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm()); + tparquet::BloomFilterHash hash; + hash.__set_XXHASH(tparquet::XxHash()); + tparquet::BloomFilterCompression compression; + compression.__set_UNCOMPRESSED(tparquet::Uncompressed()); + tparquet::BloomFilterHeader bloom_header; + bloom_header.__set_numBytes(static_cast(bloom_filter.size())); + bloom_header.__set_algorithm(algorithm); + bloom_header.__set_hash(hash); + bloom_header.__set_compression(compression); + std::vector bloom_bytes; + ThriftSerializer serializer(/*compact=*/true, 64); + ASSERT_TRUE(serializer.serialize(&bloom_header, &bloom_bytes).ok()); + bloom_bytes.insert(bloom_bytes.end(), bloom_filter.data(), + bloom_filter.data() + bloom_filter.size()); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(1); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_bloom_filter_offset(0); + column_metadata.__set_bloom_filter_length(static_cast(bloom_bytes.size())); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_version(1); + metadata.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + + format::parquet::ParquetFileContext file_context; + file_context.native_file = std::make_shared(std::move(bloom_bytes)); + auto request = request_with_bloom_conjunct( + column_schema->type, + {Field::create_field(static_cast(present_value))}); + std::vector> schema; + schema.push_back(std::move(column_schema)); + std::vector selected_row_groups; + format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NULL)), - nullptr, &selected, false, &pruning_stats) + metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0})); + EXPECT_EQ(selected_row_groups, std::vector({0})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); } -TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsupportedPaths) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, false); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, EmptyDictionaryRowGroupIsSkippedBeforeMetadataProbes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::BYTE_ARRAY); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(0); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_dictionary_page_offset(0); + column_metadata.__set_encodings({tparquet::Encoding::RLE_DICTIONARY}); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(0); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(0); + thrift_metadata.__set_row_groups({row_group}); + + format::parquet::NativeParquetMetadata native_metadata(thrift_metadata, 0); + ASSERT_TRUE(native_metadata.init_schema(false, false).ok()); + format::parquet::ParquetFileContext file_context; + file_context.native_file = + std::make_shared(std::vector {}); + file_context.native_metadata = &native_metadata; + + auto column_schema = std::make_unique(); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::BYTE_ARRAY; + column_schema->type_descriptor.is_string_like = true; + std::vector> schema; + schema.push_back(std::move(column_schema)); - std::vector selected; + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared(std::make_shared())}; + std::vector selected_row_groups; format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"gamma"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 1); - - auto plain_reader = make_reader(table, 2, false, false); - auto plain_schema = build_file_schema(*plain_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *plain_reader->metadata(), plain_reader.get(), plain_schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) + thrift_metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); + EXPECT_TRUE(selected_row_groups.empty()); } -TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, true); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, InvalidCandidateRowGroupReturnsCorruption) { + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); + format::FileScanRequest request; + const std::vector> schema; + const std::vector candidates {1}; + std::vector selected_row_groups; + + const auto status = format::parquet::select_row_groups_by_metadata( + metadata, schema, request, &candidates, &selected_row_groups, false, nullptr, nullptr, + nullptr, nullptr); + EXPECT_TRUE(status.is()) << status; +} +TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { + format::parquet::ParquetTypeDescriptor binary_type; + binary_type.physical_type = tparquet::Type::BYTE_ARRAY; + + tparquet::Statistics max_only; + max_only.__set_max("III"); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only, false)); + + tparquet::Statistics legacy_different; + legacy_different.__set_min("III"); + legacy_different.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, + legacy_different, false)); + + tparquet::Statistics legacy_equal; + legacy_equal.__set_min("same"); + legacy_equal.__set_max("same"); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal, + false)); + + tparquet::Statistics type_defined; + type_defined.__set_min_value("III"); + type_defined.__set_max_value("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + false)); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + true)); + + tparquet::Statistics mixed_fields; + mixed_fields.__set_min_value("III"); + mixed_fields.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields, + true)); +} + +TEST(NativeParquetStatisticsTest, ExplicitlyInexactNumericBoundsCannotBackMinMaxAggregate) { + format::parquet::ParquetTypeDescriptor int32_type; + int32_type.physical_type = tparquet::Type::INT32; + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(0)); + statistics.__set_max_value(encode_int32(100)); + EXPECT_TRUE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); + + statistics.__set_is_min_value_exact(false); + statistics.__set_is_max_value_exact(true); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); + + statistics.__set_is_min_value_exact(true); + statistics.__set_is_max_value_exact(false); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); +} + +TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder) { + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(1)); + statistics.__set_max_value(encode_int32(2)); + statistics.__set_null_count(0); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(1); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(100))}; + std::vector selected_row_groups; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - auto no_stats_reader = make_reader(table, 2, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *no_stats_reader->metadata(), no_stats_reader.get(), no_stats_schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + EXPECT_EQ(selected_row_groups, std::vector({0})); + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({encode_int32(1)}); + page_index.column_index.__set_max_values({encode_int32(2)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, page_index); + std::vector selected_ranges; + std::map skip_plans; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); -} - -TEST(ParquetStatisticsPruningTest, BloomFilterCacheIsScopedToRowGroupAndColumn) { - auto input = arrow::io::ReadableFile::Open( - "./be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet"); - ASSERT_TRUE(input.ok()); - auto reader = ::parquet::ParquetFileReader::Open(*input); - ASSERT_EQ(reader->metadata()->num_row_groups(), 2); - auto& bloom_filter_reader = reader->GetBloomFilterReader(); - for (int row_group_idx = 0; row_group_idx < 2; ++row_group_idx) { - auto row_group_reader = bloom_filter_reader.RowGroup(row_group_idx); - ASSERT_NE(row_group_reader, nullptr); - ASSERT_NE(row_group_reader->GetColumnBloomFilter(0), nullptr); - } - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - auto request = request_with_bloom_conjunct(std::make_shared(), - {Field::create_field(12345)}); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - false, &pruning_stats) + EXPECT_EQ(selected_ranges.size(), 1); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + metadata.__set_column_orders({order}); + selected_row_groups.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - true, &pruning_stats) + EXPECT_TRUE(selected_row_groups.empty()); + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 1); -} - -TEST(ParquetBloomFilterPruningTest, VExprEqPrunesAbsentIntValue) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2)}), ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(3)}), ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprInPrunesOnlyWhenAllValuesAreAbsent) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(4)}), - ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(3)}), - ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprBoolAndStringUseSlotBloomFilter) { - auto bool_type = std::make_shared(); - auto bool_filter = - bloom_filter_for_fields({Field::create_field(true)}, TYPE_BOOLEAN); - auto bool_ctx = bloom_context(bool_type, bool_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(false)}), - bool_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(true)}), - bool_ctx), - ZoneMapFilterResult::kMayMatch); - - auto string_type = std::make_shared(); - auto string_filter = bloom_filter_for_fields( - {Field::create_field("alpha"), Field::create_field("omega")}, - TYPE_STRING); - auto string_ctx = bloom_context(string_type, string_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("beta")}), - string_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("alpha")}), - string_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, MissingOrUnsupportedBloomContextKeepsRowGroup) { - auto int_type = std::make_shared(); - BloomFilterEvalContext missing_ctx; - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(int_type, {Field::create_field(2)}), missing_ctx), - ZoneMapFilterResult::kMayMatch); - - auto smallint_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields({Field::create_field(1)}, TYPE_INT); - auto unsupported_ctx = bloom_context(smallint_type, bloom_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(smallint_type, {Field::create_field(2)}), - unsupported_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { - const auto column_schema = uint32_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - const uint32_t present_value = 4000000000U; - int32_t physical_value; - memcpy(&physical_value, &present_value, sizeof(physical_value)); - bloom_filter->InsertHash(bloom_filter->Hash(physical_value)); - - // UINT32 is exposed to VExpr as Doris BIGINT, but Parquet stores and hashes it as a physical - // INT32 carrier. A present value above INT32_MAX must therefore be narrowed to the physical - // bit pattern before probing the file bloom filter. - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field( - static_cast(present_value))}), - *bloom_filter)); - - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts( - column_schema.type, - {Field::create_field( - static_cast(std::numeric_limits::max()) + 1)}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { - const auto column_schema = fixed_len_string_parquet_bloom_schema(4); - auto bloom_filter = parquet_bloom_filter(); - - const std::string present_value = "abcd"; - ::parquet::FLBA physical_value(reinterpret_cast(present_value.data())); - bloom_filter->InsertHash( - bloom_filter->Hash(&physical_value, column_schema.type_descriptor.fixed_length)); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(present_value)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field("abc")}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { - const auto column_schema = float16_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(1.0F)}), - *bloom_filter)); + EXPECT_TRUE(selected_ranges.empty()); +} + +TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) { + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + tparquet::FileMetaData metadata; + metadata.__set_column_orders({order}); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(0))}; + + for (const int64_t contradictory_null_count : {5, 11}) { + SCOPED_TRACE(contradictory_null_count); + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_null_pages({true}); + page_index.column_index.__set_null_counts({contradictory_null_count}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, std::move(page_index)); + std::vector selected_ranges; + std::map skip_plans; + + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 10, &selected_ranges, + &skip_plans, nullptr) + .ok()); + // ColumnIndex is optional. An impossible all-null claim must fall back to reading the + // ten-row data page instead of proving that no value can satisfy the predicate. + ASSERT_EQ(selected_ranges.size(), 1); + EXPECT_EQ(selected_ranges[0].start, 0); + EXPECT_EQ(selected_ranges[0].length, 10); + } } } // namespace diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp index 4bca77c1803b49..0680cc828a9f7a 100644 --- a/be/test/format_v2/parquet/parquet_type_test.cpp +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -17,478 +17,49 @@ #include "format_v2/parquet/parquet_type.h" -#include -#include #include -#include -#include -#include - -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/primitive_type.h" namespace doris::format::parquet { -namespace { - -::parquet::SchemaDescriptor make_descriptor(const ::parquet::schema::NodePtr& node) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {node}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - return descriptor; -} - -ParquetTypeDescriptor resolve_node(const ::parquet::schema::NodePtr& node) { - auto descriptor = make_descriptor(node); - return resolve_parquet_type(descriptor.Column(0)); -} - -PrimitiveType primitive_type(const DataTypePtr& type) { - return remove_nullable(type)->get_primitive_type(); -} - -int scale_of(const DataTypePtr& type) { - return remove_nullable(type)->get_scale(); -} - -std::shared_ptr make_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.Append(0x3E00).ok()); - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetTypeDescriptor resolve_arrow_float16_type() { - const auto schema = arrow::schema({arrow::field("f16", arrow::float16(), true)}); - const auto table = arrow::Table::Make(schema, {make_float16_array()}); - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1).ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - - auto reader = ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); - return resolve_parquet_type(reader->metadata()->schema()->Column(0)); -} - -} // namespace - -TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { - struct Case { - int bit_width; - bool is_signed; - PrimitiveType expected_type; - bool expected_unsigned; - }; - const std::vector cases = { - {8, true, TYPE_TINYINT, false}, {8, false, TYPE_SMALLINT, true}, - {16, true, TYPE_SMALLINT, false}, {16, false, TYPE_INT, true}, - {32, true, TYPE_INT, false}, {32, false, TYPE_BIGINT, true}, - {64, true, TYPE_BIGINT, false}, {64, false, TYPE_LARGEINT, true}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.bit_width); - const auto node = ::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Int(test_case.bit_width, test_case.is_signed), - test_case.bit_width == 64 ? ::parquet::Type::INT64 : ::parquet::Type::INT32); - const auto type = resolve_node(node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_TRUE(type.supports_record_reader); - } -} - -TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { - const auto time_millis = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - ASSERT_NE(time_millis.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_millis.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_millis.time_unit, ParquetTimeUnit::MILLIS); - EXPECT_EQ(time_millis.extra_type_info, ParquetExtraTypeInfo::UNIT_MS); - - const auto time_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_us", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64)); - ASSERT_NE(time_micros.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_micros.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(time_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - - const auto adjusted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_adjusted", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - EXPECT_EQ(adjusted_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_time.supports_record_reader); - EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); - - const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ns", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(timestamp_nanos.doris_type, nullptr); - EXPECT_TRUE(timestamp_nanos.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(timestamp_nanos.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(timestamp_nanos.is_timestamp); - EXPECT_TRUE(timestamp_nanos.timestamp_is_adjusted_to_utc); - EXPECT_EQ(timestamp_nanos.time_unit, ParquetTimeUnit::NANOS); - EXPECT_EQ(timestamp_nanos.extra_type_info, ParquetExtraTypeInfo::UNIT_NS); -} - -TEST(ParquetTypeTest, ResolveLogicalTimestampMatrix) { - struct Case { - ::parquet::LogicalType::TimeUnit::unit parquet_unit; - bool adjusted_to_utc; - ParquetTimeUnit expected_unit; - ParquetExtraTypeInfo expected_extra; - int expected_scale; - }; - const std::vector cases = { - {::parquet::LogicalType::TimeUnit::MILLIS, true, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MILLIS, false, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MICROS, true, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::MICROS, false, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, true, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, false, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_scale); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(test_case.adjusted_to_utc, - test_case.parquet_unit), - ::parquet::Type::INT64)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(type.doris_type), test_case.expected_scale); - EXPECT_TRUE(type.is_timestamp); - EXPECT_EQ(type.timestamp_is_adjusted_to_utc, test_case.adjusted_to_utc); - EXPECT_EQ(type.time_unit, test_case.expected_unit); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { - const auto converted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS)); - EXPECT_EQ(converted_time.doris_type, nullptr); - EXPECT_FALSE(converted_time.supports_record_reader); - EXPECT_FALSE(converted_time.unsupported_reason.empty()); - - const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MILLIS)); - ASSERT_NE(converted_timestamp.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_timestamp.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(converted_timestamp.is_timestamp); - EXPECT_TRUE(converted_timestamp.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp.time_unit, ParquetTimeUnit::MILLIS); - const auto converted_timestamp_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_us", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MICROS)); - ASSERT_NE(converted_timestamp_micros.doris_type, nullptr); - EXPECT_TRUE(converted_timestamp_micros.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(converted_timestamp_micros.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(converted_timestamp_micros.doris_type), 6); - EXPECT_TRUE(converted_timestamp_micros.is_timestamp); - EXPECT_TRUE(converted_timestamp_micros.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(converted_timestamp_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); -} - -TEST(ParquetTypeTest, ResolveConvertedIntegerMappingsAndDecodedKinds) { - struct Case { - ::parquet::ConvertedType::type converted_type; - ::parquet::Type::type physical_type; - PrimitiveType expected_type; - int bit_width; - bool expected_unsigned; - DecodedValueKind expected_value_kind; - }; - const std::vector cases = { - {::parquet::ConvertedType::INT_8, ::parquet::Type::INT32, TYPE_TINYINT, 8, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_8, ::parquet::Type::INT32, TYPE_SMALLINT, 8, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_16, ::parquet::Type::INT32, TYPE_SMALLINT, 16, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_16, ::parquet::Type::INT32, TYPE_INT, 16, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_32, ::parquet::Type::INT32, TYPE_INT, 32, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_32, ::parquet::Type::INT32, TYPE_BIGINT, 32, true, - DecodedValueKind::UINT32}, - {::parquet::ConvertedType::INT_64, ::parquet::Type::INT64, TYPE_BIGINT, 64, false, - DecodedValueKind::INT64}, - {::parquet::ConvertedType::UINT_64, ::parquet::Type::INT64, TYPE_LARGEINT, 64, true, - DecodedValueKind::UINT64}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.converted_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, test_case.physical_type, - test_case.converted_type)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_value_kind); - } -} - -TEST(ParquetTypeTest, ResolveConvertedDecimalCarriers) { - struct Case { - ::parquet::Type::type physical_type; - int type_length; - int precision; - int scale; - PrimitiveType expected_type; - ParquetExtraTypeInfo expected_extra; - }; - const std::vector cases = { - {::parquet::Type::INT32, -1, 9, 2, TYPE_DECIMAL32, ParquetExtraTypeInfo::DECIMAL_INT32}, - {::parquet::Type::INT64, -1, 18, 6, TYPE_DECIMAL64, - ParquetExtraTypeInfo::DECIMAL_INT64}, - {::parquet::Type::BYTE_ARRAY, -1, 20, 5, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16, 38, 6, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20, 39, 6, TYPE_DECIMAL256, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.physical_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, test_case.physical_type, - ::parquet::ConvertedType::DECIMAL, test_case.type_length, test_case.precision, - test_case.scale)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_TRUE(type.is_decimal); - EXPECT_FALSE(type.is_string_like); - EXPECT_EQ(type.decimal_precision, test_case.precision); - EXPECT_EQ(type.decimal_scale, test_case.scale); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ResolveLogicalStringDateAndDecimalMappings) { - const std::vector> string_like_logical_types = { - ::parquet::LogicalType::String(), ::parquet::LogicalType::Enum(), - ::parquet::LogicalType::JSON(), ::parquet::LogicalType::BSON()}; - for (const auto& logical_type : string_like_logical_types) { - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::OPTIONAL, logical_type, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_STRING); - EXPECT_TRUE(type.is_string_like); - } - - const auto uuid = resolve_node(::parquet::schema::PrimitiveNode::Make( - "uuid", ::parquet::Repetition::OPTIONAL, ::parquet::LogicalType::UUID(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(uuid.doris_type, nullptr); - EXPECT_TRUE(uuid.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(uuid.doris_type), TYPE_STRING); - EXPECT_TRUE(uuid.is_string_like); - - const auto date = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Date(), - ::parquet::Type::INT32)); - ASSERT_NE(date.doris_type, nullptr); - EXPECT_EQ(primitive_type(date.doris_type), TYPE_DATEV2); - - const auto decimal64 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d64", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(18, 6), - ::parquet::Type::INT64)); - ASSERT_NE(decimal64.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal64.doris_type), TYPE_DECIMAL64); - EXPECT_TRUE(decimal64.is_decimal); - EXPECT_EQ(decimal64.decimal_precision, 18); - EXPECT_EQ(decimal64.decimal_scale, 6); - EXPECT_EQ(decimal64.extra_type_info, ParquetExtraTypeInfo::DECIMAL_INT64); - - const auto decimal128 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d128", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(38, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(decimal128.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal128.doris_type), TYPE_DECIMAL128I); - EXPECT_TRUE(decimal128.is_decimal); - EXPECT_EQ(decimal128.decimal_precision, 38); - EXPECT_EQ(decimal128.decimal_scale, 6); - EXPECT_EQ(decimal128.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d256", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(39, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - EXPECT_FALSE(decimal256.is_string_like); -} - -TEST(ParquetTypeTest, LogicalConvertedAndPhysicalFallbackLevelsAreDistinct) { - const auto logical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Int(8, true), - ::parquet::Type::INT32)); - ASSERT_NE(logical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(logical_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(logical_type.integer_bit_width, 8); - - const auto converted_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8)); - ASSERT_NE(converted_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(converted_type.integer_bit_width, 8); - - const auto physical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); - ASSERT_NE(physical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(physical_type.doris_type), TYPE_INT); - EXPECT_EQ(physical_type.integer_bit_width, -1); -} - -TEST(ParquetTypeTest, ResolveDecimalStringLikeFloat16AndPhysicalFallback) { - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::DECIMAL, 20, 39, 6)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_FALSE(decimal256.is_string_like); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto plain_binary = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::REQUIRED, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(plain_binary.doris_type, nullptr); - EXPECT_EQ(primitive_type(plain_binary.doris_type), TYPE_STRING); - EXPECT_TRUE(plain_binary.is_string_like); - - const auto float16 = resolve_arrow_float16_type(); - ASSERT_NE(float16.doris_type, nullptr); - EXPECT_TRUE(float16.doris_type->is_nullable()); - EXPECT_EQ(float16.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(float16.fixed_length, 2); - EXPECT_EQ(primitive_type(float16.doris_type), TYPE_FLOAT); - EXPECT_EQ(float16.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(float16.is_string_like); - EXPECT_EQ(decoded_value_kind(float16), DecodedValueKind::FIXED_BINARY); -} - -TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { - const auto null_type = resolve_parquet_type(nullptr); - EXPECT_EQ(null_type.doris_type, nullptr); - EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); - EXPECT_TRUE(null_type.supports_record_reader); - - const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); - ASSERT_NE(int96.doris_type, nullptr); - EXPECT_EQ(primitive_type(int96.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(int96.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - EXPECT_EQ(decoded_value_kind(int96), DecodedValueKind::INT96); -} - -TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { - struct Case { - ::parquet::schema::NodePtr node; - PrimitiveType expected_type; - DecodedValueKind expected_kind; - bool expected_string_like = false; - }; - const std::vector cases = { - {::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BOOLEAN), - TYPE_BOOLEAN, DecodedValueKind::BOOL}, - {::parquet::schema::PrimitiveNode::Make("i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - TYPE_INT, DecodedValueKind::INT32}, - {::parquet::schema::PrimitiveNode::Make("i64", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT64), - TYPE_BIGINT, DecodedValueKind::INT64}, - {::parquet::schema::PrimitiveNode::Make("f", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FLOAT), - TYPE_FLOAT, DecodedValueKind::FLOAT}, - {::parquet::schema::PrimitiveNode::Make("d", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - TYPE_DOUBLE, DecodedValueKind::DOUBLE}, - {::parquet::schema::PrimitiveNode::Make("s", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY), - TYPE_STRING, DecodedValueKind::BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("fs", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::NONE, 4), - TYPE_STRING, DecodedValueKind::FIXED_BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("ts", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT96), - TYPE_DATETIMEV2, DecodedValueKind::INT96}, +TEST(ParquetTypeTest, DecodedValueKindUsesNativePhysicalTypes) { + ParquetTypeDescriptor descriptor; + + const std::pair cases[] = { + {tparquet::Type::BOOLEAN, DecodedValueKind::BOOL}, + {tparquet::Type::INT32, DecodedValueKind::INT32}, + {tparquet::Type::INT64, DecodedValueKind::INT64}, + {tparquet::Type::INT96, DecodedValueKind::INT96}, + {tparquet::Type::FLOAT, DecodedValueKind::FLOAT}, + {tparquet::Type::DOUBLE, DecodedValueKind::DOUBLE}, + {tparquet::Type::BYTE_ARRAY, DecodedValueKind::BINARY}, + {tparquet::Type::FIXED_LEN_BYTE_ARRAY, DecodedValueKind::FIXED_BINARY}, }; - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_type); - const auto type = resolve_node(test_case.node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); - EXPECT_EQ(type.is_string_like, test_case.expected_string_like); - EXPECT_TRUE(type.supports_record_reader); + for (const auto& [physical_type, expected] : cases) { + descriptor.physical_type = physical_type; + descriptor.is_unsigned_integer = false; + descriptor.integer_bit_width = -1; + EXPECT_EQ(decoded_value_kind(descriptor), expected); } } -TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { - EXPECT_THROW(::parquet::LogicalType::Int(24, true), ::parquet::ParquetException); +TEST(ParquetTypeTest, DecodedValueKindPreservesUnsignedWidth) { + ParquetTypeDescriptor descriptor; + descriptor.is_unsigned_integer = true; - const auto nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(nanos_time.doris_type, nullptr); - EXPECT_EQ(primitive_type(nanos_time.doris_type), TYPE_BIGINT); - EXPECT_TRUE(nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 32; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT32); - const auto adjusted_nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns_utc", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_nanos_time.supports_record_reader); - EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT64; + descriptor.integer_bit_width = 64; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT64); - EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Float16(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 4), - ::parquet::ParquetException); + // Narrow unsigned logical integers still use the signed physical decoder; conversion + // happens after decoding so the on-disk bit width remains the single source of truth. + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 16; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::INT32); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hive_reader_test.cpp b/be/test/format_v2/table/hive_reader_test.cpp index d4f9be81129c6c..98d39fbd328093 100644 --- a/be/test/format_v2/table/hive_reader_test.cpp +++ b/be/test/format_v2/table/hive_reader_test.cpp @@ -79,7 +79,7 @@ bool has_name_mapping(const ColumnDefinition& column, const std::string& name) { class HiveV2ReaderTest : public testing::Test { public: - HiveV2ReaderTest() : profile("hive_v2_reader_test") { state.set_query_options(query_options); } + HiveV2ReaderTest() : state(query_options, query_globals), profile("hive_v2_reader_test") {} protected: TQueryOptions query_options; diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 28c396371e4559..f5bd70292e51a3 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -17,21 +17,35 @@ #include "format_v2/table/hudi_reader.h" +#include +#include #include +#include +#include +#include +#include #include #include #include +#include #include #include +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format_v2/column_data.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" namespace doris::format { namespace { @@ -69,6 +83,39 @@ ColumnDefinition make_file_column(int32_t id, const std::string& name, const Dat return field; } +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = name; + column.type = make_nullable(type); + return column; +} + +Block build_table_block(const std::vector& columns) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +void write_int_parquet_file(const std::string& file_path, const std::vector& values) { + arrow::Int32Builder value_builder; + for (const auto value : values) { + ASSERT_TRUE(value_builder.Append(value).ok()); + } + std::shared_ptr value_array; + ASSERT_TRUE(value_builder.Finish(&value_array).ok()); + const auto table = arrow::Table::Make( + arrow::schema({arrow::field("id", arrow::int32(), false)}), {value_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr output = *file_result; + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, + static_cast(values.size()))); +} + TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { TTableFormatFileDesc table_format_params; table_format_params.__set_table_format_type("hudi"); @@ -80,6 +127,71 @@ TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { return table_format_params; } +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -201,5 +313,207 @@ TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 123); } +TEST(HudiHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(HudiHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(HudiHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("hudi_scanner_stateful_residual"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + run_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + +TEST(HudiHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.TEST_set_child_condition_cache_hits(2, 7); + EXPECT_EQ(reader.condition_cache_hit_count(), 9); +} + +TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_hudi_hybrid_count_star_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "base-file.parquet").string(); + write_int_parquet_file(file_path, {1, 2, 3}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + io_ctx->file_cache_stats = &file_cache_stats; + ShardedKVCache cache(1); + + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.cache = &cache; + split_options.current_split_format = FileFormat::PARQUET; + split_options.current_range.__set_path(file_path); + split_options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + split_options.current_range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + split_options.current_range.__set_table_format_params(hudi_table_format_desc(std::nullopt)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 3); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(HudiHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + verify_first_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..a219ba37f3dd8d --- /dev/null +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { +namespace { + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +TEST(IcebergPositionDeleteSysTableV2ProfileTest, UsesDistinctProfileForNestedPositionReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("position_delete_system_table_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + delete_file.__set_path("/not-opened-during-prepare.parquet"); + + IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(delete_file); + ASSERT_TRUE(reader._init_split().ok()); + + ASSERT_NE(reader._position_reader_profile, nullptr); + EXPECT_NE(reader._position_reader_profile, &profile); + EXPECT_EQ(profile.get_child("IcebergPositionDeleteFileReader"), + reader._position_reader_profile); +} + +} // namespace +} // namespace doris::format::iceberg diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index d5ce700970496e..c42d68933f2014 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -21,15 +21,19 @@ #include #include #include +#include #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -55,13 +59,14 @@ #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" +#include "exec/scan/access_path_parser.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" #include "format/format_common.h" -#include "format_v2/deletion_vector_reader.h" +#include "format/table/deletion_vector_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/ExternalTableSchema_types.h" @@ -72,6 +77,7 @@ #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { @@ -229,7 +235,9 @@ class IcebergTableReaderScanRequestTestHelper final class IcebergTableReaderMappingModeTestHelper final : public doris::format::iceberg::IcebergTableReader { public: - TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema) { + TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema, + TFileScanRangeParams* scan_params = nullptr) { + _scan_params = scan_params; _data_reader.file_schema = std::move(file_schema); return mapping_mode(); } @@ -270,6 +278,19 @@ std::shared_ptr build_nullable_int64_array( return finish_array(&builder); } +std::shared_ptr build_nullable_string_array( + const std::vector>& values) { + arrow::StringBuilder builder; + for (const auto& value : values) { + if (value.has_value()) { + EXPECT_TRUE(builder.Append(*value).ok()); + } else { + EXPECT_TRUE(builder.AppendNull().ok()); + } + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -449,6 +470,39 @@ void write_single_int_parquet_file(const std::string& file_path, const std::stri builder.build())); } +void write_unsupported_time_first_int_parquet_file(const std::string& file_path, + const std::vector& ids) { + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + // Arrow writes time32 as isAdjustedToUTC=false, which Doris supports. Use Parquet's low-level + // writer so the first physical column is the deliberately unsupported adjusted TIME_MILLIS + // shape from the review report. The query will project only the following supported `id`. + const auto unsupported_time = ::parquet::schema::PrimitiveNode::Make( + "unsupported_time", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto id = ::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32); + const auto schema_node = ::parquet::schema::GroupNode::Make( + "schema", ::parquet::Repetition::REQUIRED, {unsupported_time, id}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + + auto writer = ::parquet::ParquetFileWriter::Open(out, schema); + auto* row_group = writer->AppendRowGroup(); + auto* time_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + std::vector times(ids.size(), 1000); + const auto row_count = cast_set(ids.size()); + EXPECT_EQ(time_writer->WriteBatch(row_count, nullptr, nullptr, times.data()), row_count); + time_writer->Close(); + auto* id_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + EXPECT_EQ(id_writer->WriteBatch(row_count, nullptr, nullptr, ids.data()), row_count); + id_writer->Close(); + row_group->Close(); + writer->Close(); +} + void write_two_int_parquet_file(const std::string& file_path, const std::string& first_name, const std::vector& first_values, std::optional first_field_id, @@ -482,6 +536,77 @@ void write_two_int_parquet_file(const std::string& file_path, const std::string& builder.build())); } +void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value, + bool outer_has_field_id = true) { + const auto leaf_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"30"}); + auto leaf_field = arrow::field("leaf", arrow::int32(), false)->WithMetadata(leaf_metadata); + auto inner_result = arrow::StructArray::Make({build_int32_array({value})}, {leaf_field}); + ASSERT_TRUE(inner_result.ok()) << inner_result.status(); + auto inner_field = arrow::field("inner", arrow::struct_({leaf_field}), false); + auto outer_result = arrow::StructArray::Make({*inner_result}, {inner_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false); + if (outer_has_field_id) { + outer_field = + outer_field->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + } + auto table = arrow::Table::Make(arrow::schema({outer_field}), {*outer_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_nullable_idless_struct_with_sibling_id_parquet_file(const std::string& file_path) { + const auto sibling_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto sibling_field = arrow::field("b", arrow::int32(), false)->WithMetadata(sibling_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 42})}, {sibling_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = arrow::field("s", arrow::struct_({sibling_field}), true); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + +void write_nullable_renamed_struct_child_parquet_file(const std::string& file_path) { + const auto child_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + auto child_field = arrow::field("b", arrow::int32(), false)->WithMetadata(child_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 10})}, {child_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = + arrow::field("s", arrow::struct_({child_field}), true) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + void write_timestamp_int_parquet_file(const std::string& file_path, const std::vector& timestamps, const std::vector& ids) { @@ -626,6 +751,31 @@ void write_position_delete_parquet_file(const std::string& file_path, builder.build())); } +void write_nullable_position_delete_parquet_file( + const std::string& file_path, + const std::vector>& data_file_paths, + const std::vector>& positions) { + ASSERT_EQ(data_file_paths.size(), positions.size()); + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), true), + arrow::field("pos", arrow::int64(), true), + }); + auto table = arrow::Table::Make(schema, {build_nullable_string_array(data_file_paths), + build_nullable_int64_array(positions)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(positions.size()), + builder.build())); +} + int64_t write_iceberg_deletion_vector_file(const std::string& file_path, const std::vector& deleted_positions) { roaring::Roaring64Map rows; @@ -641,7 +791,8 @@ int64_t write_iceberg_deletion_vector_file(const std::string& file_path, BigEndian::Store32(blob.data(), total_length); constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; memcpy(blob.data() + 4, DV_MAGIC, 4); - BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + const uint32_t crc = HashUtil::zlib_crc_hash(blob.data() + 4, total_length, 0); + BigEndian::Store32(blob.data() + 8 + bitmap_size, crc); std::ofstream output(file_path, std::ios::binary); EXPECT_TRUE(output.is_open()); @@ -658,6 +809,12 @@ class ScopedDebugPoint { DebugPoints::instance()->add(_name); } + ScopedDebugPoint(std::string name, std::function callback) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_callback(_name, std::move(callback)); + } + ~ScopedDebugPoint() { DebugPoints::instance()->remove(_name); config::enable_debug_points = _enable_debug_points; @@ -847,9 +1004,19 @@ TFileScanRangeParams make_local_parquet_scan_params() { TFileScanRangeParams scan_params; scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); return scan_params; } +TColumnAccessPath nested_data_access_path(std::vector path) { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_path; + data_path.__set_path(std::move(path)); + access_path.__set_data_access_path(std::move(data_path)); + return access_path; +} + std::shared_ptr make_io_context(io::FileReaderStats* file_reader_stats, io::FileCacheStatistics* file_cache_stats) { auto io_ctx = std::make_shared(); @@ -979,11 +1146,6 @@ VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { return ctx; } -void apply_final_conjuncts(Block* block, const VExprContextSPtrs& conjuncts) { - const auto status = VExprContext::filter_block(conjuncts, block, block->columns()); - ASSERT_TRUE(status.ok()) << status; -} - TEST(IcebergV2ReaderTest, IcebergVirtualColumnsUseRowLineageMetadata) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_virtual_columns_test"; @@ -1222,9 +1384,6 @@ TEST(IcebergV2ReaderTest, IcebergRowIdPredicateFiltersAfterRowLineageMaterializa bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); @@ -1276,9 +1435,6 @@ TEST(IcebergV2ReaderTest, IcebergLastUpdatedSequencePredicateFiltersAfterMateria bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); @@ -1535,8 +1691,25 @@ TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsMissingRange) { auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("missing content offset or length"), std::string::npos); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("descriptor misses"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsInvalidRange) { + TTableFormatFileDesc table_format_desc; + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_format_version(2); + iceberg_desc.__set_delete_files({make_iceberg_deletion_vector("dv.bin", -1, 12)}); + table_format_desc.__set_iceberg_params(iceberg_desc); + + IcebergTableReaderDeleteFileTestHelper reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("offset must be non-negative"), std::string::npos); EXPECT_FALSE(has_delete_file); } @@ -1805,10 +1978,12 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeIgnoresGlobalRowIdVirtualColumn) { TableColumnMappingMode::BY_FIELD_ID); } -// Covers the fallback side of the previous case. Only synthesized columns are ignored; a real data -// column without an Iceberg field id still disables field-id mapping. -TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { +// Iceberg treats a schema as ID-bearing when any physical data field has an ID. Keeping ID mode +// preserves authoritative matches even when a sibling was written without an ID. +TEST(IcebergV2ReaderTest, IcebergMappingModeUsesAnyDataColumnFieldId) { IcebergTableReaderMappingModeTestHelper reader; + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); std::vector file_schema { make_file_column(1, "id", std::make_shared()), make_file_column(2, "name", std::make_shared()), @@ -1816,7 +1991,35 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { }; file_schema[1].identifier = Field {}; - EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), + EXPECT_EQ(reader.mapping_mode_for_schema(file_schema, &scan_params), + TableColumnMappingMode::BY_FIELD_ID); + + file_schema[0].identifier = Field {}; + file_schema[0].children.emplace_back( + make_file_column(3, "nested", std::make_shared())); + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema), &scan_params), + TableColumnMappingMode::BY_FIELD_ID); +} + +TEST(IcebergV2ReaderTest, IcebergLegacyPlanKeepsAllFieldIdsMappingRule) { + IcebergTableReaderMappingModeTestHelper reader; + TFileScanRangeParams old_fe_scan_params; + std::vector file_schema { + make_file_column(1, "a", std::make_shared()), + make_file_column(2, "b", std::make_shared()), + }; + file_schema[1].identifier = Field {}; + + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema), &old_fe_scan_params), + TableColumnMappingMode::BY_NAME); + + auto nested = make_file_column(10, "s", std::make_shared()); + nested.children = { + make_file_column(1, "a", std::make_shared()), + make_file_column(2, "b", std::make_shared()), + }; + nested.children[1].identifier = Field {}; + EXPECT_EQ(reader.mapping_mode_for_schema({std::move(nested)}, &old_fe_scan_params), TableColumnMappingMode::BY_NAME); } @@ -1900,6 +2103,9 @@ TEST(IcebergV2ReaderTest, IcebergTableLevelCountUsesAssignedRowCountWithPosition .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + // An explicit empty argument list is the new FE marker for + // COUNT(*)/COUNT(1); nullopt intentionally exercises fallback. + .push_down_count_columns = std::vector {}, }) .ok()); @@ -2109,6 +2315,45 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesNullForMissingDataColumn) std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMissingKeyDoesNotReadUnsupportedUnprojectedCarrier) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_virtual_carrier_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_unsupported_time_first_int_parquet_file(file_path, {1, 2, 3}); + write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7, "added_column"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + // The missing data key is NULL, so it does not match delete key 7. More importantly, the + // hidden row-count dependency must use virtual row position instead of the first physical + // TIME_MILLIS column, which is unsupported and was not requested by this `id` projection. + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesInitialDefaultForMissingDataColumn) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_missing_default_test"; @@ -2202,7 +2447,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default("\x00\x01\x02\xff", 4); + static_assert(sizeof("0123456789abcdef0123456789abcdef") - 1 > StringView::kInlineSize); + const std::string binary_default = "0123456789abcdef0123456789abcdef"; write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2211,10 +2457,12 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor auto scan_params = make_local_parquet_scan_params(); scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( - 100, - {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "AAEC/w==", - external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + 100, {external_schema_field("id", 0), + external_schema_field( + "added_binary", 1, {}, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + external_primitive_type(TPrimitiveType::VARBINARY, + static_cast(binary_default.size())), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -2405,7 +2653,401 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldId std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { +TEST(IcebergV2ReaderTest, ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_recursive_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42, false); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + inner.has_name_mapping = true; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + outer.has_name_mapping = true; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetUsesUnprojectedSiblingIdToRetainNullableWrapper) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_unprojected_sibling_id_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.initial_default_value = "7"; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + auto schema_a = + external_schema_field("a", 1, {}, "7", external_primitive_type(TPrimitiveType::INT)); + auto schema_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + schema::external::TStructField struct_children; + struct_children.__set_fields({schema_a, schema_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ReusedRootNameReadsNewFieldInitialDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_reused_root_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_single_int_parquet_file(file_path, "b", {10}, 1); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = + external_schema_field("b", 2, {}, "7", external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {renamed_b, current_b})}); + + auto projected_b = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_b).ok()); + std::vector projected_columns = {projected_b}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({7})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, LegacyPlanRetainsOrderedRootNameAndAliasLookupWithAllFieldIds) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_legacy_reused_root_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_two_int_parquet_file(file_path, "renamed_b", {10}, 1, "b", {20}, 2); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + old_fe_scan_params.__set_current_schema_id(100); + old_fe_scan_params.__set_history_schema_info({external_schema(100, {renamed_b, current_b})}); + + auto projected_b = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_b).ok()); + ASSERT_EQ(projected_b.get_identifier_field_id(), 1); + std::vector projected_columns = {projected_b}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &old_fe_scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({10})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ReusedNestedNameReadsNewFieldInitialDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_reused_nested_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_renamed_struct_child_parquet_file(file_path); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = + external_schema_field("b", 2, {}, "7", external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + schema::external::TStructField struct_children; + struct_children.__set_fields({renamed_b, current_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + + const auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"b"}); + auto projected_s = make_table_column(-1, "s", struct_type); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_s).ok()); + ASSERT_TRUE(context.schema_column.has_value()); + ASSERT_TRUE(AccessPathParser::build_nested_children( + &projected_s, + std::vector {nested_data_access_path({"s", "b"})}, + &*context.schema_column) + .ok()); + std::vector projected_columns = {projected_s}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_b = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_b.is_null_at(1)); + const auto& values = assert_cast(nullable_b.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, LegacyPlanUsesNestedNameMappingForMixedFieldIds) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_legacy_nested_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.name_mapping = {"b"}; + projected_a.has_name_mapping = true; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &old_fe_scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; std::filesystem::remove_all(test_dir); @@ -2413,8 +3055,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); - // The real key has no id and is mapped by its historical name. A different physical column - // carries the stale id 0, forcing the entire split into BY_NAME mode. + // Iceberg's existential hasIds contract makes the physical field carrying ID 0 authoritative; + // an ID-less alias cannot switch a mixed schema back to name projection. write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", {100, 200, 300}, 0); write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); @@ -2439,7 +3081,7 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({100, 200, 300})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); @@ -2801,7 +3443,25 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { first_split.cache = &cache; first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); - ASSERT_TRUE(reader.prepare_split(first_split).ok()); + const auto* total_timer = profile.get_counter("TableReader"); + const auto* prepare_timer = profile.get_counter("PrepareSplitTime"); + ASSERT_NE(total_timer, nullptr); + ASSERT_NE(prepare_timer, nullptr); + const int64_t total_before = total_timer->value(); + const int64_t prepare_before = prepare_timer->value(); + constexpr int64_t DELETE_FILE_DELAY_NS = 8'000'000; + { + ScopedDebugPoint delay_delete_file_scan( + "IcebergTableReader.prepare_split.before_delete_file_scan", + [] { std::this_thread::sleep_for(std::chrono::milliseconds(8)); }); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + } + const int64_t total_delta = total_timer->value() - total_before; + const int64_t prepare_delta = prepare_timer->value() - prepare_before; + // A cache-miss delete-file scan is derived prepare work; both common parent timers must + // contain it so the expensive miss cannot disappear between profile levels. + EXPECT_GE(prepare_delta, DELETE_FILE_DELAY_NS); + EXPECT_GE(total_delta, DELETE_FILE_DELAY_NS); EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); // The cached delete file contains entries for every referenced data file, so another split can @@ -2864,6 +3524,147 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteCacheIsScopedByFileSystem) { std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_nullable_position_delete_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_nullable_position_delete_parquet_file(delete_file_path, {file_path, file_path}, + {int64_t {1}, int64_t {3}}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeleteFileWithActualNulls) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_nullable_position_delete_actual_null_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_nullable_position_delete_parquet_file(delete_file_path, {file_path, std::nullopt}, + {int64_t {1}, int64_t {2}}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + auto status = reader.prepare_split(split_options); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("file_path contains null values"), std::string::npos) + << status.to_string(); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderRejectsNullablePositionDeletePosWithActualNulls) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_nullable_position_delete_pos_null_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_nullable_position_delete_parquet_file(delete_file_path, {file_path, file_path}, + {int64_t {1}, std::nullopt}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + auto status = reader.prepare_split(split_options); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("pos contains null values"), std::string::npos) + << status.to_string(); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergTableReaderIgnoresPositionDeleteFilesWhenDeletionVectorPresent) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_delete_files_merge_test"; diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 4d7fb6a75965a8..77466439c6be78 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include "core/assert_cast.h" @@ -43,10 +45,13 @@ #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/common/endian.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format/format_common.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" #include "format_v2/column_data.h" -#include "format_v2/deletion_vector_reader.h" #include "format_v2/jni/paimon_jni_reader.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -60,6 +65,71 @@ namespace doris::format { namespace { +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -461,6 +531,20 @@ TEST(PaimonReaderTest, DeletionVectorCacheKeyIncludesOffsetAndLength) { EXPECT_NE(first_desc.key, different_length_desc.key); } +TEST(PaimonReaderTest, DeletionVectorRejectsInvalidRange) { + auto table_format_params = make_paimon_table_format_desc("dv.bin", -1, 4); + + paimon::PaimonReader reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = + reader.TEST_parse_deletion_vector_file(table_format_params, &desc, &has_delete_file); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("offset must be non-negative"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + TEST(PaimonReaderTest, DecodeDeletionVectorBufferUsesSharedFormatHelper) { // Scenario: format_v2 TableReader reads a raw Paimon BitmapDeletionVector range and delegates // the binary parsing to the same helper used by the format reader path. @@ -661,6 +745,161 @@ TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 321); } +TEST(PaimonHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(PaimonHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(PaimonHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("paimon_scanner_stateful_residual"); + auto scan_params = make_local_parquet_scan_params(); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + run_split(FileFormat::JNI, make_paimon_jni_range()); + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + +TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.TEST_set_child_condition_cache_hits(3, 5); + EXPECT_EQ(reader.condition_cache_hit_count(), 8); +} + +TEST(PaimonHybridReaderTest, NativeCountColumnReportsMetadataRowsThroughHybridReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_paimon_hybrid_count_column_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "data-file.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.cache = &cache; + split_options.current_split_format = FileFormat::PARQUET; + split_options.current_range = make_paimon_native_range(TFileFormatType::FORMAT_PARQUET); + split_options.current_range.__set_path(file_path); + split_options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 3); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -696,6 +935,49 @@ TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { ASSERT_TRUE(reader.close().ok()); } +TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + verify_first_split(FileFormat::JNI, make_paimon_jni_range()); +} + TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ diff --git a/be/test/format_v2/table/remote_doris_reader_test.cpp b/be/test/format_v2/table/remote_doris_reader_test.cpp index b17f82f505c2c9..a8affbde3117a3 100644 --- a/be/test/format_v2/table/remote_doris_reader_test.cpp +++ b/be/test/format_v2/table/remote_doris_reader_test.cpp @@ -18,11 +18,17 @@ #include "format_v2/table/remote_doris_reader.h" #include +#include #include #include +#include +#include +#include #include +#include #include +#include #include #include #include @@ -48,6 +54,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "testutil/desc_tbl_builder.h" +#include "testutil/mock/mock_runtime_state.h" namespace doris::format::remote_doris { namespace { @@ -95,6 +102,122 @@ TFileRangeDesc remote_doris_range() { return range; } +class BlockingRecordBatchReader final : public arrow::RecordBatchReader { +public: + std::shared_ptr schema() const override { + return arrow::schema({arrow::field("id", arrow::int32())}); + } + + arrow::Status ReadNext(std::shared_ptr* batch) override { + { + std::lock_guard lock(_mutex); + _entered = true; + } + _cv.notify_all(); + std::unique_lock lock(_mutex); + _cv.wait(lock, [this] { return _released; }); + *batch = nullptr; + return arrow::Status::OK(); + } + + bool wait_until_entered(std::chrono::milliseconds timeout) { + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, timeout, [this] { return _entered; }); + } + + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _cv.notify_all(); + } + +private: + std::mutex _mutex; + std::condition_variable _cv; + bool _entered = false; + bool _released = false; +}; + +class BlockingFlightServer final : public arrow::flight::FlightServerBase { +public: + enum class Mode { DO_GET, NEXT }; + + explicit BlockingFlightServer(Mode mode) + : _mode(mode), _batch_reader(std::make_shared()) {} + + arrow::Status start() { + auto location = arrow::flight::Location::ForGrpcTcp("localhost", 0); + if (!location.ok()) { + return location.status(); + } + // FlightServerBase::Init starts serving immediately; the blocking Serve lifecycle wrapper + // is unnecessary for an in-process unit test. + return Init(arrow::flight::FlightServerOptions(*location)); + } + + ~BlockingFlightServer() override { + release(); + static_cast(Shutdown()); + } + + arrow::Status DoGet(const arrow::flight::ServerCallContext& context, + const arrow::flight::Ticket&, + std::unique_ptr* stream) override { + if (_mode == Mode::DO_GET) { + { + std::lock_guard lock(_mutex); + _entered = true; + } + _cv.notify_all(); + std::unique_lock lock(_mutex); + while (!_released && !context.is_cancelled()) { + _cv.wait_for(lock, std::chrono::milliseconds(5)); + } + if (context.is_cancelled()) { + return arrow::Status::Cancelled("client cancelled blocked DoGet"); + } + } + *stream = std::make_unique(_batch_reader); + return arrow::Status::OK(); + } + + bool wait_until_entered(std::chrono::milliseconds timeout) { + if (_mode == Mode::NEXT) { + return _batch_reader->wait_until_entered(timeout); + } + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, timeout, [this] { return _entered; }); + } + + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _cv.notify_all(); + _batch_reader->release(); + } + +private: + Mode _mode; + std::shared_ptr _batch_reader; + std::mutex _mutex; + std::condition_variable _cv; + bool _entered = false; + bool _released = false; +}; + +TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) { + auto range = remote_doris_range(); + auto& params = range.table_format_params.remote_doris_params; + params.__set_location_uri("grpc://localhost:" + std::to_string(server.port())); + arrow::flight::Ticket ticket {.ticket = "ticket"}; + params.__set_ticket(ticket.SerializeToString().ValueOrDie()); + return range; +} + std::vector remote_slots(ObjectPool* pool, DescriptorTbl** desc_tbl) { DescriptorTblBuilder builder(pool); builder.declare_tuple() << std::make_tuple(std::make_shared(), std::string("id")) @@ -196,6 +319,16 @@ std::unique_ptr create_reader( std::move(factory)); } +std::unique_ptr create_flight_reader( + RuntimeProfile* profile, const TFileRangeDesc& range, + const std::vector& slots, std::shared_ptr io_ctx) { + auto system_properties = std::make_shared(); + auto file_description = std::make_unique(); + file_description->path = "/dummyPath"; + return std::make_unique(system_properties, file_description, + std::move(io_ctx), profile, range, slots); +} + Block make_request_block(const std::vector& schema, const std::vector& local_ids) { Block block; @@ -467,4 +600,140 @@ TEST(RemoteDorisV2ReaderTest, RejectsInvalidRemoteDorisRange) { EXPECT_FALSE(reader->init(&state).ok()); } +TEST(RemoteDorisV2ReaderTest, RuntimeCancellationInterruptsBlockedFlightDoGet) { + BlockingFlightServer server(BlockingFlightServer::Mode::DO_GET); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_blocked_doget_test"); + auto io_ctx = std::make_shared(); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + + auto open_result = + std::async(std::launch::async, [&] { return reader->open(std::move(request)); }); + const bool entered = server.wait_until_entered(std::chrono::seconds(2)); + if (!entered && + open_result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { + FAIL() << "Flight DoGet failed before reaching the server: " << open_result.get(); + } + ASSERT_TRUE(entered); + state.cancel(Status::Cancelled("cancel blocked Flight DoGet")); + const bool interrupted = + open_result.wait_for(std::chrono::milliseconds(750)) == std::future_status::ready; + if (!interrupted) { + server.release(); + } + EXPECT_TRUE(interrupted); + EXPECT_FALSE(open_result.get().ok()); +} + +TEST(RemoteDorisV2ReaderTest, BlockedDoGetRetainsQueryResourcesUntilWorkerExits) { + BlockingFlightServer server(BlockingFlightServer::Mode::DO_GET); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + auto state = std::make_unique(); + std::weak_ptr resource_ctx = state->get_query_ctx()->resource_ctx(); + RuntimeProfile profile("remote_doris_v2_doget_resource_context_test"); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, + std::make_shared()); + ASSERT_TRUE(reader->init(state.get()).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + + auto open_result = + std::async(std::launch::async, [&] { return reader->open(std::move(request)); }); + ASSERT_TRUE(server.wait_until_entered(std::chrono::seconds(2))); + state->cancel(Status::Cancelled("cancel blocked Flight DoGet")); + ASSERT_EQ(open_result.wait_for(std::chrono::milliseconds(750)), std::future_status::ready); + EXPECT_FALSE(open_result.get().ok()); + + reader.reset(); + state.reset(); + // A detached DoGet still owns query-scoped Arrow objects, so it must retain the matching + // resource context until those objects are released on the worker. + EXPECT_FALSE(resource_ctx.expired()); + server.release(); + for (int retries = 0; retries < 100 && !resource_ctx.expired(); ++retries) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + EXPECT_TRUE(resource_ctx.expired()); +} + +TEST(RemoteDorisV2ReaderTest, ScannerStopInterruptsBlockedFlightNext) { + BlockingFlightServer server(BlockingFlightServer::Mode::NEXT); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_blocked_next_test"); + auto io_ctx = std::make_shared(); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto open_status = reader->open(std::move(request)); + ASSERT_TRUE(open_status.ok()) << open_status; + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + auto next_result = + std::async(std::launch::async, [&] { return reader->get_block(&block, &rows, &eof); }); + ASSERT_TRUE(server.wait_until_entered(std::chrono::seconds(2))); + io_ctx->should_stop = true; + const bool interrupted = + next_result.wait_for(std::chrono::milliseconds(750)) == std::future_status::ready; + if (!interrupted) { + server.release(); + } + EXPECT_TRUE(interrupted); + const auto next_status = next_result.get(); + EXPECT_TRUE(!next_status.ok() || (rows == 0 && eof)); + server.release(); +} + +TEST(RemoteDorisV2ReaderTest, CancellationStopsBeforeFlightNext) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_cancel_test"); + auto close_count = std::make_shared(0); + auto io_ctx = std::make_shared(); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"id"})}, + close_count, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + io_ctx->should_stop = true; + auto block = make_request_block(schema, {0}); + size_t rows = 99; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + EXPECT_EQ(*close_count, 1); +} + } // namespace doris::format::remote_doris diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 4ee2258d1273c0..d92bbf2b0ac927 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -41,6 +41,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -48,11 +49,13 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" -#include "exprs/vruntimefilter_wrapper.h" #include "exprs/vslot_ref.h" +#include "format/table/iceberg_scan_semantics.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -262,8 +265,7 @@ VExprSPtr runtime_filter_wrapper_expr(VExprSPtr impl) { node.__set_node_type(TExprNodeType::SLOT_REF); node.__set_type(std::make_shared()->to_thrift()); node.__set_num_children(1); - return VRuntimeFilterWrapper::create_shared(node, std::move(impl), 0, false, - /*filter_id=*/1); + return RuntimeFilterExpr::create_shared(node, std::move(impl), 0, false, /*filter_id=*/1); } class NonDeterministicPartitionPredicate final : public VExpr { @@ -295,6 +297,37 @@ class NonDeterministicPartitionPredicate final : public VExpr { const std::string _expr_name = "NonDeterministicPartitionPredicate"; }; +class StatefulSequencePredicate final : public VExpr { +public: + explicit StatefulSequencePredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(_observed_invocations != nullptr); + _observed_invocations->push_back(_invocation++); + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_observed_invocations); + return Status::OK(); + } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulSequencePredicate"; +}; + class NullableArrayBigintDefaultExpr final : public VExpr { public: explicit NullableArrayBigintDefaultExpr(DataTypePtr data_type) @@ -328,9 +361,50 @@ class NullableArrayBigintDefaultExpr final : public VExpr { class TableReaderMaterializeTestHelper final : public TableReader { public: + using TableReader::_materialize_mapping_column; using TableReader::_materialize_map_mapping_column; }; +TEST(TableReaderTest, LastProjectionDetachesNestedMapWithoutCopyingStrings) { + auto keys = ColumnString::create(); + keys->insert_data(std::string(1UL << 20, 'k').data(), 1UL << 20); + auto values = ColumnString::create(); + values->insert_data(std::string(1UL << 20, 'v').data(), 1UL << 20); + const auto* original_value_bytes = values->get_chars().data(); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto map = ColumnMap::create(std::move(keys), std::move(values), std::move(offsets)); + auto null_map = ColumnUInt8::create(1, 0); + ColumnPtr source = ColumnNullable::create(std::move(map), std::move(null_map)); + + const auto string_type = std::make_shared(); + const auto map_type = make_nullable(std::make_shared(string_type, string_type)); + Block block; + block.insert({source, map_type, "large_map"}); + source.reset(); + + ColumnMapping mapping; + mapping.global_index = GlobalIndex(0); + mapping.table_column_name = "large_map"; + mapping.file_column_name = "large_map"; + mapping.file_local_id = 0; + mapping.file_type = map_type; + mapping.table_type = map_type; + mapping.is_trivial = true; + mapping.projection = + VExprContext::create_shared(VSlotRef::create_shared(0, 0, -1, map_type, "large_map")); + TableReaderMaterializeTestHelper reader; + ColumnPtr detached; + ASSERT_TRUE(reader._materialize_mapping_column(mapping, &block, 1, &detached, + /*take_projection_result=*/true) + .ok()); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + const auto& detached_nullable = assert_cast(*detached); + const auto& detached_map = assert_cast(detached_nullable.get_nested_column()); + const auto& detached_values = assert_cast(detached_map.get_values()); + EXPECT_EQ(detached_values.get_chars().data(), original_value_bytes); +} + VExprSPtr table_int32_sum_expr(int left_slot_id, int left_column_id, int right_slot_id, int right_column_id) { const auto int_type = std::make_shared(); @@ -491,6 +565,23 @@ void write_parquet_file(const std::string& file_path, int32_t id, const std::str builder.build())); } +void write_single_int_parquet_file(const std::string& file_path, const std::string& column_name, + int32_t value) { + auto schema = arrow::schema({arrow::field(column_name, arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + void write_struct_parquet_file(const std::string& file_path, int32_t id) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false)}); arrow::StructBuilder builder( @@ -1012,7 +1103,10 @@ struct FakeFileReaderState { bool stop_during_aggregate = false; bool stop_during_read = false; bool not_found_during_init = false; + int batch_count = 1; + int get_block_count = 0; std::shared_ptr last_request; + std::optional last_aggregate_request; std::shared_ptr condition_cache_ctx; std::shared_ptr io_ctx; }; @@ -1049,7 +1143,7 @@ class FakeFileReader final : public FileReader { RETURN_IF_ERROR(FileReader::open(std::move(request))); _state->last_request = _request; ++_state->open_count; - _returned_batch = false; + _returned_batches = 0; return Status::OK(); } @@ -1058,7 +1152,8 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); DORIS_CHECK(_request != nullptr); - if (_returned_batch) { + ++_state->get_block_count; + if (_returned_batches >= _state->batch_count) { *rows = 0; *eof = true; return Status::OK(); @@ -1105,9 +1200,9 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; } - _returned_batch = true; + ++_returned_batches; *rows = 2; - *eof = _state->eof_with_first_batch; + *eof = _state->eof_with_first_batch && _returned_batches >= _state->batch_count; if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit && _state->condition_cache_ctx->filter_result != nullptr && !_state->condition_cache_ctx->filter_result->empty()) { @@ -1128,6 +1223,7 @@ class FakeFileReader final : public FileReader { if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("fake reader only supports COUNT aggregate pushdown"); } + _state->last_aggregate_request = request; if (_state->stop_during_aggregate) { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; @@ -1162,7 +1258,7 @@ class FakeFileReader final : public FileReader { private: std::vector _schema; std::shared_ptr _state; - bool _returned_batch = false; + int _returned_batches = 0; }; class FakeTableReader final : public TableReader { @@ -1336,9 +1432,257 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_FALSE(predicate_executed); + EXPECT_TRUE(predicate_executed); EXPECT_FALSE(eos); + // The file was still opened, proving constant pruning did not jump over the unsafe predicate; + // the predicate is evaluated only after the resulting table row is materialized. EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ScannerOwnedUnsafePredicateIsPassedButNotExecutedByTableReader) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .table_reader_owned_conjunct_count = 0, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_EQ(reader.TEST_conjunct_count(), 1); + EXPECT_EQ(reader.TEST_table_reader_owned_conjunct_count(), 0); + EXPECT_FALSE(predicate_executed); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualExpressionStateSurvivesAcrossSplits) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + std::vector observed_invocations; + auto stateful_predicate = std::make_shared(&observed_invocations); + stateful_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, stateful_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + for (int split_index = 0; split_index < 2; ++split_index) { + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); + } + + EXPECT_EQ(observed_invocations, std::vector({0, 1})); +} + +TEST(TableReaderTest, AllFilteredResidualReturnsAfterOneMaterializedBatch) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_EQ(fake_state->get_block_count, 1); + EXPECT_EQ(fake_state->close_count, 0); + EXPECT_TRUE(reader.last_materialized_block_stats().has_materialized_input); + EXPECT_EQ(reader.last_materialized_block_stats().rows, 2); + EXPECT_GT(reader.last_materialized_block_stats().bytes, 0); + EXPECT_GT(reader.last_materialized_block_stats().allocated_bytes, 0); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, LateConjunctFiltersAlreadyOpenSplit) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_EQ(block.rows(), 2); + + bool predicate_executed = false; + auto late_predicate = std::make_shared(&predicate_executed); + late_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + ASSERT_TRUE( + reader.append_conjuncts({VExprContext::create_shared(std::move(late_predicate))}).ok()); + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->get_block_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("scanner"); + bool predicate_executed = false; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + ASSERT_NE(profile.get_counter("ResidualFilterTime"), nullptr); + EXPECT_GT(profile.get_counter("ResidualFilterTime")->value(), 0); ASSERT_TRUE(reader.close().ok()); } @@ -1389,9 +1733,11 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { EXPECT_EQ(fake_state->open_count, 1); ASSERT_NE(fake_state->last_request, nullptr); // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. - // The later predicate must stay on the scanner's row-level path instead of running inside the + // The later predicate must stay on the post-materialization path instead of running inside the // file reader before the unsafe conjunct. EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); ASSERT_TRUE(reader.close().ok()); } @@ -1511,6 +1857,7 @@ TEST(TableReaderTest, RefreshedConjunctDisablesTableLevelCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1553,6 +1900,7 @@ TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1570,6 +1918,10 @@ TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_EQ(fake_state->open_count, 1); EXPECT_EQ(block.rows(), 2); + ASSERT_NE(fake_state->last_request, nullptr); + // Aggregate pushdown is disabled while a runtime filter is pending, but COUNT(*) semantics do + // not change. The retained output slot remains a value-less placeholder during row fallback. + EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); ASSERT_TRUE(reader.close().ok()); } @@ -1637,24 +1989,314 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { set_name_identifiers(&projected_columns); RuntimeState state {TQueryOptions(), TQueryGlobals()}; - bool predicate_executed = false; + bool predicate_executed = false; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + // The slotless predicate cannot become a TableFilter or a file-reader conjunct, but its + // presence still prevents the fake aggregate count (3) from replacing the two physical rows. + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + // The two physical rows are then filtered at the table boundary, where slotless predicates are + // evaluated exactly even though they cannot be localized to a file column. + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->not_found_during_init = true; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("missing-fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.is()) << status; + ASSERT_TRUE(reader.abort_split().ok()); + EXPECT_EQ(fake_state->init_count, 1); + EXPECT_EQ(fake_state->close_count, 1); + + fake_state->not_found_during_init = false; + split_options.current_range.__set_path("existing-fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->init_count, 2); + EXPECT_EQ(fake_state->close_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + fake_state->io_ctx = io_ctx; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + EXPECT_EQ(file_reader_stats.read_rows, 3); + EXPECT_EQ(fake_state->close_count, 1); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + ASSERT_TRUE(fake_state->last_request != nullptr); + EXPECT_TRUE(fake_state->last_request->count_star_placeholder_columns.empty()); + ASSERT_TRUE(fake_state->last_aggregate_request.has_value()); + ASSERT_EQ(fake_state->last_aggregate_request->columns.size(), 1); + // A primitive COUNT(col) projection must reach the file reader just like a complex one. + EXPECT_EQ(fake_state->last_aggregate_request->columns[0].projection.local_id(), 0); +} + +TEST(TableReaderTest, PushDownCountEmitsAtMostOneRuntimeBatch) { + const auto nullable_bigint_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_bigint_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_bigint_type)); + set_name_identifiers(&projected_columns); + + TQueryOptions query_options; + query_options.__set_batch_size(2); + RuntimeState state {query_options, TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 5; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + bool eos = false; + for (const size_t expected_rows : {2, 2, 1}) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), expected_rows); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + } + + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->open_count, 1); + EXPECT_EQ(fake_state->close_count, 1); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PushDownCountStarIgnoresProjectedPlaceholderColumn) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "nullable_id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "nullable_id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + // COUNT(*) deliberately has no explicit count columns. The + // nullable_id projection is only the planner's scan placeholder. + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + ASSERT_TRUE(fake_state->last_request != nullptr); + ASSERT_EQ(fake_state->last_request->count_star_placeholder_columns.size(), 1); + EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); + ASSERT_TRUE(fake_state->last_aggregate_request.has_value()); + // Passing nullable_id here would implement COUNT(nullable_id) and reproduce the external ORC + // and Parquet failures where footer row counts were reduced by null values. + EXPECT_TRUE(fake_state->last_aggregate_request->columns.empty()); +} + +TEST(TableReaderTest, PushDownCountFallsBackWhenSemanticArgumentsAreAbsent) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + // Simulate an old FE: it can request COUNT pushdown but cannot + // serialize push_down_count_slot_ids. + .push_down_count_columns = std::nullopt, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + // The explicit aggregate_count=3 would be returned if absence were confused with COUNT(*). + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); + EXPECT_FALSE(reader.current_split_uses_metadata_count()); +} + +TEST(TableReaderTest, PushDownCountFallsBackForMultipleArguments) { + const auto nullable_int_type = make_nullable(std::make_shared()); + const auto nullable_string_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + file_schema.push_back(make_file_column(1, "name", nullable_string_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + projected_columns.push_back(make_table_column(1, "name", nullable_string_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto fake_state = std::make_shared(); fake_state->aggregate_count = 3; FakeTableReader reader(file_schema, fake_state); - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {prepared_conjunct( - &state, - std::make_shared( - &predicate_executed))}, - .format = FileFormat::PARQUET, - .scan_params = nullptr, - .io_ctx = nullptr, - .runtime_state = &state, - .scanner_profile = nullptr, - .push_down_agg_type = TPushAggOp::type::COUNT, - }) - .ok()); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0), GlobalIndex(1)}, + }) + .ok()); SplitReadOptions split_options; split_options.current_range.__set_path("fake-table-reader-input"); @@ -1663,25 +2305,27 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - // The slotless predicate cannot become a TableFilter or a file-reader conjunct, but its - // presence still prevents the fake aggregate count (3) from replacing the two physical rows. - ASSERT_NE(fake_state->last_request, nullptr); - EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_FALSE(eos); EXPECT_EQ(block.rows(), 2); - EXPECT_TRUE(predicate_executed); - ASSERT_TRUE(reader.close().ok()); + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); + EXPECT_FALSE(reader.current_split_uses_metadata_count()); } -TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { +TEST(TableReaderTest, PushDownCountFallsBackForNullableToRequiredMapping) { + const auto nullable_int_type = make_nullable(std::make_shared()); std::vector file_schema; - file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + std::vector projected_columns; projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + // make_table_column models the usual nullable external-table descriptor. Override it here to + // reproduce an evolved table contract that declares the mapped physical column required. + projected_columns[0].type = std::make_shared(); set_name_identifiers(&projected_columns); RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto fake_state = std::make_shared(); - fake_state->not_found_during_init = true; + fake_state->aggregate_count = 3; FakeTableReader reader(file_schema, fake_state); ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, @@ -1691,55 +2335,49 @@ TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { .io_ctx = nullptr, .runtime_state = &state, .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, }) .ok()); SplitReadOptions split_options; - split_options.current_range.__set_path("missing-fake-table-reader-input"); + split_options.current_range.__set_path("fake-table-reader-input"); ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); bool eos = false; - const auto status = reader.get_block(&block, &eos); - ASSERT_TRUE(status.is()) << status; - ASSERT_TRUE(reader.abort_split().ok()); - EXPECT_EQ(fake_state->init_count, 1); - EXPECT_EQ(fake_state->close_count, 1); - - fake_state->not_found_during_init = false; - split_options.current_range.__set_path("existing-fake-table-reader-input"); - ASSERT_TRUE(reader.prepare_split(split_options).ok()); - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_EQ(fake_state->init_count, 2); - EXPECT_EQ(fake_state->close_count, 2); - ASSERT_TRUE(reader.close().ok()); + // The normal scan rejects the nullable physical column because it cannot satisfy the required + // table contract. Footer COUNT would bypass that validation and incorrectly return 3. + EXPECT_FALSE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); } -TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { +TEST(TableReaderTest, PushDownCountFallsBackForCastMapping) { + const auto nullable_int_type = make_nullable(std::make_shared()); + const auto nullable_bigint_type = make_nullable(std::make_shared()); std::vector file_schema; - file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); std::vector projected_columns; - projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", nullable_bigint_type)); set_name_identifiers(&projected_columns); - io::FileReaderStats file_reader_stats; - auto io_ctx = std::make_shared(); - io_ctx->file_reader_stats = &file_reader_stats; - RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto fake_state = std::make_shared(); fake_state->aggregate_count = 3; - fake_state->io_ctx = io_ctx; FakeTableReader reader(file_schema, fake_state); ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, .conjuncts = {}, .format = FileFormat::PARQUET, .scan_params = nullptr, - .io_ctx = io_ctx, + .io_ctx = nullptr, .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, }) .ok()); @@ -1751,9 +2389,10 @@ TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_FALSE(eos); - EXPECT_EQ(block.rows(), 3); - EXPECT_EQ(file_reader_stats.read_rows, 3); - EXPECT_EQ(fake_state->close_count, 1); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(block.get_by_position(0).type->equals(*nullable_bigint_type)); + // INT->BIGINT is a non-trivial mapping, so COUNT must not skip the cast/materialization path. + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); } TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { @@ -1781,6 +2420,7 @@ TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1821,6 +2461,7 @@ TEST(TableReaderTest, DebugStringCoversReaderStateAndEnumNames) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1943,6 +2584,224 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); } +TEST(TableReaderTest, AnnotateProjectedColumnPrefersCurrentNameOverHistoricalAlias) { + auto renamed_field = external_schema_field("renamed_b", 1, {"b"}); + renamed_field.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_field = external_schema_field("b", 2); + current_field.field_ptr->__set_name_mapping({}); + current_field.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {renamed_field, current_field})}); + + ColumnDefinition projected = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &projected).ok()); + + EXPECT_EQ(projected.get_identifier_field_id(), 2); + ASSERT_TRUE(projected.has_name_mapping); + EXPECT_TRUE(projected.name_mapping.empty()); +} + +TEST(TableReaderTest, LegacyPlanRetainsOrderedCurrentNameAndAliasLookup) { + auto renamed_field = external_schema_field("renamed_b", 1, {"b"}); + renamed_field.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_field = external_schema_field("b", 2); + current_field.field_ptr->__set_name_mapping({}); + current_field.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_current_schema_id(100); + old_fe_scan_params.__set_history_schema_info( + {external_schema(100, {renamed_field, current_field})}); + + ColumnDefinition projected = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &projected).ok()); + + EXPECT_EQ(projected.get_identifier_field_id(), 1); + ASSERT_TRUE(projected.has_name_mapping); + EXPECT_EQ(projected.name_mapping, std::vector({"b"})); +} + +TEST(TableReaderTest, IcebergInitialDefaultMetadataOverridesGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_top_level_binary_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto int_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); + auto id_column = make_table_column(-1, "id", int_type); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + ASSERT_TRUE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(binary_column.initial_default_value_is_base64); + + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, IcebergLegacyPlanKeepsGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_legacy_binary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_current_schema_id(1); + old_fe_scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto varbinary_type = std::make_shared(16); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + + EXPECT_FALSE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(context.schema_column.has_value()); + EXPECT_FALSE(context.schema_column->initial_default_value.has_value()); + + auto nested_child = external_schema_field("added_binary", 2); + nested_child.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + nested_child.field_ptr->__set_initial_default_value_is_base64(true); + auto nested_field = external_struct_field("s", 10, {nested_child}); + TFileScanRangeParams old_fe_nested_params; + old_fe_nested_params.__set_current_schema_id(1); + old_fe_nested_params.__set_history_schema_info({external_schema(1, {nested_field})}); + auto struct_type = + std::make_shared(DataTypes {varbinary_type}, Strings {"added_binary"}); + auto struct_column = make_table_column(-1, "s", struct_type); + ProjectedColumnBuildContext nested_context {.scan_params = &old_fe_nested_params}; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &nested_context, &struct_column).ok()); + ASSERT_TRUE(nested_context.schema_column.has_value()); + ASSERT_EQ(nested_context.schema_column->children.size(), 1); + EXPECT_FALSE(nested_context.schema_column->children[0].initial_default_value.has_value()); + + auto id_column = make_table_column(-1, "id", std::make_shared()); + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader data_reader; + ASSERT_TRUE(data_reader + .init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &old_fe_scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(data_reader.prepare_split(build_split_options(file_path)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(data_reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + "Ej5FZ+ibEtOkVkJmFBdAAA=="); + ASSERT_TRUE(data_reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { + auto unmapped_field = external_schema_field("b", 2); + unmapped_field.field_ptr->__set_name_mapping({}); + unmapped_field.field_ptr->__set_name_mapping_is_authoritative(true); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {unmapped_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_TRUE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(TableReaderTest, LegacyFeEmptyNameMappingStillMatchesCurrentFileName) { + auto legacy_field = external_schema_field("b", 2); + legacy_field.field_ptr->__set_name_mapping({}); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {legacy_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_FALSE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); +} + TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { const auto string_type = std::make_shared(); const auto nullable_string_type = make_nullable(string_type); @@ -2467,6 +3326,7 @@ TEST(TableReaderTest, PushDownCountFromNewParquetReader) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -2508,6 +3368,7 @@ TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); auto split_options = build_split_options(file_path); @@ -2539,6 +3400,56 @@ TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, TableLevelCountRequiresExplicitCountStarArguments) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_table_count_arguments_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + const std::vector>> unsafe_count_arguments { + std::nullopt, std::vector {GlobalIndex(0)}}; + for (const auto& count_arguments : unsafe_count_arguments) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = count_arguments, + }) + .ok()); + auto split_options = build_split_options(file_path); + // Five metadata rows deliberately disagree with the three physical rows. nullopt models an + // old FE, while the non-empty vector models COUNT(id); neither may be treated as COUNT(*). + set_table_level_row_count(&split_options, 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + size_t total_rows = 0; + bool eos = false; + while (!eos) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + total_rows += block.rows(); + } + EXPECT_EQ(3, total_rows); + ASSERT_TRUE(reader.close().ok()); + } + + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, PushDownMinMaxFromNewParquetReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_minmax_test"; std::filesystem::remove_all(test_dir); @@ -3355,6 +4266,7 @@ TEST(TableReaderTest, PushDownCountOnlyUsesSelectedRowGroupInFileRange) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options_for_row_group_mid(file_path, 2)).ok()); @@ -3394,6 +4306,7 @@ TEST(TableReaderTest, PushDownCountFallsBackWithTableConjunct) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -3435,6 +4348,7 @@ TEST(TableReaderTest, PushDownCountFallsBackWithFilter) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -3661,6 +4575,59 @@ TEST(TableReaderTest, VExprPredicateSurvivesReopenSplit) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, RecomputesPredicateExecutionLayerForEverySplit) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_split_local_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto local_file = (test_dir / "local.parquet").string(); + const auto missing_file = (test_dir / "missing.parquet").string(); + write_single_int_parquet_file(local_file, "id", 3); + write_single_int_parquet_file(missing_file, "other", 9); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(local_file)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + expect_int32_column_values(*block.get_by_position(0).column, {3}); + ASSERT_TRUE(reader.close().ok()); + + // The same predicate cannot be file-local when this split omits `id`. It must be rebuilt as a + // table-level predicate over the materialized NULL instead of inheriting the previous split's + // file-local ownership or escaping without exact evaluation. + ASSERT_TRUE(reader.prepare_split(build_split_options(missing_file)).ok()); + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { const auto int_type = std::make_shared(); const std::vector projected_columns = { @@ -3735,6 +4702,7 @@ TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + EXPECT_TRUE(file_request.predicate_only_columns.empty()); ASSERT_EQ(file_request.local_positions.size(), 2); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); @@ -4030,7 +4998,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -4040,10 +5008,12 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { write_struct_parquet_file(file_path, 7); const auto int_type = std::make_shared(); - const auto string_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); auto id_child = make_table_column(0, "id", int_type); - auto missing_child = make_table_column(99, "missing_child", string_type); - auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + auto missing_child = make_table_column(99, "missing_child", varbinary_type); + missing_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + missing_child.initial_default_value_is_base64 = true; + auto struct_type = std::make_shared(DataTypes {int_type, varbinary_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); struct_column.children = {id_child, missing_child}; @@ -4077,7 +5047,10 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - expect_nullable_column_all_null(struct_result.get_column(1)); + const auto& defaults = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(1))); + EXPECT_EQ(defaults.get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); @@ -4246,49 +5219,6 @@ TEST(TableReaderTest, ProjectedPartitionColumnUsesSplitPartitionValue) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedNullPartitionColumnPreservesNull) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_table_reader_null_partition_value_test"; - std::filesystem::remove_all(test_dir); - std::filesystem::create_directories(test_dir); - - const auto file_path = (test_dir / "split.parquet").string(); - write_parquet_file(file_path, 1, "one"); - - std::vector projected_columns; - auto partition_column = make_table_column(1, "value", std::make_shared()); - partition_column.is_partition_key = true; - projected_columns.push_back(std::move(partition_column)); - - RuntimeState state {TQueryOptions(), TQueryGlobals()}; - set_name_identifiers(&projected_columns); - TableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = nullptr, - .io_ctx = nullptr, - .runtime_state = &state, - .scanner_profile = nullptr, - }) - .ok()); - - auto split_options = build_split_options(file_path); - split_options.partition_values.emplace("value", Field::create_field(Null())); - ASSERT_TRUE(reader.prepare_split(split_options).ok()); - - Block block = build_table_block(projected_columns); - bool eos = false; - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - ASSERT_FALSE(eos); - - expect_nullable_column_all_null(*block.get_by_position(0).column); - - ASSERT_TRUE(reader.close().ok()); - std::filesystem::remove_all(test_dir); -} - TEST(TableReaderTest, ConstantPartitionFilterSkipsSplitWhenFalse) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_constant_partition_filter_skip_test"; diff --git a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp index 60b6f37b8ceeba..64795dace19a14 100644 --- a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp +++ b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp @@ -89,6 +89,18 @@ TEST_F(AdaptiveBlockSizePredictorTest, NoHistoryReturnsMaxRows) { EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), expected_bpr); } +TEST_F(AdaptiveBlockSizePredictorTest, ExplicitMaterializedSampleUsesPreFilterShape) { + AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0); + + // Callers that filter a block before returning it can still report the rows and bytes that + // were actually materialized upstream. + pred.update(32, 32 * 4096); + + EXPECT_TRUE(pred.has_history_for_test()); + EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), 4096.0); + EXPECT_EQ(pred.predict_next_rows(), 2048); +} + // ── Test 2: EWMA convergence ────────────────────────────────────────────────── // When every update delivers the same sample, the EWMA stays exactly at that // value (0.9*v + 0.1*v == v for any v). diff --git a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql index 039ddd1d3690fb..d45b878d1ed29d 100644 --- a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql @@ -342,6 +342,14 @@ CREATE TABLE doris_test.extreme_test ) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE doris_test.distributed_type AS doris_test.type +ENGINE = Distributed(default, doris_test, type, rand()); + +CREATE MATERIALIZED VIEW doris_test.materialized_view_type +ENGINE = MergeTree +ORDER BY k1 +AS SELECT * FROM doris_test.type; + CREATE TABLE doris_test.extreme_test_multi_block ( id UInt64, @@ -392,4 +400,4 @@ CREATE TABLE doris_test.extreme_test_multi_block ipv6_col IPv6, ipv6_nullable Nullable(IPv6) ) ENGINE = MergeTree() -ORDER BY id; \ No newline at end of file +ORDER BY id; diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index b05ea798ee46b0..82a3ba03667753 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -99,14 +99,135 @@ format-specific checklist when reviewing Parquet or ORC. - Keep index construction, predicate translation, cache lookup, and virtual-column setup out of per-row and repeated batch paths unless the work is inherently row-local. Avoid repeated schema traversal, expression cloning, metadata parsing, allocation, and conversion. +- Keep Profile counters in the visible `FileScannerV2 -> TableReader -> FileReader -> IO` hierarchy. + Format-specific readers, such as `ParquetReader`, belong below `FileReader`; lifecycle, metadata, + index, predicate, decode, materialization, and physical I/O paths must all have timers at the layer + that owns the work. Flush recursively aggregated child-reader statistics at every batch boundary, + including empty-selection and error exits, so a slow in-progress scan is diagnosable before close. - Require format readers to populate the common `ReaderStatistics` accurately where applicable: filtered/read row groups, Bloom and min/max pruning, filtered group/page/lazy rows, read rows and bytes, metadata/footer/cache timing, page-index work, predicate time, dictionary rewrite, and Bloom read time. +- Reject dead or ambiguous counters. In particular, `FilteredBytes` counts compressed bytes of + projected physical chunks avoided by pruning, not every child in the Row Group; footer read, + footer parse, lazy page-index materialization, and page-index predicate evaluation need distinct + timers. Raw I/O counters stay under `IO` even when a format reader initiates the request. - Evaluate performance with representative format versions, writers, data ordering, predicate selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the optimization overhead and the avoided work; a low pruning ratio alone is not a defect. +## Parquet Native Decode Boundary + +- V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the + v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. +- Footer parsing, schema-ID assignment, and the cached native metadata tree must also be v2-owned. + Reusing a stable base file identity is allowed, but require a v2 cache type discriminator so + v1/v2 metadata objects can never be cross-cast. Production planning must not retain or rebuild an + Arrow `FileMetaData` tree from the serialized footer. +- Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. + Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, + builders, `DecodedColumnView`, or another decoded leaf batch. +- Verify physical/logical metadata is immutable per leaf reader and complete for signed integers, + decimal precision/scale, date/time/timestamp units and UTC adjustment, INT96, UUID, FLOAT16, and + fixed-width binary. Unsupported combinations return explicit errors before plausible output. +- Treat legacy Parquet `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` converted types as UTC-adjusted. + Do not give them the local/unspecified semantics of an unannotated INT64 timestamp; data decode, + statistics conversion, and min/max pruning must use the same timezone rule. +- Route plain, dictionary, and decoded timestamp inputs through one checked conversion contract. + Validate INT96 nanos-of-day before widened Julian-day arithmetic, reject unit scaling overflow, + and enforce Doris year 0001-9999 before materialization. Conversion failures must follow the same + strict/non-strict and dictionary-ID propagation rules as other direct types. +- Verify schema-change routing separately from physical decode. The reader must emit the projected + file type, while `ColumnMapper`/`TableReader` perform file-to-table casts after file predicates. + Any different requested type at the native reader boundary must fail as an invariant violation; + do not add a reader-local conversion column or decoder-facing conversion ABI. +- Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload + ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate + every index before access, and invalidate cached dictionary state at Row Group/file/type changes. + Dictionary-entry predicate evaluation and later row-value flattening must reuse that same typed + generation rather than serializing, parsing, or converting the dictionary twice. +- Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded + values without allocating output; null runs must append defaults without advancing payload. +- For predicate-only fixed-width PLAIN primitives, allow direct comparison only after proving the + whole Column Chunk uses compatible PLAIN value pages and every Expr advertises raw fixed-value + evaluation with identical Doris comparison semantics, including NaN ordering. The fallback + decision must precede definition-level consumption. Disable the direct path when a residual or + delete conjunct still references the hidden slot, because it needs the materialized payload. + Verify sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed + comparisons, mixed-encoding fallback, residual slot reuse, and a stable row-shaped placeholder. +- For a filtered scalar page fragment, require one SerDe entry and one batch-level selected-decode + dispatch. Nullable selections must first map logical rows to selected non-NULL physical ranges, + decode those ranges once, and restore NULL slots in place; falling back per NULL run is a review + blocker for a scalar destination that supports in-place expansion. Selection ranges belong to + persistent reader scratch; per-range virtual SerDe/decoder calls in the hot path are a review + blocker. Fixed PLAIN should + bulk-gather spans, BYTE_ARRAY PLAIN should scan lengths once, dictionary decode should validate + every ID before gathering selected IDs, and stateful encodings should batch-decode/reconstruct and + compact. Any remaining NULL-interleaving fallback must preserve logical output order without a + decoded intermediate column and be counted by `HybridSelectionNullFallbackBatches`. +- Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, + null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct + without materializing an intermediate complex column. +- For MAP, the non-null key leaf owns the outer entry shape. Validate the materialized key/value + entry counts, but do not compare their raw repetition vectors: a nested value legitimately has + deeper levels. For STRUCT, compare each sibling only at the current parent boundary and ignore + repetition owned by a deeper child collection. +- Require a bounded high-water policy for persistent definition/repetition, null, selection, + conversion, dictionary-index, and decoder-owned scratch. Distinguish active bytes from retained + capacity: never release an oversized buffer while the current batch still needs it, and require + three ordinary/idle batches before releasing capacity above the high-water limit. Test both + outlier release and steady-state reuse so the policy does not create allocation thrash. +- Review all decoder read and skip paths as equally exposed corruption boundaries. Check requested + counts against remaining/declared values before pointer arithmetic or narrowing; use checked + addition/multiplication for byte extents; bound BYTE_ARRAY dictionary entry counts and IDs before + allocation/indexing; and require DELTA_BYTE_ARRAY prefixes to fit the previous reconstructed + value. BOOLEAN RLE and DELTA skip paths must consume bounded chunks and fail on short streams. +- Validate every decoded definition/repetition level against the schema maximum in batch, run, and + single-value cursor APIs. Page value counts must not drive eager nested scratch allocation; + reserve from the requested parent-row frontier and cover tiny-payload huge-count Page V1/V2 files. +- Before Snappy decompression, inspect the encoded uncompressed length and validate destination + capacity. Page V1, Page V2, and dictionary pages must produce exactly the declared decoded size; + an UNCOMPRESSED dictionary page must also declare equal compressed and uncompressed sizes. +- For a STRUCT whose projected children are all missing after schema evolution, require a + levels-only physical reference leaf. It must advance and validate encoded payload cursors while + deriving the synthetic child count, without constructing a discarded string/complex column. +- `CountColumnReader` must use the native levels-only reader and must not decode payload or call + Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, + value decode, SerDe materialization, hybrid selection batches/ranges/NULL fallback, + filtered-value skips, and page fragmentation. +- Validate every signed Column Chunk offset/length before converting it to `size_t`. The dictionary + offset is usable only when it is non-negative and precedes the data offset; the complete range + must fit the file. Apply the PARQUET-816 tail padding only to affected parquet-mr versions, cap it + at 100 bytes, and keep it inside the file. Scalar and levels-only COUNT readers share this helper. +- Treat OffsetIndex as one optional, all-or-nothing navigation structure. Require first row zero, + the first physical location to equal the owning ColumnMetaData `data_page_offset`, strictly + increasing row ordinals and physical offsets, positive sizes, non-overlapping page ranges, and + containment in the owning Column Chunk. Discard a malformed index before selecting the indexed + reader. +- Page iteration skips `INDEX_PAGE` and unknown auxiliary pages before initializing a data decoder. + Dictionary pages retain their special first-page handling; a later dictionary page is corrupt. +- Derive writer workarounds once from `created_by` and pass them through scalar, nested, page-cache, + and COUNT paths. Pre-Arrow-3 parquet-cpp Data Page V2 payloads remain compressed despite the + historical `is_compressed=false` flag. +- Preserve nullable conversion semantics in direct native materialization. Numeric, DATE, + DATETIME, TIME, and DECIMAL failures insert a default nested value and mark the corresponding NULL + only in non-strict mode; strict or non-nullable reads return the error. Dictionary failures follow + the selected dictionary IDs to output rows. +- Keep Parquet decimals in a source-width or wider intermediate until exact scaling, target + precision, and overflow checks succeed. Scale-down with a non-zero remainder is a conversion + failure; plain and dictionary integer/binary paths must narrow only afterward. +- For cold small-file tests, separate footer I/O/Thrift parse from native schema and index planning. + V2 must not retain serialized footer bytes after the native metadata tree is initialized; v1 opens + remain independent. +- For HTTP Parquet objects at or below `in_memory_file_size`, v2 stages the complete object from + byte zero before native page access. Verify both cold and footer-cache-hit scans: some Range + servers accept the capability probe but return HTTP 200 for a near-EOF overlong range, so warm + scans must not depend on the footer read having populated an incidental transport buffer. +- Identical fixed-width POD values append with one bulk copy. FIXED_LEN_BYTE_ARRAY strings copy the + dense byte span once and synthesize offsets; validate this execution contract without flaky + wall-clock assertions. + ## Parquet Multi-Level Filtering - Use [FileScannerV2 Parquet Scan Design](file-scanner-v2-parquet-scan-design.md) as the detailed @@ -129,12 +250,37 @@ format-specific checklist when reviewing Parquet or ORC. - Verify lazy materialization avoids reading and decoding non-predicate columns for rejected rows while advancing all readers correctly. Predicate columns should be read/prefetched first; output prefetch should wait for survivors when filtering is active. +- For safe staged single-column predicates, review the observed cost/rejection ordering and its + cold-start behavior. Reordering is allowed only after every candidate has a sample; prefetch may + stop at a low-probability reach prefix, while output-column prefetch may start early only after a + learned high survival ratio. Static conjunct schedules and position maps belong to the scanner + lifecycle; after eight warm-up samples, per-predicate clocks should be sampled only periodically. + Cache the batch SelectionVector's dense bitmap by generation so a wide lazy projection does not + rebuild the same O(batch) filter for every column. +- Single-column predicate rounds may keep previously read columns in their original row mappings. + Require one alignment compaction before multi-column, delete, or output boundaries, and expose + its time, bytes, and count instead of charging repeated movement invisibly to predicate time. + Reused compaction masks must be fully cleared when coordinate-space sizes shrink. +- PLAIN BYTE_ARRAY must not parse lengths in both decoder and destination column. Review the direct + payload-offset/cumulative-offset contract, uint32 overflow checks, surviving-span coverage, and + the legacy consumer fallback for non-string logical types. +- Production PageIndex planning must consume native Compact Thrift ColumnIndex/OffsetIndex objects. + Coalesce adjacent serialized index ranges and transfer validated OffsetIndexes into execution so + the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; a production Arrow + metadata adapter or rebuilt `FileMetaData` tree is a review blocker. +- Fixed-width conversion fast paths may remove row branches only when the source domain provably + fits the target domain. Narrowing, timestamp, decimal scaling, strict rollback, and non-strict + NULL marking must retain corrupt-value tests. +- Recursive reader-profile publication and retained-scratch inspection should be amortized, but Row + Group reset/EOF/close must force the final profile delta before reader destruction. +- Accumulated lazy-column skips must not allocate one dense byte per rejected prefix row. Require a + fixed chunk bound (including multiple lazy columns) or a level-only/all-filtered cursor contract. - Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. - Require counters for Statistics/Dictionary/Bloom pruning, Page Index selected ranges and skipped - rows/pages, raw and filtered rows, dictionary-row filtering, lazy-read savings, cache sources, and - remote I/O. + rows/pages, raw and filtered rows, dictionary-row filtering, PLAIN direct-predicate batches/rows, + lazy-read savings, cache sources, and remote I/O. - Differential tests must cover absent/invalid statistics, missing or partial Page Index, mixed dictionary/plain encoding, Bloom false positives, NULL/NaN/type conversion, cross-Page batches, nested/repeated columns, multiple Row Groups/Splits, and all/none filtered. diff --git a/docs/file-scanner-v2-design.md b/docs/file-scanner-v2-design.md index 66b96de1204d14..b2fe43ae76f6b0 100644 --- a/docs/file-scanner-v2-design.md +++ b/docs/file-scanner-v2-design.md @@ -282,6 +282,23 @@ Scan optimization remains maintainable only when costs are visible, sources are and failure semantics are explicit. V2 provides three complementary views: Query Profile, query resource context, and global metrics. +Query Profile uses one stable ownership tree: + +```text +FileScannerV2 +└── TableReader + └── FileReader + ├── format-specific reader (ParquetReader, OrcReader, ...) + └── IO +``` + +Scanner lifecycle and Split scheduling are charged to `FileScannerV2`; table-schema restoration, +delete handling, and reader lifecycle are charged to `TableReader`; metadata, index, decode, and +materialization are charged to `FileReader` and its format subtree; physical reads, bytes, cache +waits, and remote/local attribution remain under `IO`. Every executable lifecycle path needs a +timer, and cumulative child-reader statistics must be published at batch boundaries as well as +close, so an active slow query never presents an unexplained timing gap. + ```mermaid flowchart LR R[FileReader and FileCache Raw Statistics] --> P[Query Profile] diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 63f63819670dd2..70c041a89ccd48 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -25,8 +25,9 @@ predicate columns for surviving ranges, and finally defer output-column reads un - **Layered caches:** File-block cache, Parquet page cache, condition-result cache, and merged small I/O solve different problems and are not interchangeable. -**Scope:** This document focuses on the FileScannerV2 Parquet Reader design and core pipeline. It -does not cover Arrow decoder internals, complex-type reconstruction, or expression implementation. +**Scope:** This document covers the FileScannerV2 Parquet Reader pipeline, the native page/decode +kernel, selection-aware materialization, and complex-type reconstruction contracts. Expression +implementation details remain outside its scope. ## 2. Overall Architecture @@ -40,9 +41,10 @@ flowchart TB B --> C[TableReader
Schema Mapping, Partition/Default Values, Predicate Localization] C --> D[ParquetReader
Footer/Schema and Row Group Scan Planning] D --> E[ParquetScanScheduler
Row Group Lifecycle and Batch Reads] - E --> F[ParquetColumnReader
Page Skipping, Decompression, Decoding, Materialization] - F --> G[ParquetFileContext / Arrow RandomAccessFile
Page Cache, MergeRange, Prefetch] - G --> H[Doris FileReader / FileCache / Remote FS] + E --> F[ParquetColumnReader
Persistent Column State and Complex Reconstruction] + F --> G[Native ColumnChunk / Page / Encoding Decoders
Selection-aware Doris Materialization] + G --> H[ParquetFileContext / Stream Adapter
Page Cache, MergeRange, Prefetch] + H --> I[Doris FileReader / FileCache / Remote FS] ``` | Layer | Core responsibilities | Responsibilities intentionally excluded | @@ -58,6 +60,64 @@ flowchart TB > layer can use footer, page index, dictionary, and other format knowledge while upper layers retain > uniform scan semantics. +### 2.1 Native Reader Target and Migration State + +The target execution path does not use Arrow builders or `ReadRecords` to decode data pages. Doris +owns the Column Chunk, page, level, encoding, selection, conversion, and materialization state, and +writes directly into Doris columns. This follows the same broad separation used by DuckDB: metadata +planning is distinct from a persistent per-column reader, and page/encoding decoders expose narrow +cursor-based contracts rather than an Arrow array as an intermediate result. + +V1 remains the differential baseline, but v2 owns an independent reader implementation. Its page +and encoding algorithms started from proven Doris behavior and are maintained under the v2 tree; +the production v2 path never instantiates the v1 `ParquetColumnReader`: + +| Stage | State and boundary | +| --- | --- | +| Native encoding kernel | V2 owns the Column Chunk, page, level, and encoding decoders under `be/src/format_v2/parquet/reader/native/`. Decoders expose raw fixed/binary spans, validated dictionary indices, and skip operations; they never write Doris columns. | +| Logical materialization | `DataTypeSerDe` interprets Parquet physical/logical metadata and writes decoded spans directly into the final Doris column. Typed dictionaries and scratch are persistent per leaf reader. | +| Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | +| Complex reconstruction | Choose a Dremel shape-owning leaf per requested parent-row range; derive parent offsets/nulls once from it and keep sibling leaf streams aligned to the same parent rows. | +| Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | +| Compatibility removal | Production v2 has no Arrow data-read or metadata adapter and never falls back to the v1 reader; unsupported combinations return explicit errors. Arrow remains only in test oracles and fixture writers. | + +Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or +builders. V2 parses and owns the Thrift footer, schema parser, field-ID assignment, physical schema, +statistics, dictionary metadata, bloom filters, and page indexes. Production planning never builds +an Arrow metadata tree; Arrow is limited to fixture writers and differential test oracles. + +All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness +and performance control. Compatibility is demonstrated through differential tests and the explicit +type/encoding matrix, not through a runtime dependency on v1 code. + +### 2.2 Native Interface Ownership + +```mermaid +flowchart LR + A[Native Footer / Physical Schema] --> B[RowGroupReadPlan] + B --> C[Persistent ParquetColumnReader] + C --> D[Persistent ColumnChunkReader] + D --> E[Page Reader + Level Decoders] + E --> F[Encoding Decoder] + F --> G[ColumnSelectVector] + G --> H[Doris MutableColumn] + C --> I[Shared Complex Level Plan] + I --> G +``` + +- **Physical schema contract:** Immutable physical type, fixed length, maximum definition and + repetition levels, and repeated-parent definition threshold. It owns no Arrow descriptor and + borrows no temporary table-schema object. +- **Persistent column state:** Page/decompress buffers, dictionary decoder, SerDe/conversion state, + null maps, selection ranges, binary-value references, and builder capacity live with the column + reader and are logically reset rather than recreated for each batch. +- **Decoder contract:** Consume a known number of logical level entries and encoded payload values, + then materialize only selected rows. Decoders do not decide table projection, predicate meaning, + or parent complex offsets. +- **Complex plan contract:** The shape-owning leaf's definition/repetition levels are parsed once + into parent-row boundaries and child-presence/null decisions. Sibling physical streams consume + the same parent-row range and validate their counts so offsets and null maps cannot drift. + ## 3. From File Open to Scan Plan After a reader receives a Split, it opens the file and builds the scan plan. This phase determines @@ -75,7 +135,7 @@ sequenceDiagram FS->>TR: prepare/open split TR->>TR: Map schema and localize predicates TR->>PR: FileScanRequest - PR->>FC: Open FileReader + PR->>FC: Open FileReader / native stream adapter FC->>META: Read footer and schema META-->>PR: Row Group / Column Chunk metadata PR->>PLAN: Candidate Row Groups and local predicates @@ -94,8 +154,17 @@ sequenceDiagram delete conjuncts, and local column-position mappings. - **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` produced by page-index pruning, and the `page_skip_plan` for each leaf column. -- **ParquetFileContext:** Adapts Doris FileReader to Arrow RandomAccessFile and owns Page Cache, - FileCache prefetch, and MergeRange routing. +- **ParquetFileContext:** Adapts Doris FileReader to the native metadata/data stream interface and + owns Page Cache, FileCache prefetch, and MergeRange routing. Its immutable footer cache entry owns + the V2 `NativeFieldDescriptor` tree directly; repeated V2 opens neither re-read the footer nor + serialize and parse another metadata representation. + +For every selected Row Group, native dictionary/index probes finish on the cached metadata tree first. The +scheduler then computes projected physical Column Chunk ranges and installs one shared native +`MergeRangeFileReader` when their average size is below v1's small-I/O threshold. All predicate and +lazy output readers share that wrapper; their internal `BufferedFileStreamReader` prefetch is +disabled in this mode, avoiding duplicate buffers. Large chunks and in-memory files use the base +reader, while remote FileCache prefetch remains the non-MergeRange path. > Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the > candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data @@ -279,6 +348,258 @@ flowchart LR > output columns must decode and copy. This is the main benefit of lazy materialization in a > columnar format. +### 7.1 Selection and Cursor Contract + +The native decoder receives a sorted selection over logical rows. Logical positions include nulls; +they are not offsets into the encoded non-null payload. Definition levels and selection are merged +as two ordered streams into four run types: + +| Run | Consume logical level entries | Consume encoded payload | Append output | +| --- | --- | --- | --- | +| Selected non-null (`CONTENT`) | Yes | Yes | Value | +| Selected null (`NULL_DATA`) | Yes | No | Default plus null-map bit | +| Filtered non-null (`FILTERED_CONTENT`) | Yes | Yes or decoder-native skip | No | +| Filtered null (`FILTERED_NULL`) | Yes | No | No | + +The contract keeps three counts explicit: logical level entries consumed, encoded non-null values +consumed, and Doris output values appended. The page reader may cross page boundaries while +satisfying one batch, but all three counts must be aligned when it returns. A dense identity +selection, an empty selection, and an arbitrary fragmented selection use the same decoder API. +Selection inputs are borrowed only for the duration of the decode call. + +For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes +definition-level runs and builds the four-way selection plan in persistent scratch whose capacity +survives adaptive batch changes. The plan is normalized to sorted physical non-NULL ranges and +enters `DataTypeSerDe` and the encoding decoder once. If selected NULLs are interleaved, the reader +separately records their logical positions, decodes the compact physical payload, and expands the +final scalar column backwards in place. This is the hybrid selection path: it keeps the dense +direct-materialization path, but moves sparse range traversal inside the concrete decoder instead +of repeatedly constructing a SerDe consumer for every selected or NULL run. + +The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk +copies; PLAIN BYTE_ARRAY scans every length once and publishes compact source offsets, cumulative +output offsets, and coalesced surviving spans directly to the SerDe; dictionary encoding validates +IDs in bounded batches and either gathers repeated/literal runs directly or builds one compact +selected-ID batch when a sparse lookup would exceed the L2 cache; BOOLEAN, +DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected +values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate +Decoder/SerDe ownership boundary. A filtered non-null value always advances and validates encoding +state even when it is not copied, preventing the next batch from decoding shifted values. + +Scalar destinations with in-place expansion no longer use the per-run NULL fallback. The fallback +remains only for unsupported destination shapes and is visible through +`HybridSelectionNullFallbackBatches`; it is a correctness boundary, not an Arrow fallback. + +### 7.2 Page and Encoding Kernel + +The native page reader parses Page V1 and Page V2 headers, obtains level streams and payload bytes, +decompresses only the required region, and installs a decoder selected from physical type plus data +encoding. Page V1 and V2 differ in where levels are stored and which bytes are compressed; after +that parsing step both feed the same level and value-decoder contracts. + +| Encoding family | Native responsibility | +| --- | --- | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and variable strings reuse decoder-produced offsets without a `StringRef[]` staging pass | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, validate IDs in bounded batches, fuse RLE runs with direct gather for cache-resident dictionaries, and use one compact selected-ID gather for large sparse dictionaries | +| RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | +| DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | +| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, then retain selected references only | +| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value state, then retain selected references only | +| BYTE_STREAM_SPLIT | Reassemble selected primitive lane ranges directly into one compact batch without an Arrow intermediate | + +Unsupported physical-type/encoding combinations return an explicit error. They never fall back to +Arrow and never produce a plausible result through a decoder selected only by logical Doris type. +Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated +payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. + +Read and skip are the same trust boundary. Every decoder validates the requested count against its +declared/remaining values before pointer advancement, allocation, multiplication, or integer +narrowing. BYTE_ARRAY dictionaries bound the entry count by the available four-byte length prefixes +before reserving storage and validate every decoded ID. DELTA_BYTE_ARRAY requires each prefix to fit +the preceding reconstructed value and checks the reconstructed and aggregate byte lengths. BOOLEAN +RLE and DELTA skip paths operate in bounded chunks and reject short streams instead of advancing a +partially consumed cursor as if the request succeeded. + +Level bit width is only a storage bound: for a schema maximum of 2, two bits can still encode the +invalid value 3. Batch, run, and single-value level cursors therefore reject every decoded value +above the schema maximum. Nested page traversal reserves only the requested parent-row frontier; +an untrusted Page V1/V2 `num_values` cannot trigger an eager page-sized allocation before the level +payload proves those entries exist. + +Compression has an exact-size contract. Snappy's encoded uncompressed length is checked against the +destination capacity before decompression; Page V1, Page V2, and dictionary decode must then produce +exactly the size declared in the page header. For the UNCOMPRESSED codec, dictionary compressed and +uncompressed sizes must be equal. These rules turn malformed size metadata into corruption instead +of a buffer overrun, silent truncation, or plausible shifted values. + +Before a stream is created, scalar and levels-only readers call one signed, file-bounded Column +Chunk range validator. Writer compatibility derived from `created_by` may add at most 100 bytes of +file-bounded PARQUET-816 padding, or override the pre-Arrow-3 Data Page V2 compressed flag. Page +iteration ignores auxiliary `INDEX_PAGE`/unknown pages without consuming a logical data-page +ordinal. OffsetIndex is published only when its rows and non-overlapping physical page ranges are +strictly increasing, its first location equals the owning `data_page_offset`, and every location +remains inside that validated Column Chunk. The metadata anchor rejects uniformly shifted indexes +that row monotonicity alone cannot detect; otherwise sequential traversal preserves correctness. + +### 7.3 Direct Materialization and Scratch Reuse + +Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected +`DataTypeSerDe` consumes those spans and writes directly into the Doris mutable column. Fixed-width +types append contiguous runs. PLAIN BYTE_ARRAY publishes payload offsets, cumulative destination +offsets, and coalesced survivor spans so string columns perform larger range copies without a +`StringRef[]` staging array. Other binary encodings may use persistent references that remain valid +only while the page or dictionary buffer is pinned by the persistent leaf reader. + +The direct path materializes the projected file type. Table-schema changes, including numeric +widening, decimal precision/scale changes, and string/date conversions, are represented by +`ColumnMapper` and executed by `TableReader` after file-local predicates finish. The native reader +rejects a different requested type because accepting it would move schema evolution into physical +decode and could change predicate semantics. No intermediate physical-value batch is introduced. + +`DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded +physical values and is useful to generic format conversion code, but routing every Parquet value +through it would recreate an intermediate materialization layer. The v2 native path does not use it: +ColumnReader handles levels/selection, Decoder parses encoding streams, and DataTypeSerDe performs +physical/logical conversion while appending to the final column. + +Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian +two's-complement values with correct sign extension, and apply precision/scale conversion exactly +once. Decimal integer and binary inputs stay in a 256-bit intermediate through exact scale-down, +scale-up overflow, and target-precision checks; discarded non-zero digits are conversion failures, +and narrowing happens only after success. Date, timestamp, INT96, unsigned annotations, +CHAR/VARCHAR, and timezone conversions retain +the same semantic checks as the general conversion path. A fast path is enabled only when those +checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and +`TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct +local/unspecified semantics; data decode and statistics pruning share this interpretation. +Timestamp conversion validates millisecond scaling before multiplication, validates INT96 +nanos-of-day before widened Julian-day arithmetic, and rejects values outside Doris years +0001-9999. Plain, dictionary, and decoded-value inputs share these bounds. + +Direct conversion also preserves load-mode error semantics. A nullable target in non-strict mode +stores the nested default and marks the exact output row NULL for numeric, date/time, timestamp, and +decimal conversion failures. Strict mode and non-nullable targets return the error. Typed +dictionary materialization records failing dictionary entries once and propagates those failures +through decoded dictionary IDs, so dictionary and plain pages have identical behavior. + +The typed dictionary is materialized through the logical SerDe once per decoder dictionary +generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving +row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation +change invalidates it, and every ID is checked before access. Cache-resident or dense reads stream +validated RLE runs directly into the destination column without a page-sized ID vector. Sparse +reads whose dictionary exceeds L2 retain only selected IDs, then perform one gather; malformed late +IDs roll back the destination to its pre-batch size. + +The persistent leaf reader owns reusable conversion objects, null map, selection ranges, +definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris +column capacity. Logical sizes are reset at batch boundaries and normal-size capacity is retained. +After the top-level complex reader has consumed the level plan, retained capacity above the 4 MiB +high-water limit becomes eligible for release. The reader accounts separately for active and +retained bytes, including decoder-owned buffers: active oversized scratch is never released, and +eligible capacity is released only after three consecutive ordinary/idle batches. This hysteresis +prevents a repeated-value outlier from being pinned for the Row Group without turning a legitimate +large steady-state batch into allocate/free thrash. The native path does not create an Arrow +builder or Arrow array. + +### 7.4 Complex Types and Parent Shape Plans + +Repeated Parquet leaves cannot interpret a requested parent-row count as a leaf-value count. One +parent row may contain zero, one, or many level entries, and its final entry can reside on the next +page. The complex reader therefore builds a parent shape plan from one owning leaf with: + +- parent-row start/end boundaries derived from repetition levels; +- ancestor-null, collection-null, empty-collection, element-null, and present-value decisions from + definition thresholds; +- child payload positions and selected parent rows; +- cross-page continuation state for an unfinished parent row. + +ARRAY and MAP offsets and null maps are derived from that plan. Parquet stores separate level +streams for separate physical leaves, so sibling readers still consume their own streams; they must +advance over the same parent-row range and validate their payload counts instead of redefining the +parent shape. STRUCT uses a representative present leaf for its null map and parent count. MAP uses +the key leaf as the entry-shape owner, requires the materialized key and value columns to match that +outer entry count, and validates Parquet's non-null key requirement rather than repairing it. Raw +key/value repetition vectors are intentionally not compared because a nested MAP value owns +additional repetition levels. STRUCT siblings similarly normalize repetition to the current parent +boundary; deeper child collection repetition cannot redefine or invalidate the sibling shape. + +Level scratch grows with level entries that were actually decoded, while its initial reservation is +bounded by the requested parent-row frontier rather than the page header's untrusted value count. +Long repeated rows can still grow incrementally beyond that frontier, and long null/non-null runs +are split into representable internal runs without introducing a new row boundary. Tests cover null +ancestors, empty collections, null elements/values, nested +STRUCT-in-ARRAY and ARRAY-in-STRUCT shapes, sibling page misalignment, and rows spanning pages and +batches. + +#### Complex-reader interface and materialization cost + +The original v2 prototype placed an Arrow-decoded leaf batch and separate load/build/consume phases +between encoded pages and Doris columns. That design required implicit phase ordering, duplicated +level traversal in ARRAY/MAP/STRUCT wrappers, and retained decoded binary payload even when a caller +needed only nullability. The prototype hierarchy and its batch container have been removed. + +The production boundary is now the same compact cursor contract used for scalar columns: +`NativeColumnReader::read/select/skip`. Its persistent native reader owns page, decoder, level, +conversion, and complex-column state and materializes the complete result directly into the caller's +Doris column. No Arrow value reader, intermediate decoded leaf container, temporary nested Doris +column, or public load-before-build protocol participates in predicate or output scans. + +Internally, complex decoding still has to solve the parent-shape problem. For example, levels +representing `[["a", "b"], NULL, []]` produce entry counts `[2, 0, 0]`, parent nulls `[0, 1, 0]`, +and string payload ordinals `[0, 1]`. MAP uses the key leaf as the entry-shape owner and validates +the value leaf against it; STRUCT children advance in parent-row lockstep. This state belongs behind +the native reader boundary rather than in caller-visible phases. + +When schema evolution makes every projected STRUCT child missing, the native reader retains one +physical reference leaf solely for shape. Its levels-only interface advances definition, +repetition, and encoded payload cursors (including dictionary-index validation), counts STRUCT +instances from the parent thresholds, and discards the payload without allocating a temporary +string or nested Doris column. + +`CountColumnReader` selects one representative leaf (the key for MAP) and uses the v2 native +`LevelReader` to consume definition/repetition levels without decoding payload values. It exposes +neither decoded values nor the ordinary scan-reader API, so COUNT pushdown cannot become a value +fallback and large complex payloads never enter aggregate state. + +Doris v1 remains the behavior/performance baseline: `read_column_data()` owns physical decode and +the collection reader consumes persistent level buffers. DuckDB provides the same useful design +principle through its single `Read(input, vector)` boundary and reusable child vectors. See DuckDB's +official [LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), +[STRUCT reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/struct_column_reader.cpp), +and [base column reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/column_reader.cpp). + +### 7.5 Index Coordinate Domains and Composition + +Index correctness depends on keeping its coordinate systems separate: + +| Identity | Scope | Must not be confused with | +| --- | --- | --- | +| Table/local column ID | Current file request and Doris block | Physical Parquet leaf ordinal | +| Physical leaf-column ID | Footer Row Group Column Chunk array | Logical parent STRUCT/ARRAY/MAP ordinal | +| Row Group ID | File metadata | Split ordinal or batch ordinal | +| Data-page ordinal | One Column Chunk, excluding dictionary pages | PageHeader sequence including dictionary page | +| OffsetIndex row ordinal | Logical row inside a Row Group | Encoded non-null value position | +| Selection index | Logical row inside the current batch, nulls included | Dictionary ID or compact output position | +| Dictionary-entry ID | One Column Chunk dictionary | Row ordinal, global dictionary ID, or the next Row Group's dictionary | + +Row Group statistics, dictionary pruning, Bloom, ColumnIndex/OffsetIndex, page skip plans, cache +ranges, row selection, and lazy predicate/output readers are composed in that order. Each stage may +only remove candidates already expressed in the same Row Group logical-row domain. Page ordinals +from ColumnIndex and OffsetIndex are validated together before they become `selected_ranges` and a +per-leaf physical page-skip plan. + +Dictionary row filtering starts only when metadata proves every data page in the Column Chunk uses +a dictionary encoding. The predicate is evaluated against the current dictionary to produce an +entry bitmap; decoded IDs are checked against both dictionary length and bitmap length. A mixed +dictionary/plain transition falls back before consuming data. Once selected dictionary reading has +advanced a page cursor, loss of dictionary output is corruption rather than a retry through another +path with shifted state. + +Missing optional indexes or unsupported predicate/type combinations retain rows. Malformed +offsets, inconsistent page counts, out-of-range dictionary IDs, overlapping/unsorted invalid ranges, +or an impossible cursor relationship are reported as corruption. Cache hits and misses do not +change any page, level, value, or dictionary cursor. + ## 8. Supported Indexes and Their Boundaries V2 uses native Parquet metadata and encoding information. It does not construct Doris-internal @@ -315,12 +636,12 @@ flowchart TD ## 9. Cache and I/O Optimization -Parquet V2 has four complementary cache and I/O paths: cache remote file blocks, cache serialized -Parquet ranges, cache predicate results, and merge small random reads. +Parquet V2 has five complementary cache and I/O paths: cache footer/parsed metadata, cache remote +file blocks, cache serialized Parquet ranges, cache predicate results, and merge small random reads. ```mermaid flowchart TB - A[Parquet Column Reader ReadAt] --> B{Parquet Page Cache Hit?} + A[Parquet Column Reader Range Read] --> B{Parquet Page Cache Hit?} B -- "Yes" --> C[Return Cached Serialized Range Bytes] B -- "No" --> D{MergeRange Active?} D -- "Yes" --> E[MergeRangeFileReader
Merge Adjacent Small I/O] @@ -334,11 +655,46 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | +| Footer metadata cache | V2-owned immutable Thrift metadata and native physical schema tree | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, and unsafe cross-version cache casts | +| Small HTTP object staging | Complete object bytes for files at or below `in_memory_file_size` | Per-reader v2 wrapper; loaded once from byte zero and released with the file context | Collapse cold small-file requests and keep footer-cache-hit scans independent of server-specific near-EOF Range behavior | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | | MergeRangeFileReader | Not a cache; merges small ranges into larger slices | Installed temporarily for projected chunks of the current Row Group | Reduce remote small-I/O count and request overhead | +### Footer Cache Parity with V1 + +V2 reuses the common stable file identity policy, but owns its footer parser, schema lifecycle, +metadata payload type, and cache-key suffix entirely under `format_v2`. A v2 cache entry can be +shared by v2 readers but can never be cast as a v1 metadata object. Mutable Row Group, selection, +decoder, and scratch state remains per reader. Path-only keys are insufficient. Parse failures, +short files, unsupported metadata, and schema-affecting option changes cannot populate or reuse a +successful entry. No change to the v1 parser or metadata cache behavior is required. + +On a v2 cold miss, the parser deserializes the footer once into the V2-owned Thrift object and builds +the native schema tree directly from `SchemaElement` into V2's `NativeFieldDescriptor`. Row-group statistics, +dictionary and Bloom metadata planning consume the same native tree; no Arrow `FileMetaData` tree +or serialized-footer copy is retained. Production ColumnIndex/OffsetIndex planning reads and parses +Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and validated +OffsetIndex objects are transferred into Row Group execution instead of being fetched a second +time. Arrow metadata and PageIndexReader paths remain only as test oracles. + +Before the footer lookup, v2 wraps bounded HTTP Parquet objects in an in-memory reader. The wrapper +loads from byte zero on its first physical access, whether that access is a cold footer read or a +warm data-page read after a footer-cache hit. This preserves identical cold/warm behavior for HTTP +servers that advertise Range support but answer an overlong near-EOF range with HTTP 200, and keeps +the compatibility policy out of the v1 reader. + +### Page Cache Parity with V1 + +The native path uses v1's page-cache semantics as its minimum contract: the same stable file +identity, Column Chunk/page byte ranges, compressed versus decompressed entry distinction, checksum +and decompression ownership, subrange coverage, invalidation, admission, and fallback I/O rules. +Mutable decoder and dictionary state is never cached. A hit returns immutable bytes and advances the +reader exactly as the corresponding base read would; a miss performs normal I/O before optional +admission. An alternative policy is accepted only with correctness coverage plus warm/cold, +selective/dense, local/remote benchmarks showing it is no worse than v1. + ### Why Page Cache registers only surviving chunks The footer is read before Row Group planning and before Page Cache ranges are registered, so @@ -349,8 +705,10 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. -- When average projected chunks are small and the reader is not in-memory, install - MergeRangeFileReader so subsequent Arrow `ReadAt` calls actually use merged reads. +- After dictionary probing, small projected chunks share one Row-Group-scoped + MergeRangeFileReader on the native data-page path. Large chunks and in-memory files keep the base + reader. Native metadata/index reads use explicit immutable ranges and never own a decoder stream + cursor. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. @@ -396,6 +754,14 @@ flowchart LR E --> F[Bound by batch_size and Selected Range] ``` +Changing the requested row cap changes only the amount of work performed by the persistent reader. +It does not recreate Column Readers, Column Chunk readers, dictionaries, SerDe/conversion objects, +builders, or scratch. The probe is charged as a normal batch and estimates completed Doris rows and +bytes after table-level materialization, matching v1's measurement point. Empty or highly filtered +probes do not permanently collapse the batch size; the scheduler retains enough evidence before +updating its estimate and respects page/range boundaries without turning each fragment into a new +reader lifecycle. + ### 10.3 Aggregate Pushdown When TableReader proves that no filter or delete semantics can change the result, COUNT / MIN / MAX @@ -408,6 +774,34 @@ Without row-level filtering, output columns may be warmed together. With filteri columns first and defer non-predicate columns until at least one row survives, aligning network bandwidth with lazy materialization. +For safe single-column conjuncts, the scheduler records an exponentially weighted cost per input +row and survival ratio. Cold batches preserve declaration order; after every candidate has a sample, +the next batch orders predicates by cost per rejected row. The same learned order limits predicate +prefetch to the prefix with a meaningful probability of being reached, and a sustained high overall +survival ratio estimate may warm output chunks early. This retains correctness because only +independently safe conjuncts move and all readers still advance over the same logical batch. + +Predicate-only `INT`, `BIGINT`, `FLOAT`, and `DOUBLE` comparisons can bypass Doris-column +materialization when every value page in the Column Chunk is fixed-width PLAIN and the expression +is an exact `=`, `!=`, `<`, `<=`, `>`, or `>=` comparison with a literal. The scheduler compiles +the conjunction into physical-value predicates, the decoder compares selected non-NULL spans in +place, and the reader remaps the compact match bytes to logical nullable rows. NULL comparisons are +false. Mixed encodings, projected predicate columns, casts, logical conversions, compound +expressions, and unsupported types make the decision before either the definition-level or value +cursor advances, so they safely retain ordinary expression evaluation. An exact direct result +leaves only a row-shaped placeholder for the hidden predicate slot and avoids both predicate-column +materialization and later predicate compaction. + +Selection state is scheduler-owned across adaptive batches. Its dense filter bitmap is cached by +selection generation and batch shape, so every lazily materialized output reader consumes the same +bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary +capacity remains reusable. + +Fully rejected batches accumulate a logical lag for lazy columns. When a later batch survives, each +lazy reader consumes that lag in at most 65,535-row dense-filter chunks. This keeps the scheduler's +single logical skip while bounding adapter-owned bitmap capacity independently of prefix length and +lazy projection width. + ## 11. Correctness, Fallback, and Capability Boundaries V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be @@ -442,6 +836,15 @@ split safely, or read anomalies must never change query semantics. Troubleshoot in this order: verify planning effectiveness, row filtering, lazy materialization, and then I/O/cache health. Total ScanTime alone does not identify the cause. +The visible timer hierarchy is `FileScannerV2 -> TableReader -> FileReader -> IO`; the +format-specific `ParquetReader` subtree belongs to `FileReader`. Scanner lifecycle and Split work, +table-semantic restoration, format metadata/index/decode/materialization, and physical I/O are +charged to their owning layer. Native child-reader statistics accumulate in plain integers and are +published every 16 batches. Row Group reset, EOF, and reader close force the final delta before +destroying the reader tree, so short files retain complete attribution without paying recursive +profile publication on every tiny batch. Retained-scratch inspection uses the same amortized cadence +and a Row Group remains its hard lifetime bound. + ```mermaid flowchart TD A[Slow Scan] --> B{Many Row Groups Pruned?} @@ -465,10 +868,18 @@ flowchart TD | Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | +| PLAIN direct predicate | How many eligible predicate-only physical batches and input rows bypassed Doris-column materialization? Inspect `PlainPredicateDirectBatches/Rows`. | +| Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | +| Metadata lifecycle | How much time was spent reading/parsing the native footer and schema tree, natively reading/parsing page indexes, and evaluating row-group/page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | | Condition Cache | How many rows were skipped early after a cache hit? | +| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? Compare `HybridSelectionBatches`, `HybridSelectionRanges`, and `HybridSelectionNullFallbackBatches` to distinguish batched sparse decode from NULL-interleaving fallback. | +| Batch fragmentation | How does `TotalBatches` divide into adaptive probes, dense, selected, empty, page-crossing, and nested/fragmented batches? | +| Index decisions | How often were statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skips attempted, accepted, conservatively rejected, or rejected as corrupt? | +| Cache lifecycle | For footer/page/file/condition caches, what were request, hit, miss, bypass, admission/write, byte, wait, and underlying-I/O counts using v1-compatible meanings? | > Interpret pruning ratios in the context of write layout. Unsorted data produces wide min/max > ranges, so Row Group/Page pruning may be ineffective even when the reader and indexes work @@ -476,14 +887,16 @@ flowchart TD ## 13. Summary -The FileScannerV2 Parquet scan pipeline has three primary threads: +The FileScannerV2 Parquet scan pipeline has four primary threads: 1. **Semantic thread:** TableReader maps table schema and predicates into stable file-local semantics, preserving schema evolution, partition columns, and missing columns. 2. **Pruning thread:** Split → Row Group → Page → Row progressively applies Runtime Filters, Statistics, Dictionary, Bloom, Page Index, and actual-value filters. -3. **I/O thread:** Predicate-first reads, SelectionVector, lazy materialization, adaptive batches, - FileCache/Page Cache/Condition Cache, and MergeRange reduce read amplification together. +3. **Decode thread:** Persistent native page/encoding readers merge Dremel levels with Selection, + reuse scratch, reconstruct complex values from shared plans, and materialize directly to Doris. +4. **I/O thread:** Predicate-first reads, adaptive batches, Footer/File/Page/Condition caches, and + MergeRange reduce read amplification together. ```mermaid flowchart LR diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java index efebe6924136db..b4b15c591e27b1 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java @@ -53,6 +53,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; @@ -589,7 +590,7 @@ protected void setValidationQuery(HikariDataSource ds) { protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, String sql) throws SQLException { if (config.getOp() == TJdbcOperation.READ) { - conn.setAutoCommit(false); + disableAutoCommitIfSupported(conn); Preconditions.checkArgument(sql != null, "SQL statement cannot be null for READ operation."); stmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); stmt.setFetchSize(config.getBatchSize()); // set fetch size to batch size @@ -601,6 +602,20 @@ protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, } } + static void disableAutoCommitIfSupported(Connection conn) throws SQLException { + // Hikari invalidates its proxy when setAutoCommit throws, so check the transaction + // capability first to keep non-transactional driver connections usable for reads. + if (!conn.getMetaData().supportsTransactions()) { + LOG.info("JDBC driver does not support transactions; continuing in auto-commit read mode"); + return; + } + try { + conn.setAutoCommit(false); + } catch (SQLFeatureNotSupportedException e) { + LOG.info("JDBC driver does not support disabling auto-commit; continuing in read mode"); + } + } + protected abstract Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException; @@ -838,4 +853,3 @@ protected String defaultByteArrayToHexString(byte[] bytes) { return hexString.toString(); } } - diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java new file mode 100644 index 00000000000000..a0d0968ea67979 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/BaseJdbcExecutorTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.jdbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.concurrent.atomic.AtomicBoolean; + +class BaseJdbcExecutorTest { + @Test + void disableAutoCommitSkipsNonTransactionalDrivers() { + AtomicBoolean setAutoCommitCalled = new AtomicBoolean(); + Connection connection = connectionWithTransactionSupport(false, setAutoCommitCalled, null); + + Assertions.assertDoesNotThrow( + () -> BaseJdbcExecutor.disableAutoCommitIfSupported(connection)); + Assertions.assertFalse(setAutoCommitCalled.get()); + } + + @Test + void disableAutoCommitAllowsUnsupportedTransactionalDrivers() { + AtomicBoolean setAutoCommitCalled = new AtomicBoolean(); + Connection connection = connectionWithTransactionSupport( + true, setAutoCommitCalled, new SQLFeatureNotSupportedException("unsupported")); + + Assertions.assertDoesNotThrow( + () -> BaseJdbcExecutor.disableAutoCommitIfSupported(connection)); + Assertions.assertTrue(setAutoCommitCalled.get()); + } + + @Test + void disableAutoCommitPropagatesOtherSqlErrors() { + Connection connection = connectionWithTransactionSupport( + true, new AtomicBoolean(), new SQLException("connection failed")); + + Assertions.assertThrows(SQLException.class, + () -> BaseJdbcExecutor.disableAutoCommitIfSupported(connection)); + } + + private static Connection connectionWithTransactionSupport( + boolean supportsTransactions, AtomicBoolean setAutoCommitCalled, SQLException exception) { + DatabaseMetaData metadata = (DatabaseMetaData) Proxy.newProxyInstance( + BaseJdbcExecutorTest.class.getClassLoader(), + new Class[] {DatabaseMetaData.class}, + (proxy, method, args) -> { + if (method.getName().equals("supportsTransactions")) { + return supportsTransactions; + } + throw new UnsupportedOperationException(method.getName()); + }); + return (Connection) Proxy.newProxyInstance( + BaseJdbcExecutorTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, args) -> { + if (method.getName().equals("getMetaData")) { + return metadata; + } + if (method.getName().equals("setAutoCommit")) { + setAutoCommitCalled.set(true); + if (exception != null) { + throw exception; + } + return null; + } + throw new UnsupportedOperationException(method.getName()); + }); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index bbc7e4994094ce..29848358d63ca3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -82,8 +82,12 @@ public class SummaryProfile { public static final String INIT_SCAN_NODE_TIME = "Init Scan Node Time"; public static final String FINALIZE_SCAN_NODE_TIME = "Finalize Scan Node Time"; public static final String GET_SPLITS_TIME = "Get Splits Time"; + public static final String EXTERNAL_TABLE_META_TIME = "External Table Meta Time"; + public static final String EXTERNAL_TABLE_GET_TABLE_META_TIME = "External Table Get Table Meta Time"; + public static final String EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME = "External Table Get Partition Values Time"; public static final String GET_PARTITIONS_TIME = "Get Partitions Time"; public static final String GET_PARTITION_FILES_TIME = "Get Partition Files Time"; + public static final String EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME = "External Table Get File Scan Tasks Time"; public static final String CREATE_SCAN_RANGE_TIME = "Create Scan Range Time"; public static final String SINK_SET_PARTITION_VALUES_TIME = "Sink Set Partition Values Time"; public static final String PLAN_TIME = "Plan Time"; @@ -173,8 +177,12 @@ public boolean isWarmup() { INIT_SCAN_NODE_TIME, FINALIZE_SCAN_NODE_TIME, GET_SPLITS_TIME, + EXTERNAL_TABLE_META_TIME, + EXTERNAL_TABLE_GET_TABLE_META_TIME, + EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, GET_PARTITIONS_TIME, GET_PARTITION_FILES_TIME, + EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, SINK_SET_PARTITION_VALUES_TIME, CREATE_SCAN_RANGE_TIME, ICEBERG_SCAN_METRICS, @@ -225,10 +233,14 @@ public boolean isWarmup() { .put(INIT_SCAN_NODE_TIME, 2) .put(FINALIZE_SCAN_NODE_TIME, 2) .put(GET_SPLITS_TIME, 3) + .put(EXTERNAL_TABLE_META_TIME, 4) + .put(EXTERNAL_TABLE_GET_TABLE_META_TIME, 5) + .put(EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, 5) .put(NEREIDS_DISTRIBUTE_TIME, 1) .put(NEREIDS_BE_FOLD_CONST_TIME, 2) .put(GET_PARTITIONS_TIME, 3) .put(GET_PARTITION_FILES_TIME, 3) + .put(EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, 5) .put(SINK_SET_PARTITION_VALUES_TIME, 3) .put(CREATE_SCAN_RANGE_TIME, 2) .put(ICEBERG_SCAN_METRICS, 3) @@ -379,6 +391,26 @@ public boolean isWarmup() { private long nereidsMvRewriteTime = 0; @SerializedName(value = "externalCatalogMetaTime") private long externalCatalogMetaTime = 0; + // Total time to get table meta, including time to get table meta from external catalog and time to do some + // process based on the meta, such as partition prune. + @SerializedName(value = "externalTableGetTableMetaTime") + private long externalTableGetTableMetaTime = 0; + // Total time to get partition values, including time to get partition values from external catalog and time to do + // some process based on the partition values, such as partition prune. + @SerializedName(value = "externalTableGetPartitionValuesTime") + private long externalTableGetPartitionValuesTime = 0; + // Total time to get partitions, including time to get partitions from external catalog and time to do some + // process based on the partitions, such as partition prune. + @SerializedName(value = "externalTableGetPartitionsTime") + private long externalTableGetPartitionsTime = 0; + // Total time to get partition files, including time to get partition files from external catalog and time to do + // some process based on the partition files, such as creating scan range. + @SerializedName(value = "externalTableGetPartitionFilesTime") + private long externalTableGetPartitionFilesTime = 0; + // Total time to get file scan tasks, including time to get file scan tasks from external catalog and time to do + // some process based on the file scan tasks, such as creating scan range. + @SerializedName(value = "externalTableGetFileScanTasksTime") + private long externalTableGetFileScanTasksTime = 0; @SerializedName(value = "externalTvfInitTime") private long externalTvfInitTime = 0; @SerializedName(value = "nereidsPartitiionPruneTime") @@ -504,10 +536,19 @@ private void updateExecutionSummaryProfile() { getPrettyTime(finalizeScanNodeFinishTime, finalizeScanNodeStartTime, TUnit.TIME_MS)); executionSummaryProfile.addInfoString(GET_SPLITS_TIME, getPrettyTime(getSplitsFinishTime, getSplitsStartTime, TUnit.TIME_MS)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_META_TIME, + getPrettyAccumulatedTime(externalCatalogMetaTime)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_TABLE_META_TIME, + getPrettyAccumulatedTime(externalTableGetTableMetaTime)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME, + getPrettyAccumulatedTime(externalTableGetPartitionValuesTime)); executionSummaryProfile.addInfoString(GET_PARTITIONS_TIME, - getPrettyTime(getPartitionsFinishTime, getSplitsStartTime, TUnit.TIME_MS)); + getExternalTableMetaTime(externalTableGetPartitionsTime, getPartitionsFinishTime, getSplitsStartTime)); executionSummaryProfile.addInfoString(GET_PARTITION_FILES_TIME, - getPrettyTime(getPartitionFilesFinishTime, getPartitionsFinishTime, TUnit.TIME_MS)); + getExternalTableMetaTime(externalTableGetPartitionFilesTime, + getPartitionFilesFinishTime, getPartitionsFinishTime)); + executionSummaryProfile.addInfoString(EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME, + getPrettyAccumulatedTime(externalTableGetFileScanTasksTime)); executionSummaryProfile.addInfoString(SINK_SET_PARTITION_VALUES_TIME, getPrettyTime(sinkSetPartitionValuesFinishTime, sinkSetPartitionValuesStartTime, TUnit.TIME_MS)); executionSummaryProfile.addInfoString(CREATE_SCAN_RANGE_TIME, @@ -1062,6 +1103,20 @@ private String getPrettyTime(long end, long start, TUnit unit) { return RuntimeProfile.printCounter(end - start, unit); } + private String getPrettyAccumulatedTime(long timeMs) { + if (timeMs <= 0) { + return "N/A"; + } + return RuntimeProfile.printCounter(timeMs, TUnit.TIME_MS); + } + + private String getExternalTableMetaTime(long accumulatedTimeMs, long end, long start) { + if (accumulatedTimeMs > 0) { + return RuntimeProfile.printCounter(accumulatedTimeMs, TUnit.TIME_MS); + } + return getPrettyTime(end, start, TUnit.TIME_MS); + } + public void setTransactionBeginTime(TransactionType type) { this.transactionCommitBeginTime = TimeUtils.getStartTimeMs(); this.transactionType = type; @@ -1181,14 +1236,43 @@ public long getCloudMetaTimeMs() { return TimeUnit.NANOSECONDS.toMillis(getPartitionVersionTime + getTableVersionTime); } - public void addExternalCatalogMetaTime(long ms) { - this.externalCatalogMetaTime += ms; - } - public long getExternalCatalogMetaTimeMs() { return externalCatalogMetaTime; } + public synchronized void addExternalTableGetTableMetaTime(long ms) { + this.externalTableGetTableMetaTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetPartitionValuesTime(long ms) { + this.externalTableGetPartitionValuesTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetPartitionsTime(long ms) { + this.externalTableGetPartitionsTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetPartitionFilesTime(long ms) { + this.externalTableGetPartitionFilesTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalTableGetFileScanTasksTime(long ms) { + this.externalTableGetFileScanTasksTime += ms; + addExternalCatalogMetaTimeInternal(ms); + } + + public synchronized void addExternalCatalogMetaTime(long ms) { + addExternalCatalogMetaTimeInternal(ms); + } + + private void addExternalCatalogMetaTimeInternal(long ms) { + this.externalCatalogMetaTime += ms; + } + public void addExternalTvfInitTime(long ms) { this.externalTvfInitTime += ms; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 6a986da45303d6..37dbf4cdd390ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -124,8 +124,7 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { - initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, List columns, Map> nameMapping, Map base64InitialDefaults) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), base64InitialDefaults); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); + tSchema.setRootField(getExternalSchemaForAllColumn( + columns, nameMapping, hasNameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - child.getType(), child, nameMapping, base64InitialDefaults)); + child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -161,11 +170,13 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { - return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + return getExternalSchema(columnType, dorisColumn, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } private static TField getExternalSchema(Type columnType, Column dorisColumn, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); @@ -178,9 +189,12 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } - if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { - // for iceberg set name mapping. - root.setNameMapping(new ArrayList<>(nameMapping.get(dorisColumn.getUniqueId()))); + if (hasNameMapping) { + // The explicit capability keeps old-FE plans on legacy fallback while making an empty + // per-field mapping authoritative for plans produced by a compatible FE. + root.setNameMapping(new ArrayList<>( + nameMapping.getOrDefault(dorisColumn.getUniqueId(), Collections.emptyList()))); + root.setNameMappingIsAuthoritative(true); } TNestedField nestedField = new TNestedField(); @@ -199,7 +213,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); fieldPtr.setFieldPtr(getExternalSchema( - subField.getType(), subColumn, nameMapping, base64InitialDefaults)); + subField.getType(), subColumn, nameMapping, hasNameMapping, + base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -212,7 +227,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -223,13 +238,13 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 3bd819bea62a16..bfe34e9a32325e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -34,6 +34,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.source.HiveSplit; @@ -46,6 +47,7 @@ import org.apache.doris.statistics.StatisticalType; import org.apache.doris.system.Backend; import org.apache.doris.tablefunction.ExternalFileTableValuedFunction; +import org.apache.doris.thrift.TColumnCategory; import org.apache.doris.thrift.TExternalScanRange; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileCompressType; @@ -105,6 +107,7 @@ public abstract class FileQueryScanNode extends FileScanNode { protected TableScanParams scanParams; protected FileSplitter fileSplitter; + protected SummaryProfile summaryProfile; // The data cache function only works for queries on Hive, Iceberg, Hudi(via HMS), and Paimon tables. // See: https://doris.incubator.apache.org/docs/dev/lakehouse/data-cache @@ -136,11 +139,12 @@ public FileQueryScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeNam public void init() throws UserException { super.init(); if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeStartTime(); + summaryProfile = ConnectContext.get().getExecutor().getSummaryProfile(); + summaryProfile.setInitScanNodeStartTime(); } doInitialize(); if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeFinishTime(); + summaryProfile.setInitScanNodeFinishTime(); } } @@ -161,6 +165,13 @@ protected void doInitialize() throws UserException { sessionVariable.maxInitialSplitNum); } + protected SummaryProfile getSummaryProfile() { + if (summaryProfile == null) { + summaryProfile = SummaryProfile.getSummaryProfile(ConnectContext.get()); + } + return summaryProfile; + } + // Init schema (Tuple/Slot) related params. protected void initSchemaParams() throws UserException { destSlotDescByName = Maps.newHashMap(); @@ -175,11 +186,9 @@ protected void initSchemaParams() throws UserException { for (SlotDescriptor slot : desc.getSlots()) { TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); slotInfo.setSlotId(slot.getId().asInt()); - boolean isFileSlot = !partitionKeys.contains(slot.getColumn().getName()); - if (isIcebergRowIdColumn(slot)) { - isFileSlot = false; - } - slotInfo.setIsFileSlot(isFileSlot); + TColumnCategory category = classifyColumn(slot, partitionKeys); + slotInfo.setCategory(category); + slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false); @@ -192,23 +201,42 @@ protected void initSchemaParams() throws UserException { } private void updateRequiredSlots() throws UserException { + Map existingSlotInfoById = Maps.newHashMap(); + if (params.getRequiredSlots() != null) { + for (TFileScanSlotInfo slotInfo : params.getRequiredSlots()) { + existingSlotInfoById.put(slotInfo.getSlotId(), slotInfo); + } + } params.unsetRequiredSlots(); + List partitionKeys = getPathPartitionKeys(); for (SlotDescriptor slot : desc.getSlots()) { - TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); - slotInfo.setSlotId(slot.getId().asInt()); - boolean isFileSlot = !getPathPartitionKeys().contains(slot.getColumn().getName()); - if (isIcebergRowIdColumn(slot)) { - isFileSlot = false; + TFileScanSlotInfo slotInfo = existingSlotInfoById.get(slot.getId().asInt()); + if (slotInfo == null) { + slotInfo = new TFileScanSlotInfo(); } - slotInfo.setIsFileSlot(isFileSlot); + slotInfo.setSlotId(slot.getId().asInt()); + TColumnCategory category = classifyColumn(slot, partitionKeys); + slotInfo.setCategory(category); + slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } // Update required slots and column_idxs in scanRangeLocations. setColumnPositionMapping(); } - private boolean isIcebergRowIdColumn(SlotDescriptor slot) { - return Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName()); + /** + * Classify a column's category for the BE reader. + * Subclasses override this for format-specific classification. + */ + protected TColumnCategory classifyColumn(SlotDescriptor slot, List partitionKeys) { + if (partitionKeys.contains(slot.getColumn().getName())) { + return TColumnCategory.PARTITION_KEY; + } + return TColumnCategory.REGULAR; + } + + protected boolean isFileSlot(TColumnCategory category) { + return category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED; } public void setTableSample(TableSample tSample) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index e36ebefe646eb2..f017c444c69122 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -92,6 +92,8 @@ public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, @Override protected void toThrift(TPlanNode planNode) { planNode.setPushDownAggTypeOpt(pushDownAggNoGroupingOp); + planNode.setPushDownCountSlotIds( + pushDownCountSlotIds.stream().map(id -> id.asInt()).collect(Collectors.toList())); planNode.setNodeType(TPlanNodeType.FILE_SCAN_NODE); TFileScanNode fileScanNode = new TFileScanNode(); @@ -107,6 +109,21 @@ protected void setPushDownCount(long count) { tableLevelRowCount = count; } + /** + * Return whether FE may replace real table-format splits with metadata COUNT splits. + * + *

The aggregate opcode alone is insufficient because both {@code COUNT(*)} and + * {@code COUNT(col)} use {@link TPushAggOp#COUNT}. The semantic argument list distinguishes + * them: it is empty only for {@code COUNT(*)}/{@code COUNT(1)}. For example, if an Iceberg + * table has 100 data files, retaining one representative split is correct for a snapshot + * {@code COUNT(*)}. Doing that for {@code COUNT(required_col)} is unsafe: BE deliberately + * falls back to reading the column, but it would then see only the representative file and + * undercount the table. + */ + protected boolean isTableLevelCountStarPushdown() { + return pushDownAggNoGroupingOp == TPushAggOp.COUNT && pushDownCountSlotIds.isEmpty(); + } + private long getPushDownCount() { return tableLevelRowCount; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 05d78c37a649e5..03a37937562952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -32,6 +32,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheKey; import org.apache.doris.datasource.SchemaCacheValue; @@ -237,7 +238,15 @@ public boolean isSupportedHmsTable() { protected synchronized void makeSureInitialized() { super.makeSureInitialized(); if (!objectCreated) { - remoteTable = loadHiveTable(); + long startTime = System.currentTimeMillis(); + try { + remoteTable = loadHiveTable(); + } finally { + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } + } if (remoteTable == null) { throw new IllegalArgumentException("Hms table not exists, table: " + getNameWithFullQualifiers()); } else { @@ -1290,11 +1299,18 @@ private Table loadHiveTable() { } public HiveExternalMetaCache.HivePartitionValues getHivePartitionValues(Optional snapshot) { + long startTime = System.currentTimeMillis(); HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .hive(getCatalog().getId()); try { List partitionColumnTypes = this.getPartitionColumnTypes(snapshot); - return cache.getPartitionValues(this, partitionColumnTypes); + HiveExternalMetaCache.HivePartitionValues partitionValues = cache.getPartitionValues(this, + partitionColumnTypes); + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis() - startTime); + } + return partitionValues; } catch (Exception e) { if (e.getMessage().contains(HiveExternalMetaCache.ERR_CACHE_INCONSISTENCY)) { LOG.warn("Hive metastore cache inconsistency detected for table: {}.{}.{}. " @@ -1303,7 +1319,13 @@ public HiveExternalMetaCache.HivePartitionValues getHivePartitionValues(Optional Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableByEngine( getCatalog().getId(), getMetaCacheEngine(), getDbName(), getName()); List partitionColumnTypes = this.getPartitionColumnTypes(snapshot); - return cache.getPartitionValues(this, partitionColumnTypes); + HiveExternalMetaCache.HivePartitionValues partitionValues = cache.getPartitionValues(this, + partitionColumnTypes); + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis() - startTime); + } + return partitionValues; } else { throw e; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index ad7539af255150..f5acd7834b6402 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -59,7 +59,6 @@ import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TFileTextScanRangeParams; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; import org.apache.doris.thrift.TTransactionalHiveDesc; @@ -150,41 +149,51 @@ static void markTransactionalHiveScanParams(TFileScanRangeParams scanParams) { } protected List getPartitions() throws AnalysisException { + long startTime = System.currentTimeMillis(); List resPartitions = Lists.newArrayList(); - HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .hive(hmsTable.getCatalog().getId()); - List partitionColumnTypes = hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable)); - if (!partitionColumnTypes.isEmpty()) { - // partitioned table - Collection partitionItems; - // partitions has benn pruned by Nereids, in PruneFileScanPartition, - // so just use the selected partitions. - this.totalPartitionNum = selectedPartitions.totalPartitionNum; - partitionItems = selectedPartitions.selectedPartitions.values(); - Preconditions.checkNotNull(partitionItems); - this.selectedPartitionNum = partitionItems.size(); - - // get partitions from cache - List> partitionValuesList = Lists.newArrayListWithCapacity(partitionItems.size()); - for (PartitionItem item : partitionItems) { - partitionValuesList.add( - ((ListPartitionItem) item).getItems().get(0).getPartitionValuesAsStringListForHive()); + try { + HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .hive(hmsTable.getCatalog().getId()); + List partitionColumnTypes = + hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable)); + if (!partitionColumnTypes.isEmpty()) { + // partitioned table + Collection partitionItems; + // partitions has benn pruned by Nereids, in PruneFileScanPartition, + // so just use the selected partitions. + this.totalPartitionNum = selectedPartitions.totalPartitionNum; + partitionItems = selectedPartitions.selectedPartitions.values(); + Preconditions.checkNotNull(partitionItems); + this.selectedPartitionNum = partitionItems.size(); + + // get partitions from cache + List> partitionValuesList = Lists.newArrayListWithCapacity(partitionItems.size()); + for (PartitionItem item : partitionItems) { + partitionValuesList.add( + ((ListPartitionItem) item).getItems().get(0).getPartitionValuesAsStringListForHive()); + } + resPartitions = cache.getAllPartitionsWithCache(hmsTable, partitionValuesList); + } else { + // non partitioned table, create a dummy partition to save location and inputformat, + // so that we can unify the interface. + HivePartition dummyPartition = new HivePartition(hmsTable.getOrBuildNameMapping(), true, + hmsTable.getRemoteTable().getSd().getInputFormat(), + hmsTable.getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()); + this.totalPartitionNum = 1; + this.selectedPartitionNum = 1; + resPartitions.add(dummyPartition); } - resPartitions = cache.getAllPartitionsWithCache(hmsTable, partitionValuesList); - } else { - // non partitioned table, create a dummy partition to save location and inputformat, - // so that we can unify the interface. - HivePartition dummyPartition = new HivePartition(hmsTable.getOrBuildNameMapping(), true, - hmsTable.getRemoteTable().getSd().getInputFormat(), - hmsTable.getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()); - this.totalPartitionNum = 1; - this.selectedPartitionNum = 1; - resPartitions.add(dummyPartition); - } - if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionsFinishTime(); + if (ConnectContext.get().getExecutor() != null) { + getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); + getSummaryProfile().setGetPartitionsFinishTime(); + } + return resPartitions; + } catch (RuntimeException e) { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); + } + throw e; } - return resPartitions; } @Override @@ -228,6 +237,7 @@ public void startSplit(int numBackends) { Executor scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); String bindBrokerName = hmsTable.getCatalog().bindBrokerName(); AtomicInteger numFinishedPartitions = new AtomicInteger(0); + long startTime = System.currentTimeMillis(); CompletableFuture.runAsync(() -> { for (HivePartition partition : prunedPartitions) { if (batchException.get() != null || splitAssignment.isStop()) { @@ -255,6 +265,10 @@ public void startSplit(int numBackends) { splitAssignment.setException(batchException.get()); } if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionFilesTime( + System.currentTimeMillis() - startTime); + } splitAssignment.finishSchedule(); } } @@ -294,6 +308,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List allFiles, String bindBrokerName, int numBackends, boolean isBatchMode) throws IOException, UserException { List fileCaches; + long startTime = System.currentTimeMillis(); if (hiveTransaction != null) { try { fileCaches = getFileSplitByTransaction(cache, partitions, bindBrokerName); @@ -308,6 +323,9 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List 1, directoryLister, hmsTable); } + if (!isBatchMode && getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionFilesTime(System.currentTimeMillis() - startTime); + } long targetFileSplitSize = determineTargetFileSplitSize(fileCaches, isBatchMode); if (tableSample != null) { @@ -317,7 +335,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List table } public static long getLastTimeStamp(HMSExternalTable hmsTable) { + long startTime = System.currentTimeMillis(); HoodieTableMetaClient hudiClient = hmsTable.getHudiClient(); HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); Option snapshotInstant = timeline.lastInstant(); + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } if (!snapshotInstant.isPresent()) { return 0L; } @@ -279,40 +285,49 @@ public static long getLastTimeStamp(HMSExternalTable hmsTable) { public static TablePartitionValues getPartitionValues(Optional tableSnapshot, HMSExternalTable hmsTable) { - TablePartitionValues partitionValues = new TablePartitionValues(); - - HoodieTableMetaClient hudiClient = hmsTable.getHudiClient(); - HudiExternalMetaCache hudiExternalMetaCache = - Env.getCurrentEnv().getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()); - boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition(); - - if (tableSnapshot.isPresent()) { - if (tableSnapshot.get().getType() == TableSnapshot.VersionType.VERSION) { - // Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"; - return partitionValues; - } - String queryInstant = tableSnapshot.get().getValue().replaceAll("[-: ]", ""); - try { - partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> - hudiExternalMetaCache.getSnapshotPartitionValues(hmsTable, queryInstant, useHiveSyncPartition)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } - } else { - HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); - Option snapshotInstant = timeline.lastInstant(); - if (!snapshotInstant.isPresent()) { - return partitionValues; + long startTime = System.currentTimeMillis(); + try { + TablePartitionValues partitionValues = new TablePartitionValues(); + + HoodieTableMetaClient hudiClient = hmsTable.getHudiClient(); + HudiExternalMetaCache hudiExternalMetaCache = + Env.getCurrentEnv().getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()); + boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition(); + + if (tableSnapshot.isPresent()) { + if (tableSnapshot.get().getType() == TableSnapshot.VersionType.VERSION) { + // Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"; + return partitionValues; + } + String queryInstant = tableSnapshot.get().getValue().replaceAll("[-: ]", ""); + try { + partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> + hudiExternalMetaCache.getSnapshotPartitionValues( + hmsTable, queryInstant, useHiveSyncPartition)); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } else { + HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); + Option snapshotInstant = timeline.lastInstant(); + if (!snapshotInstant.isPresent()) { + return partitionValues; + } + try { + partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() + -> hudiExternalMetaCache.getPartitionValues(hmsTable, useHiveSyncPartition)); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } } - try { - partitionValues = hmsTable.getCatalog().getExecutionAuthenticator().execute(() - -> hudiExternalMetaCache.getPartitionValues(hmsTable, useHiveSyncPartition)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + return partitionValues; + } finally { + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(null); + if (summaryProfile != null) { + summaryProfile.addExternalTableGetPartitionValuesTime(System.currentTimeMillis() - startTime); } } - return partitionValues; } public static HoodieTableMetaClient buildHudiTableMetaClient(String hudiBasePath, Configuration conf) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 75014c83ffd552..d5b454a8a5023c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -172,60 +172,67 @@ protected void doInitialize() throws UserException { initBackendPolicy(); initSchemaParams(); - hudiClient = hmsTable.getHudiClient(); - hudiClient.reloadActiveTimeline(); - basePath = hmsTable.getRemoteTable().getSd().getLocation(); - inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); - serdeLib = hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib(); - - if (scanParams != null && !scanParams.incrementalRead()) { - // Only support incremental read - throw new UserException("Not support function '" + scanParams.getParamType() + "' in hudi table"); - } - if (incrementalRead) { - if (isCowTable) { - try { - Map serd = hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters(); - if ("true".equals(serd.get("hoodie.query.as.ro.table")) - && hmsTable.getRemoteTable().getTableName().endsWith("_ro")) { - // Incremental read RO table as RT table, I don't know why? - isCowTable = false; - LOG.warn("Execute incremental read on RO table: {}", hmsTable.getFullQualifiers()); + long tableMetaStartTime = System.currentTimeMillis(); + try { + hudiClient = hmsTable.getHudiClient(); + hudiClient.reloadActiveTimeline(); + basePath = hmsTable.getRemoteTable().getSd().getLocation(); + inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); + serdeLib = hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib(); + + if (scanParams != null && !scanParams.incrementalRead()) { + // Only support incremental read + throw new UserException("Not support function '" + scanParams.getParamType() + "' in hudi table"); + } + if (incrementalRead) { + if (isCowTable) { + try { + Map serd = hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters(); + if ("true".equals(serd.get("hoodie.query.as.ro.table")) + && hmsTable.getRemoteTable().getTableName().endsWith("_ro")) { + // Incremental read RO table as RT table, I don't know why? + isCowTable = false; + LOG.warn("Execute incremental read on RO table: {}", hmsTable.getFullQualifiers()); + } + } catch (Exception e) { + // ignore } - } catch (Exception e) { - // ignore + } + if (incrementalRelation == null) { + throw new UserException("Failed to create incremental relation"); } } - if (incrementalRelation == null) { - throw new UserException("Failed to create incremental relation"); - } - } - timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); - TableSnapshot tableSnapshot = getQueryTableSnapshot(); - if (tableSnapshot != null) { - if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) { - throw new UserException("Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"); + timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); + TableSnapshot tableSnapshot = getQueryTableSnapshot(); + if (tableSnapshot != null) { + if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) { + throw new UserException("Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"); + } + queryInstant = tableSnapshot.getValue().replaceAll("[-: ]", ""); + } else { + Option snapshotInstant = timeline.lastInstant(); + if (!snapshotInstant.isPresent()) { + prunedPartitions = Collections.emptyList(); + partitionInit = true; + return; + } + queryInstant = snapshotInstant.get().requestedTime(); } - queryInstant = tableSnapshot.getValue().replaceAll("[-: ]", ""); - } else { - Option snapshotInstant = timeline.lastInstant(); - if (!snapshotInstant.isPresent()) { - prunedPartitions = Collections.emptyList(); - partitionInit = true; - return; + + HudiSchemaCacheValue hudiSchemaCacheValue = HudiUtils.getSchemaCacheValue(hmsTable, queryInstant); + columnNames = hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList()); + columnTypes = hudiSchemaCacheValue.getColTypes(); + + fsView = Env.getCurrentEnv() + .getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()) + .getFsView(hmsTable.getOrBuildNameMapping()); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - tableMetaStartTime); } - queryInstant = snapshotInstant.get().requestedTime(); } - - HudiSchemaCacheValue hudiSchemaCacheValue = HudiUtils.getSchemaCacheValue(hmsTable, queryInstant); - columnNames = hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList()); - columnTypes = hudiSchemaCacheValue.getColTypes(); - - fsView = Env.getCurrentEnv() - .getExtMetaCacheMgr() - .hudi(hmsTable.getCatalog().getId()) - .getFsView(hmsTable.getOrBuildNameMapping()); // Todo: Get the current schema id of the table, instead of using -1. // In Be Parquet/Rrc reader, if `current table schema id == current file schema id`, then its // `table_info_node_ptr` will be `TableSchemaChangeHelper::ConstNode`. When using `ConstNode`, @@ -370,21 +377,30 @@ private List getPrunedPartitions(HoodieTableMetaClient metaClient } private List getIncrementalSplits() { + long startTime = System.currentTimeMillis(); if (canUseNativeReader()) { List splits = incrementalRelation.collectSplits(); noLogsSplitNum.addAndGet(splits.size()); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } return splits; } Option partitionColumns = hudiClient.getTableConfig().getPartitionFields(); List partitionNames = partitionColumns.isPresent() ? Arrays.asList(partitionColumns.get()) : Collections.emptyList(); - return incrementalRelation.collectFileSlices().stream().map(fileSlice -> generateHudiSplit(fileSlice, - HudiPartitionUtils.parsePartitionValues(partitionNames, fileSlice.getPartitionPath()), - incrementalRelation.getEndTs())).collect(Collectors.toList()); + List splits = incrementalRelation.collectFileSlices().stream() + .map(fileSlice -> generateHudiSplit(fileSlice, + HudiPartitionUtils.parsePartitionValues(partitionNames, fileSlice.getPartitionPath()), + incrementalRelation.getEndTs())) + .collect(Collectors.toList()); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } + return splits; } private void getPartitionSplits(HivePartition partition, List splits) throws IOException { - String partitionName; if (partition.isDummyPartition()) { partitionName = ""; @@ -424,6 +440,7 @@ private void getPartitionsSplits(List partitions, List spl Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); CountDownLatch countDownLatch = new CountDownLatch(partitions.size()); AtomicReference throwable = new AtomicReference<>(); + long startTime = System.currentTimeMillis(); partitions.forEach(partition -> executor.execute(() -> { try { getPartitionSplits(partition, splits); @@ -441,6 +458,9 @@ private void getPartitionsSplits(List partitions, List spl if (throwable.get() != null) { throw new RuntimeException(throwable.get().getMessage(), throwable.get()); } + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } @Override @@ -448,15 +468,7 @@ public List getSplits(int numBackends) throws UserException { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { return getIncrementalSplits(); } - if (!partitionInit) { - try { - prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() - -> getPrunedPartitions(hudiClient)); - } catch (Exception e) { - throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); - } - partitionInit = true; - } + initPrunedPartitions(); List splits = Collections.synchronizedList(new ArrayList<>()); try { hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> { @@ -469,6 +481,23 @@ public List getSplits(int numBackends) throws UserException { return splits; } + private void initPrunedPartitions() throws UserException { + if (partitionInit) { + return; + } + long startTime = System.currentTimeMillis(); + try { + prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() + -> getPrunedPartitions(hudiClient)); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetPartitionsTime(System.currentTimeMillis() - startTime); + } + } catch (Exception e) { + throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); + } + partitionInit = true; + } + @Override public void startSplit(int numBackends) { if (prunedPartitions.isEmpty()) { @@ -477,6 +506,7 @@ public void startSplit(int numBackends) { } AtomicInteger numFinishedPartitions = new AtomicInteger(0); ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor(); + long startTime = System.currentTimeMillis(); CompletableFuture.runAsync(() -> { for (HivePartition partition : prunedPartitions) { if (batchException.get() != null || splitAssignment.isStop()) { @@ -506,6 +536,10 @@ public void startSplit(int numBackends) { splitAssignment.setException(batchException.get()); } if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime( + System.currentTimeMillis() - startTime); + } splitAssignment.finishSchedule(); } } @@ -522,15 +556,10 @@ public boolean isBatchMode() { if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { return false; } - if (!partitionInit) { - // Non partition table will get one dummy partition - try { - prunedPartitions = hmsTable.getCatalog().getExecutionAuthenticator().execute(() - -> getPrunedPartitions(hudiClient)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } - partitionInit = true; + try { + initPrunedPartitions(); + } catch (UserException e) { + throw new RuntimeException(e.getMessage(), e); } int numPartitions = sessionVariable.getNumPartitionsInBatchMode(); return numPartitions >= 0 && prunedPartitions.size() >= numPartitions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 05378977443f49..047272916fbee6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -231,7 +231,8 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTabl icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index d621b24d10253d..7dfdf6aed929bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -341,19 +341,6 @@ public Map getSupportedSysTables() { return IcebergSysTable.SUPPORTED_SYS_TABLES; } - @Override - public Optional findSysTable(String tableNameWithSysTableName) { - Optional sysTable = MTMVRelatedTableIf.super.findSysTable(tableNameWithSysTableName); - if (sysTable.isPresent()) { - return sysTable; - } - String sysTableName = SysTable.getTableNameWithSysTableName(tableNameWithSysTableName).second; - if (IcebergSysTable.POSITION_DELETES.equals(sysTableName)) { - return Optional.of(IcebergSysTable.UNSUPPORTED_POSITION_DELETES_TABLE); - } - return Optional.empty(); - } - @Override public boolean isView() { makeSureInitialized(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index db98bb283d41ee..04fc1c2865c82b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -361,10 +361,16 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx Schema schema = new Schema(visit.asNestedType().asStructType().fields()); Map properties = createTableInfo.getProperties(); properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE); - properties.putIfAbsent(TableProperties.FORMAT_VERSION, "2"); + Map catalogProperties = dorisCatalog.getProperties(); + if (!properties.containsKey(TableProperties.FORMAT_VERSION) + && !IcebergUtils.hasIcebergCatalogFormatVersion(catalogProperties)) { + properties.put(TableProperties.FORMAT_VERSION, "2"); + } properties.putIfAbsent(TableProperties.DELETE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName()); properties.putIfAbsent(TableProperties.UPDATE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName()); properties.putIfAbsent(TableProperties.MERGE_MODE, RowLevelOperationMode.MERGE_ON_READ.modeName()); + createTableInfo.validateIcebergRowLineageColumns( + IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalogProperties)); PartitionSpec partitionSpec = IcebergUtils.solveIcebergPartitionSpec(createTableInfo.getPartitionDesc(), schema); // Build and create table with optional sort order diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 95c9a6f26cc5c5..0c78be1accf76c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,14 +17,34 @@ package org.apache.doris.datasource.iceberg; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; + private final Optional>> nameMapping; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { + this(partitionInfo, snapshot, Optional.empty()); + } + + public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + this.nameMapping = nameMapping.map(mapping -> { + Map> copy = new HashMap<>(); + // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. + mapping.forEach((id, names) -> copy.put(id, + Collections.unmodifiableList(new ArrayList<>(names)))); + return Collections.unmodifiableMap(copy); + }); } public IcebergPartitionInfo getPartitionInfo() { @@ -34,4 +54,8 @@ public IcebergPartitionInfo getPartitionInfo() { public IcebergSnapshot getSnapshot() { return snapshot; } + + public Optional>> getNameMapping() { + return nameMapping; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index b8b8d5967491df..5022495d993994 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -77,14 +77,17 @@ import com.google.gson.reflect.TypeToken; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsUtil; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -106,6 +109,10 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mapping.MappedField; +import org.apache.iceberg.mapping.MappedFields; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; @@ -190,6 +197,33 @@ public Integer initialValue() { private static final Pattern SNAPSHOT_ID = Pattern.compile("\\d+"); + public static boolean hasIcebergCatalogFormatVersion(Map catalogProperties) { + return catalogProperties.containsKey(CatalogProperties.TABLE_OVERRIDE_PREFIX + TableProperties.FORMAT_VERSION) + || catalogProperties.containsKey(CatalogProperties.TABLE_DEFAULT_PREFIX + + TableProperties.FORMAT_VERSION); + } + + public static int getEffectiveIcebergFormatVersion(Map tableProperties, + Map catalogProperties) { + String formatVersion = catalogProperties.get(CatalogProperties.TABLE_OVERRIDE_PREFIX + + TableProperties.FORMAT_VERSION); + if (formatVersion == null) { + formatVersion = tableProperties.get(TableProperties.FORMAT_VERSION); + if (formatVersion == null) { + formatVersion = catalogProperties.get(CatalogProperties.TABLE_DEFAULT_PREFIX + + TableProperties.FORMAT_VERSION); + } + } + if (formatVersion == null) { + return 2; + } + try { + return Integer.parseInt(formatVersion); + } catch (NumberFormatException ignored) { + return 2; + } + } + public static Expression convertToIcebergExpr(Expr expr, Schema schema) { if (expr == null) { return null; @@ -982,9 +1016,9 @@ public static Object parsePartitionValueFromString(String valueStr, org.apache.i case LONG: return Long.parseLong(valueStr); case FLOAT: - return Float.parseFloat(valueStr); + return Float.parseFloat(normalizeFloatingPointPartitionValue(valueStr)); case DOUBLE: - return Double.parseDouble(valueStr); + return Double.parseDouble(normalizeFloatingPointPartitionValue(valueStr)); case BOOLEAN: return Boolean.parseBoolean(valueStr); case DATE: @@ -1003,6 +1037,20 @@ public static Object parsePartitionValueFromString(String valueStr, org.apache.i } } + private static String normalizeFloatingPointPartitionValue(String valueStr) { + if ("nan".equalsIgnoreCase(valueStr)) { + return "NaN"; + } + if ("inf".equalsIgnoreCase(valueStr) || "+inf".equalsIgnoreCase(valueStr) + || "infinity".equalsIgnoreCase(valueStr) || "+infinity".equalsIgnoreCase(valueStr)) { + return "Infinity"; + } + if ("-inf".equalsIgnoreCase(valueStr) || "-infinity".equalsIgnoreCase(valueStr)) { + return "-Infinity"; + } + return valueStr; + } + /** * Parse timestamp string to microseconds using Doris's built-in datetime * parser. @@ -1045,8 +1093,18 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { + private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, + boolean enableMappingTimestampTz) { column.setUniqueId(icebergField.fieldId()); + if (icebergField.initialDefault() != null) { + String serializedDefault = serializeInitialDefault( + icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz); + // Column constructs complex children without Iceberg field metadata. Copy through the + // public default-info API so recursive fields retain their logical pre-add value. + Column defaultCarrier = new Column(column.getName(), column.getType(), false, null, + column.isAllowNull(), serializedDefault, ""); + column.setDefaultValueInfo(defaultCarrier); + } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1065,7 +1123,8 @@ private static void updateIcebergColumnUniqueId(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); + updateIcebergColumnMetadata( + childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); } } } @@ -1122,7 +1181,7 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, initialDefault, field.doc(), true, -1); - updateIcebergColumnUniqueId(column, field); + updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { @@ -1771,7 +1830,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( } return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), - new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId())); + new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), + getNameMapping(icebergTable)); } return getLatestSnapshotCacheValue(dorisTable); } @@ -1826,6 +1886,43 @@ private static Optional loadTableSchemaCacheValue(ExternalTabl return Optional.of(new IcebergSchemaCacheValue(schema, tmpColumns)); } + /** + * Extract the Iceberg name mapping while retaining the distinction between an absent property + * and a valid empty mapping. + */ + public static Optional>> getNameMapping(Table icebergTable) { + String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); + if (nameMappingJson == null || nameMappingJson.isEmpty()) { + return Optional.empty(); + } + try { + NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); + if (mapping == null) { + return Optional.empty(); + } + Map> result = new HashMap<>(); + extractMappingsFromNameMapping(mapping.asMappedFields(), result); + return Optional.of(result); + } catch (Exception e) { + LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + return Optional.empty(); + } + } + + private static void extractMappingsFromNameMapping( + MappedFields mappingFields, Map> result) { + if (mappingFields == null) { + return; + } + for (MappedField mappedField : mappingFields.fields()) { + // Iceberg permits id-less wrapper entries; only their nested ID-bearing aliases can + // participate in Doris field-id lookup and in the immutable snapshot cache. + if (mappedField.id() != null) { + result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + } + extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + } + } public static boolean isIcebergRowLineageColumn(Column column) { return column.nameEquals(IcebergUtils.ICEBERG_ROW_ID_COL, false) @@ -1862,10 +1959,25 @@ public static Schema appendRowLineageFieldsForV3(Schema schema) { MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER)); } + public static boolean shouldCollectColumnStats(Table table, Schema writerSchema) { + MetricsConfig metricsConfig = MetricsConfig.forTable(table); + if (getFileFormat(table) == FileFormat.ORC) { + // Match the footer collectors: ORC reports top-level collection counts, while Parquet reports leaf fields. + return writerSchema.columns().stream() + .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) + != MetricsModes.None.get()); + } + return TypeUtil.indexById(writerSchema.asStruct()).values().stream() + .filter(field -> field.type().isPrimitiveType()) + .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) + != MetricsModes.None.get()); + } + public static int getFormatVersion(Table table) { int formatVersion = 2; // default format version : 2 - if (table instanceof BaseTable) { - formatVersion = ((BaseTable) table).operations().current().formatVersion(); + if (table instanceof HasTableOperations) { + // TransactionTable exposes the real format version through operations, not table properties. + formatVersion = ((HasTableOperations) table).operations().current().formatVersion(); } else if (table != null && table.properties() != null) { String version = table.properties().get(TableProperties.FORMAT_VERSION); if (version != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java index b67a5911b64384..54a791e7e18133 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java @@ -30,12 +30,21 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsUtil; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.BinaryUtil; +import org.apache.iceberg.util.UnicodeUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,6 +70,12 @@ public static WriteResult convertToWriterResult( // Get table specification information PartitionSpec spec = table.spec(); FileFormat fileFormat = IcebergUtils.getFileFormat(table); + MetricsConfig metricsConfig = MetricsConfig.forTable(table); + Schema schema = table.schema(); + if (IcebergUtils.getFormatVersion(table) >= IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION) { + // Rewrite and merge writers emit v3 lineage columns that are absent from the table schema. + schema = IcebergUtils.appendRowLineageFieldsForV3(schema); + } for (TIcebergCommitData commitData : commitDataList) { //get the files path @@ -70,7 +85,7 @@ public static WriteResult convertToWriterResult( long fileSize = commitData.getFileSize(); long recordCount = commitData.getRowCount(); CommonStatistics stat = new CommonStatistics(recordCount, DEFAULT_FILE_COUNT, fileSize); - Metrics metrics = buildDataFileMetrics(table, fileFormat, commitData); + Metrics metrics = buildDataFileMetrics(commitData, schema, metricsConfig, fileFormat); Optional partitionData = Optional.empty(); //get and check partitionValues when table is partitionedTable if (spec.isPartitioned()) { @@ -153,7 +168,9 @@ private static PartitionData convertToPartitionData( return partitionData; } - private static Metrics buildDataFileMetrics(Table table, FileFormat fileFormat, TIcebergCommitData commitData) { + private static Metrics buildDataFileMetrics( + TIcebergCommitData commitData, Schema schema, MetricsConfig metricsConfig, FileFormat fileFormat) { + Map fieldParents = TypeUtil.indexParents(schema.asStruct()); Map columnSizes = new HashMap<>(); Map valueCounts = new HashMap<>(); Map nullValueCounts = new HashMap<>(); @@ -178,8 +195,100 @@ private static Metrics buildDataFileMetrics(Table table, FileFormat fileFormat, } } - return new Metrics(commitData.getRowCount(), columnSizes, valueCounts, - nullValueCounts, null, lowerBounds, upperBounds); + // Physical file stats may contain every column, but manifest metrics must honor the table's metadata policy. + return new Metrics(commitData.getRowCount(), + filterDisabledMetrics(columnSizes, schema, metricsConfig), + filterLogicalMetrics(valueCounts, schema, metricsConfig, fieldParents), + filterLogicalMetrics(nullValueCounts, schema, metricsConfig, fieldParents), + null, + filterBounds(lowerBounds, schema, metricsConfig, fieldParents, fileFormat, true), + filterBounds(upperBounds, schema, metricsConfig, fieldParents, fileFormat, false)); + } + + private static Map filterDisabledMetrics( + Map metrics, Schema schema, MetricsConfig metricsConfig) { + Map filteredMetrics = new HashMap<>(); + metrics.forEach((fieldId, value) -> { + if (MetricsUtil.metricsMode(schema, metricsConfig, fieldId) != MetricsModes.None.get()) { + filteredMetrics.put(fieldId, value); + } + }); + return filteredMetrics; + } + + private static Map filterLogicalMetrics( + Map metrics, Schema schema, MetricsConfig metricsConfig, + Map fieldParents) { + Map filteredMetrics = new HashMap<>(); + metrics.forEach((fieldId, value) -> { + // Definition-level values below list/map do not represent logical element counts. + if (!isInRepeatedField(fieldId, schema, fieldParents) + && MetricsUtil.metricsMode(schema, metricsConfig, fieldId) != MetricsModes.None.get()) { + filteredMetrics.put(fieldId, value); + } + }); + return filteredMetrics; + } + + private static Map filterBounds( + Map bounds, Schema schema, MetricsConfig metricsConfig, + Map fieldParents, FileFormat fileFormat, boolean lowerBound) { + Map filteredBounds = new HashMap<>(); + bounds.forEach((fieldId, value) -> { + if (isInRepeatedField(fieldId, schema, fieldParents)) { + return; + } + MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema, metricsConfig, fieldId); + if (mode == MetricsModes.None.get() || mode == MetricsModes.Counts.get()) { + return; + } + + ByteBuffer filteredValue = value; + if (mode instanceof MetricsModes.Truncate) { + Type type = schema.findType(fieldId); + int length = ((MetricsModes.Truncate) mode).length(); + // Truncated upper bounds must round up so file pruning cannot exclude matching values. + filteredValue = truncateBound(type, value, length, fileFormat, lowerBound); + } + if (filteredValue != null) { + filteredBounds.put(fieldId, filteredValue); + } + }); + return filteredBounds; + } + + private static boolean isInRepeatedField( + int fieldId, Schema schema, Map fieldParents) { + Integer parentId = fieldId; + while ((parentId = fieldParents.get(parentId)) != null) { + Types.NestedField parent = schema.findField(parentId); + if (parent != null && !parent.type().isStructType()) { + return true; + } + } + return false; + } + + private static ByteBuffer truncateBound( + Type type, ByteBuffer value, int length, FileFormat fileFormat, boolean lowerBound) { + switch (type.typeId()) { + case STRING: + String stringValue = Conversions.fromByteBuffer(type, value).toString(); + String truncatedString = lowerBound + ? UnicodeUtil.truncateStringMin(stringValue, length) + : UnicodeUtil.truncateStringMax(stringValue, length); + // ORC keeps the full maximum when no safe truncated successor exists. + if (!lowerBound && truncatedString == null && fileFormat == FileFormat.ORC) { + return value; + } + return truncatedString == null ? null : Conversions.toByteBuffer(type, truncatedString); + case BINARY: + return lowerBound + ? BinaryUtil.truncateBinaryMin(value, length) + : BinaryUtil.truncateBinaryMax(value, length); + default: + return value; + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java index c1626e915c65bb..65de2f9249df30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java @@ -37,22 +37,20 @@ public class IcebergApiSource implements IcebergSource { private final ExternalTable targetTable; - private final Table originTable; private final TupleDescriptor desc; + private Table originTable; public IcebergApiSource(ExternalTable table, TupleDescriptor desc, Map columnNameToRange) { + if (!(table instanceof IcebergExternalTable) && !(table instanceof IcebergSysExternalTable)) { + throw new IllegalArgumentException( + "Expected Iceberg table but got " + table.getClass().getSimpleName()); + } if (table instanceof IcebergExternalTable) { IcebergExternalTable icebergExtTable = (IcebergExternalTable) table; if (icebergExtTable.isView()) { throw new UnsupportedOperationException("IcebergApiSource does not support view"); } - this.originTable = IcebergUtils.getIcebergTable(icebergExtTable); - } else if (table instanceof IcebergSysExternalTable) { - this.originTable = ((IcebergSysExternalTable) table).getSysIcebergTable(); - } else { - throw new IllegalArgumentException( - "Expected Iceberg table but got " + table.getClass().getSimpleName()); } this.targetTable = table; this.desc = desc; @@ -64,12 +62,19 @@ public TupleDescriptor getDesc() { } @Override - public String getFileFormat() { - return IcebergUtils.getFileFormat(originTable).name(); + public String getFileFormat() throws MetaNotFoundException { + return IcebergUtils.getFileFormat(getIcebergTable()).name(); } @Override - public Table getIcebergTable() throws MetaNotFoundException { + public synchronized Table getIcebergTable() throws MetaNotFoundException { + if (originTable == null) { + if (targetTable instanceof IcebergExternalTable) { + originTable = IcebergUtils.getIcebergTable((IcebergExternalTable) targetTable); + } else { + originTable = ((IcebergSysExternalTable) targetTable).getSysIcebergTable(); + } + } return originTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java index 32de4ebfdd9c0c..083409adda136c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java @@ -54,16 +54,46 @@ public static PositionDelete createPositionDelete(DeleteFile deleteFile) { String deleteFilePath = deleteFile.path().toString(); if (deleteFile.format() == FileFormat.PUFFIN) { + long fileSize = deleteFile.fileSizeInBytes(); + Long contentOffset = deleteFile.contentOffset(); + Long contentLength = deleteFile.contentSizeInBytes(); + validateDeletionVectorMetadata(deleteFilePath, fileSize, contentOffset, contentLength); // The content_offset and content_size_in_bytes fields are used to reference // a specific blob for direct access to a deletion vector. return new DeletionVector(deleteFilePath, positionLowerBound.orElse(-1L), positionUpperBound.orElse(-1L), - deleteFile.fileSizeInBytes(), deleteFile.contentOffset(), deleteFile.contentSizeInBytes()); + fileSize, contentOffset, contentLength); } else { return new PositionDelete(deleteFilePath, positionLowerBound.orElse(-1L), positionUpperBound.orElse(-1L), deleteFile.fileSizeInBytes(), deleteFile.format()); } } + static void validateDeletionVectorMetadata( + String deleteFilePath, long fileSize, Long contentOffset, Long contentLength) { + if (contentOffset == null || contentLength == null) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata misses content offset or length: %s", deleteFilePath)); + } + if (fileSize < 0 || contentOffset < 0 || contentLength < 0) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata must be non-negative, file: %s, file size: %d, " + + "content offset: %d, content length: %d", + deleteFilePath, fileSize, contentOffset, contentLength)); + } + if (contentOffset > Long.MAX_VALUE - contentLength) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata range overflows, file: %s, content offset: %d, " + + "content length: %d", + deleteFilePath, contentOffset, contentLength)); + } + if (contentOffset + contentLength > fileSize) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata range exceeds file size, file: %s, file size: %d, " + + "content offset: %d, content length: %d", + deleteFilePath, fileSize, contentOffset, contentLength)); + } + } + public static EqualityDelete createEqualityDelete(String deleteFilePath, List fieldIds, long fileSize, FileFormat fileformat) { // todo: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java index a87b5408d5938b..cf33119cd60f11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java @@ -29,12 +29,11 @@ public class IcebergHMSSource implements IcebergSource { private final HMSExternalTable hmsTable; private final TupleDescriptor desc; - private final org.apache.iceberg.Table icebergTable; + private org.apache.iceberg.Table icebergTable; public IcebergHMSSource(HMSExternalTable hmsTable, TupleDescriptor desc) { this.hmsTable = hmsTable; this.desc = desc; - this.icebergTable = IcebergUtils.getIcebergTable(hmsTable); } @Override @@ -44,10 +43,13 @@ public TupleDescriptor getDesc() { @Override public String getFileFormat() throws DdlException, MetaNotFoundException { - return IcebergUtils.getFileFormat(icebergTable).name(); + return IcebergUtils.getFileFormat(getIcebergTable()).name(); } - public org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException { + public synchronized org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException { + if (icebergTable == null) { + icebergTable = IcebergUtils.getIcebergTable(hmsTable); + } return icebergTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 4e576a8bd67c9d..d4167d7bc8aa40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -18,9 +18,11 @@ package org.apache.doris.datasource.iceberg.source; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.UserException; @@ -38,35 +40,43 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.profile.IcebergMetricsReporter; import org.apache.doris.datasource.iceberg.source.IcebergDeleteFileFilter.EqualityDelete; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TColumnCategory; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TIcebergDeleteFileDesc; import org.apache.doris.thrift.TIcebergFileDesc; import org.apache.doris.thrift.TPlanNode; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.gson.JsonObject; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.BaseFileScanTask; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.BatchScan; +import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeleteFileIndex; @@ -76,13 +86,16 @@ import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; +import org.apache.iceberg.PositionDeletesScanTask; +import org.apache.iceberg.ScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -91,10 +104,8 @@ import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.mapping.MappedField; -import org.apache.iceberg.mapping.MappedFields; -import org.apache.iceberg.mapping.NameMapping; -import org.apache.iceberg.mapping.NameMappingParser; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.ScanTaskUtil; import org.apache.iceberg.util.SerializationUtil; import org.apache.iceberg.util.TableScanUtil; @@ -102,6 +113,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -117,6 +129,7 @@ public class IcebergScanNode extends FileQueryScanNode { public static final int MIN_DELETE_FILE_SUPPORT_VERSION = 2; + static final int ICEBERG_SCAN_SEMANTICS_VERSION = 1; private static final Logger LOG = LogManager.getLogger(IcebergScanNode.class); private IcebergSource source; @@ -183,6 +196,18 @@ public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckCol scanContext, needCheckColumnPriv, sv); ExternalTable table = (ExternalTable) desc.getTable(); + initIcebergSource(table); + } + + public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, IcebergSysExternalTable sysExternalTable, + SessionVariable sv, ScanContext scanContext) { + super(id, desc, "ICEBERG_SCAN_NODE", StatisticalType.ICEBERG_SCAN_NODE, + scanContext, false, sv); + isSystemTable = true; + initIcebergSource(sysExternalTable); + } + + private void initIcebergSource(ExternalTable table) { if (table instanceof HMSExternalTable) { source = new IcebergHMSSource((HMSExternalTable) table, desc); } else if (table instanceof IcebergExternalTable || table instanceof IcebergSysExternalTable) { @@ -212,59 +237,42 @@ public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckCol @Override protected void doInitialize() throws UserException { - icebergTable = source.getIcebergTable(); - partitionMapInfos = new HashMap<>(); - isPartitionedTable = icebergTable.spec().isPartitioned(); - // Metadata tables (system tables) are not BaseTable instances, so we need to handle this case - if (icebergTable instanceof BaseTable) { - formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); - } else { - // For metadata tables (e.g., snapshots, history), use a default format version - // These tables are always readable regardless of format version - formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION; - } - preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); - storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( - source.getCatalog().getCatalogProperty().getMetastoreProperties(), - source.getCatalog().getCatalogProperty().getStoragePropertiesMap(), - icebergTable - ); - backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); - super.doInitialize(); - } - - /** - * Extract name mapping from Iceberg table properties. - * Returns a map from field ID to list of mapped names. - */ - private Map> extractNameMapping() { - Map> result = new HashMap<>(); + long startTime = System.currentTimeMillis(); try { - String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); - if (nameMappingJson != null && !nameMappingJson.isEmpty()) { - NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); - if (mapping != null) { - // Extract mappings from NameMapping - // NameMapping contains field mappings, we need to convert them to our format - extractMappingsFromNameMapping(mapping.asMappedFields(), result); - } + icebergTable = source.getIcebergTable(); + partitionMapInfos = new HashMap<>(); + isPartitionedTable = icebergTable.spec().isPartitioned(); + // Metadata tables (system tables) are not BaseTable instances, so we need to handle this case + if (icebergTable instanceof BaseTable) { + formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + } else { + // For metadata tables (e.g., snapshots, history), use a default format version + // These tables are always readable regardless of format version + formatVersion = MIN_DELETE_FILE_SUPPORT_VERSION; + } + preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); + storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + source.getCatalog().getCatalogProperty().getMetastoreProperties(), + source.getCatalog().getCatalogProperty().getStoragePropertiesMap(), + icebergTable + ); + backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); } - } catch (Exception e) { - // If name mapping parsing fails, continue without it - LOG.warn("Failed to parse name mapping from Iceberg table properties", e); } - return result; + super.doInitialize(); } - private void extractMappingsFromNameMapping(MappedFields mappingFields, Map> result) { - if (mappingFields == null) { - return; - } - for (MappedField mappedField : mappingFields.fields()) { - result.put(mappedField.id(), new ArrayList<>(mappedField.names())); - extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + private Optional>> extractNameMapping() { + Optional snapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + // The mapping must come from the same metadata generation as the pinned schema; a + // property-only refresh can otherwise change alias semantics within one statement. + return ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue().getNameMapping(); } - + return IcebergUtils.getNameMapping(icebergTable); } @Override @@ -278,12 +286,19 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value()); TIcebergFileDesc fileDesc = new TIcebergFileDesc(); + if (isSystemTable && icebergSplit.isPositionDeleteSystemTableSplit()) { + setIcebergPositionDeleteSysTableParams(rangeDesc, icebergSplit, tableFormatFileDesc, fileDesc); + return; + } if (isSystemTable) { rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI); tableFormatFileDesc.setTableLevelRowCount(-1); fileDesc.setSerializedSplit(icebergSplit.getSerializedSplit()); tableFormatFileDesc.setIcebergParams(fileDesc); rangeDesc.setTableFormatParams(tableFormatFileDesc); + rangeDesc.unsetColumnsFromPath(); + rangeDesc.unsetColumnsFromPathKeys(); + rangeDesc.unsetColumnsFromPathIsNull(); return; } // update for every split file format @@ -378,6 +393,41 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli rangeDesc.setTableFormatParams(tableFormatFileDesc); } + private void setIcebergPositionDeleteSysTableParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit, + TTableFormatFileDesc tableFormatFileDesc, TIcebergFileDesc fileDesc) { + rangeDesc.setFormatType(icebergSplit.getPositionDeleteFileFormat()); + tableFormatFileDesc.setTableLevelRowCount(-1); + fileDesc.setContent(icebergSplit.getPositionDeleteContent()); + + if (icebergSplit.getPartitionSpecId() != null) { + fileDesc.setPartitionSpecId(icebergSplit.getPartitionSpecId()); + } + if (icebergSplit.getPartitionDataJson() != null) { + fileDesc.setPartitionDataJson(icebergSplit.getPartitionDataJson()); + } + + TIcebergDeleteFileDesc deleteFileDesc = new TIcebergDeleteFileDesc(); + deleteFileDesc.setPath(rangeDesc.getPath()); + deleteFileDesc.setOriginalPath(icebergSplit.getPositionDeleteOriginalPath()); + deleteFileDesc.setFileFormat(icebergSplit.getPositionDeleteFileFormat()); + deleteFileDesc.setContent(icebergSplit.getPositionDeleteContent()); + if (icebergSplit.getPositionDeleteContentOffset() != null) { + deleteFileDesc.setContentOffset(icebergSplit.getPositionDeleteContentOffset()); + } + if (icebergSplit.getPositionDeleteContentSizeInBytes() != null) { + deleteFileDesc.setContentSizeInBytes(icebergSplit.getPositionDeleteContentSizeInBytes()); + } + if (icebergSplit.getPositionDeleteReferencedDataFilePath() != null) { + deleteFileDesc.setReferencedDataFilePath(icebergSplit.getPositionDeleteReferencedDataFilePath()); + } + fileDesc.setDeleteFiles(Lists.newArrayList(deleteFileDesc)); + tableFormatFileDesc.setIcebergParams(fileDesc); + rangeDesc.setTableFormatParams(tableFormatFileDesc); + rangeDesc.unsetColumnsFromPath(); + rangeDesc.unsetColumnsFromPathKeys(); + rangeDesc.unsetColumnsFromPathIsNull(); + } + @Override protected List getDeleteFiles(TFileRangeDesc rangeDesc) { List deleteFiles = new ArrayList<>(); @@ -437,29 +487,49 @@ private String getDeleteFileContentType(int content) { public void createScanRangeLocations() throws UserException { super.createScanRangeLocations(); + enableCurrentIcebergScanSemantics(); // Extract name mapping from Iceberg table properties - Map> nameMapping = extractNameMapping(); + Optional>> nameMapping = extractNameMapping(); // Equality-delete keys are hidden scan dependencies and need not appear in the query // projection. Both scanners need the complete current schema to resolve field ids, // historical names, types, and initial defaults when an old data file lacks such a key. ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), - nameMapping, getBase64EncodedInitialDefaultsForScan()); + nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), + getBase64EncodedInitialDefaultsForScan()); + } + + @VisibleForTesting + void enableCurrentIcebergScanSemantics() { + // This explicit capability is the rollout boundary: old FE plans must keep legacy values + // when fragments run on a mixture of old and new BEs. + params.setIcebergScanSemanticsVersion(ICEBERG_SCAN_SEMANTICS_VERSION); } @VisibleForTesting Map getBase64EncodedInitialDefaultsForScan() throws UserException { - TableScan tableScan = createTableScan(); - Snapshot snapshot = tableScan.snapshot(); - // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. - // Resolve the selected snapshot's schema id explicitly so this metadata describes the same - // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make - // BE decode a historical non-binary default as Base64, or fail to decode a binary default. - Schema scanSchema = snapshot == null - ? tableScan.schema() - : tableScan.table().schemas().get(snapshot.schemaId()); + if (isSystemTable) { + // System-table columns are derived from the metadata table schema. Some metadata + // tables, such as position_deletes, do not support Table.newScan(). Use the same + // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. + return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); + } + IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); + Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Schema scanSchema = null; + if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + .getSnapshotCacheValue().getSnapshot().getSchemaId(); + scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); + } else { + scanSchema = selectedSnapshot == null + ? icebergTable.schema() + : icebergTable.schemas().get(selectedSnapshot.getSchemaId()); + } + // A branch can expose a schema newer than its data snapshot. The statement-pinned schema + // produced the target columns, so default markers must not be recomputed from that snapshot. return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); } @Override @@ -515,15 +585,23 @@ public void doStartSplit() throws UserException { try { preExecutionAuthenticator.execute( () -> { - CloseableIterable fileScanTasks = planFileScanTask(scan); - taskRef.set(fileScanTasks); - - CloseableIterator iterator = fileScanTasks.iterator(); - while (splitAssignment.needMoreSplit() && iterator.hasNext()) { - try { - splitAssignment.addToQueue(Lists.newArrayList(createIcebergSplit(iterator.next()))); - } catch (UserException e) { - throw new RuntimeException(e); + long startTime = System.currentTimeMillis(); + try { + CloseableIterable fileScanTasks = planFileScanTask(scan); + taskRef.set(fileScanTasks); + CloseableIterator iterator = fileScanTasks.iterator(); + while (splitAssignment.needMoreSplit() && iterator.hasNext()) { + try { + splitAssignment.addToQueue( + Lists.newArrayList(createIcebergSplit(iterator.next()))); + } catch (UserException e) { + throw new RuntimeException(e); + } + } + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime( + System.currentTimeMillis() - startTime); } } } @@ -625,11 +703,11 @@ private CloseableIterable splitFiles(TableScan scan) { return TableScanUtil.splitFiles(CloseableIterable.withNoopClose(fileScanTaskList), targetSplitSize); } - private long determineTargetFileSplitSize(Iterable tasks) { + private long determineTargetFileSplitSize(Iterable> tasks) { long result = sessionVariable.getMaxInitialSplitSize(); long accumulatedTotalFileSize = 0; boolean exceedInitialThreshold = false; - for (FileScanTask task : tasks) { + for (ContentScanTask task : tasks) { accumulatedTotalFileSize += ScanTaskUtil.contentSizeInBytes(task.file()); if (!exceedInitialThreshold && accumulatedTotalFileSize >= sessionVariable.getMaxSplitSize() * sessionVariable.getMaxInitialSplitNum()) { @@ -643,6 +721,13 @@ private long determineTargetFileSplitSize(Iterable tasks) { return result; } + private long determinePositionDeleteTargetSplitSize(Iterable tasks) { + if (sessionVariable.getFileSplitSize() > 0) { + return sessionVariable.getFileSplitSize(); + } + return determineTargetFileSplitSize(tasks); + } + private CloseableIterable planFileScanTaskWithManifestCache(TableScan scan) throws IOException { // Get the snapshot from the scan; return empty if no snapshot exists Snapshot snapshot = scan.snapshot(); @@ -887,14 +972,152 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { return split; } - private Split createIcebergSysSplit(FileScanTask fileScanTask) { - long rowCount = fileScanTask.file() == null ? 1 : fileScanTask.file().recordCount(); + private Split createIcebergSysSplit(ScanTask scanTask) { + long rowCount = Math.max(scanTask.estimatedRowsCount(), 1L); + if (scanTask.isFileScanTask() && scanTask.asFileScanTask().file() != null) { + rowCount = Math.max(scanTask.asFileScanTask().file().recordCount(), 1L); + } IcebergSplit split = IcebergSplit.newSysTableSplit( - SerializationUtil.serializeToBase64(fileScanTask), rowCount); + SerializationUtil.serializeToBase64(scanTask), rowCount); split.setTableFormatType(TableFormatType.ICEBERG); return split; } + private Split createIcebergPositionDeleteSysSplit(PositionDeletesScanTask task) throws UserException { + DeleteFile deleteFile = task.file(); + String originalPath = deleteFile.path().toString(); + LocationPath locationPath = createLocationPathWithCache(originalPath); + IcebergSplit split = IcebergSplit.newPositionDeleteSysTableSplit( + locationPath, task.start(), task.length(), deleteFile.fileSizeInBytes(), + storagePropertiesMap, originalPath); + split.setTableFormatType(TableFormatType.ICEBERG); + split.setPositionDeleteFileFormat(getNativePositionDeleteFileFormat(deleteFile.format())); + split.setPositionDeleteOriginalPath(originalPath); + if (deleteFile.format() == FileFormat.PUFFIN) { + Long contentOffset = deleteFile.contentOffset(); + Long contentLength = deleteFile.contentSizeInBytes(); + IcebergDeleteFileFilter.validateDeletionVectorMetadata( + originalPath, deleteFile.fileSizeInBytes(), contentOffset, contentLength); + split.setPositionDeleteContent(IcebergDeleteFileFilter.DeletionVector.type()); + split.setPositionDeleteReferencedDataFilePath(deleteFile.referencedDataFile()); + split.setPositionDeleteContentOffset(contentOffset); + split.setPositionDeleteContentSizeInBytes(contentLength); + } else { + split.setPositionDeleteContent(IcebergDeleteFileFilter.PositionDelete.type()); + } + + split.setPartitionSpecId(deleteFile.specId()); + PartitionSpec partitionSpec = icebergTable.specs().get(deleteFile.specId()); + Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s", + deleteFile.specId(), icebergTable.name()); + if (partitionSpec.isPartitioned() && deleteFile.partition() != null + && isPositionDeletesPartitionColumnRequested()) { + split.setPartitionDataJson(getPartitionDataObjectJson( + (PartitionData) deleteFile.partition(), partitionSpec, + getPositionDeletesOutputPartitionFields())); + } + return split; + } + + @SuppressWarnings("unchecked") + private Iterable splitPositionDeleteScanTask(PositionDeletesScanTask task) { + return ((SplittableScanTask) task).split(targetSplitSize); + } + + private TFileFormatType getNativePositionDeleteFileFormat(FileFormat fileFormat) { + if (fileFormat == FileFormat.PARQUET || fileFormat == FileFormat.PUFFIN) { + return TFileFormatType.FORMAT_PARQUET; + } else if (fileFormat == FileFormat.ORC) { + return TFileFormatType.FORMAT_ORC; + } + throw new UnsupportedOperationException( + "Unsupported Iceberg position delete file format: " + fileFormat); + } + + private List getPositionDeletesOutputPartitionFields() { + NestedField partitionField = icebergTable.schema().findField("partition"); + Preconditions.checkNotNull(partitionField, + "Partition field not found in Iceberg position_deletes metadata table schema"); + return partitionField.type().asNestedType().fields(); + } + + private boolean isPositionDeletesPartitionColumnRequested() { + return desc.getSlots().stream() + .anyMatch(slot -> "partition".equalsIgnoreCase(slot.getColumn().getName())); + } + + private String getPartitionDataObjectJson(PartitionData partitionData, PartitionSpec partitionSpec, + List outputPartitionFields) throws UserException { + List partitionTypes = partitionData.getPartitionType().asNestedType().fields(); + boolean enableMappingVarbinary = getEnableMappingVarbinary(); + for (int i = 0; i < partitionTypes.size(); i++) { + Type type = partitionTypes.get(i).type(); + if (partitionData.get(i) != null && (type.typeId() == Type.TypeID.BINARY + || type.typeId() == Type.TypeID.FIXED + || (type.typeId() == Type.TypeID.UUID && enableMappingVarbinary))) { + throw new UserException("Iceberg position_deletes cannot materialize non-null partition field '" + + partitionTypes.get(i).name() + "' of type " + type + + " without a binary-safe partition transport"); + } + } + List partitionValues = IcebergUtils.getPartitionValues( + partitionData, partitionSpec, sessionVariable.getTimeZone()); + Map partitionValueByFieldId = new HashMap<>(); + List fields = partitionSpec.fields(); + for (int i = 0; i < fields.size(); i++) { + partitionValueByFieldId.put(fields.get(i).fieldId(), + getPartitionJsonValue(partitionTypes.get(i).type(), partitionValues.get(i))); + } + JsonObject partitionJson = new JsonObject(); + for (NestedField outputPartitionField : outputPartitionFields) { + partitionJson.add(outputPartitionField.name(), + GsonUtils.GSON.toJsonTree(partitionValueByFieldId.get(outputPartitionField.fieldId()))); + } + return GsonUtils.GSON.toJson(partitionJson); + } + + private static Object getPartitionJsonValue(Type type, String partitionValue) { + if (partitionValue == null) { + return null; + } + switch (type.typeId()) { + case BOOLEAN: + return Boolean.parseBoolean(partitionValue); + case INTEGER: + return Integer.parseInt(partitionValue); + case LONG: + return Long.parseLong(partitionValue); + case FLOAT: + return Float.parseFloat(partitionValue); + case DOUBLE: + return Double.parseDouble(partitionValue); + case DECIMAL: + return new BigDecimal(partitionValue); + case STRING: + case UUID: + case DATE: + case TIME: + case TIMESTAMP: + return partitionValue; + default: + return partitionValue; + } + } + + @Override + protected TColumnCategory classifyColumn(SlotDescriptor slot, List partitionKeys) { + if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName())) { + return TColumnCategory.SYNTHESIZED; + } + if (slot.getColumn().getName().startsWith(Column.GLOBAL_ROWID_COL)) { + return TColumnCategory.SYNTHESIZED; + } + if (IcebergUtils.isIcebergRowLineageColumn(slot.getColumn())) { + return TColumnCategory.GENERATED; + } + return super.classifyColumn(slot, partitionKeys); + } + private List doGetSplits(int numBackends) throws UserException { if (isSystemTable) { return doGetSystemTableSplits(); @@ -916,10 +1139,12 @@ private List doGetSplits(int numBackends) throws UserException { // Normal table scan planning TableScan scan = createTableScan(); + long startTime = System.currentTimeMillis(); try (CloseableIterable fileScanTasks = planFileScanTask(scan)) { if (tableLevelPushDownCount) { int needSplitCnt = countFromSnapshot < COUNT_WITH_PARALLEL_SPLITS - ? 1 : sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()) * numBackends; + ? 1 : sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()) + * numBackends; for (FileScanTask next : fileScanTasks) { splits.add(createIcebergSplit(next)); if (splits.size() >= needSplitCnt) { @@ -935,6 +1160,10 @@ private List doGetSplits(int numBackends) throws UserException { } } catch (IOException e) { throw new UserException(e.getMessage(), e.getCause()); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } selectedPartitionNum = partitionMapInfos.size(); @@ -943,17 +1172,94 @@ private List doGetSplits(int numBackends) throws UserException { } private List doGetSystemTableSplits() throws UserException { + if (isPositionDeletesSystemTable()) { + return doGetPositionDeletesSystemTableSplits(); + } List splits = new ArrayList<>(); TableScan scan = createTableScan(); + long startTime = System.currentTimeMillis(); try (CloseableIterable fileScanTasks = scan.planFiles()) { fileScanTasks.forEach(task -> splits.add(createIcebergSysSplit(task))); } catch (IOException e) { throw new UserException(e.getMessage(), e); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } selectedPartitionNum = 0; return splits; } + private boolean isPositionDeletesSystemTable() { + TableIf targetTable = source.getTargetTable(); + return targetTable instanceof IcebergSysExternalTable + && "position_deletes".equalsIgnoreCase(((IcebergSysExternalTable) targetTable).getSysTableType()); + } + + private List doGetPositionDeletesSystemTableSplits() throws UserException { + checkPositionDeletesBackendCompatibility(backendPolicy.getBackends()); + List splits = new ArrayList<>(); + List positionDeleteTasks = new ArrayList<>(); + BatchScan scan = icebergTable.newBatchScan().metricsReporter(new IcebergMetricsReporter()); + + IcebergTableQueryInfo info = getSpecifiedSnapshot(); + if (info != null) { + if (info.getRef() != null) { + scan = scan.useRef(info.getRef()); + } else { + scan = scan.useSnapshot(info.getSnapshotId()); + } + } + + List expressions = new ArrayList<>(); + for (Expr conjunct : conjuncts) { + Expression expression = IcebergUtils.convertToIcebergExpr(conjunct, icebergTable.schema()); + if (expression != null) { + expressions.add(expression); + } + } + for (Expression predicate : expressions) { + scan = scan.filter(predicate); + this.pushdownIcebergPredicates.add(predicate.toString()); + } + + long startTime = System.currentTimeMillis(); + scan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); + try (CloseableIterable scanTasks = scan.planFiles()) { + for (ScanTask task : scanTasks) { + if (!(task instanceof PositionDeletesScanTask)) { + throw new UserException("Unexpected Iceberg position_deletes scan task: " + task); + } + positionDeleteTasks.add((PositionDeletesScanTask) task); + } + } catch (IOException e) { + throw new UserException(e.getMessage(), e); + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } + } + targetSplitSize = determinePositionDeleteTargetSplitSize(positionDeleteTasks); + for (PositionDeletesScanTask task : positionDeleteTasks) { + for (PositionDeletesScanTask splitTask : splitPositionDeleteScanTask(task)) { + splits.add(createIcebergPositionDeleteSysSplit(splitTask)); + } + } + selectedPartitionNum = 0; + return splits; + } + + @VisibleForTesting + static void checkPositionDeletesBackendCompatibility(Iterable backends) throws UserException { + for (Backend backend : backends) { + if (backend.isSmoothUpgradeSrc()) { + throw new UserException("Iceberg position_deletes system table is unavailable while backend " + + backend.getId() + " is a smooth upgrade source"); + } + } + } + @Override public boolean isBatchMode() { if (isSystemTable) { @@ -964,8 +1270,7 @@ public boolean isBatchMode() { if (cached != null) { return cached; } - TPushAggOp aggOp = getPushDownAggNoGroupingOp(); - if (aggOp.equals(TPushAggOp.COUNT)) { + if (isTableLevelCountStarPushdown()) { try { countFromSnapshot = getCountFromSnapshot(); } catch (UserException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java index 2987b12b1d52be..eeeff694b8ebce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java @@ -20,6 +20,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.FileSplit; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.thrift.TFileFormatType; import lombok.Data; import org.apache.iceberg.DeleteFile; @@ -55,6 +56,13 @@ public class IcebergSplit extends FileSplit { private String serializedSplit; // maybe mixed file format type in one table. so need record it for every split private FileFormat splitFileFormat; + private boolean positionDeleteSystemTableSplit = false; + private TFileFormatType positionDeleteFileFormat; + private int positionDeleteContent; + private String positionDeleteOriginalPath; + private String positionDeleteReferencedDataFilePath; + private Long positionDeleteContentOffset; + private Long positionDeleteContentSizeInBytes; // File path will be changed if the file is modified, so there's no need to get modification time. public IcebergSplit(LocationPath file, long start, long length, long fileLength, String[] hosts, @@ -81,4 +89,13 @@ public static IcebergSplit newSysTableSplit(String serializedSplit, long rowCoun split.setSelfSplitWeight(Math.max(rowCount, 1L)); return split; } + + public static IcebergSplit newPositionDeleteSysTableSplit(LocationPath file, long start, long length, + long fileLength, Map config, String originalPath) { + IcebergSplit split = new IcebergSplit(file, start, length, fileLength, null, null, config, + Collections.emptyList(), originalPath); + split.setPositionDeleteSystemTableSplit(true); + split.setSelfSplitWeight(Math.max(length, 1L)); + return split; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java index 3837b7bbc7979f..fbe2889d8cebaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java @@ -39,12 +39,9 @@ public class JdbcClickHouseClient extends JdbcClient { protected JdbcClickHouseClient(JdbcClientConfig jdbcClientConfig) { super(jdbcClientConfig); try (Connection conn = getConnection()) { - String jdbcUrl = conn.getMetaData().getURL(); - if (!isNewClickHouseDriver(getJdbcDriverVersion())) { - this.databaseTermIsCatalog = false; - } else { - this.databaseTermIsCatalog = "catalog".equalsIgnoreCase(getDatabaseTermFromUrl(jdbcUrl)); - } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + this.databaseTermIsCatalog = isDatabaseTermCatalog( + databaseMetaData, databaseMetaData.getDriverVersion()); } catch (SQLException e) { throw new JdbcClientException("Failed to initialize JdbcClickHouseClient: %s", e.getMessage()); } @@ -128,7 +125,8 @@ protected String getRemoteDatabaseName(ResultSet resultSet) throws SQLException @Override protected String[] getTableTypes() { - return new String[] {"TABLE", "VIEW", "SYSTEM TABLE"}; + // ClickHouse JDBC V2 filters engines by these vendor-specific table type names. + return new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}; } @Override @@ -236,14 +234,9 @@ private static boolean isNewClickHouseDriver(String driverVersion) { } } - /** - * Extract databaseterm parameters from the jdbc url. - */ - private String getDatabaseTermFromUrl(String jdbcUrl) { - if (jdbcUrl != null && jdbcUrl.toLowerCase().contains("databaseterm=schema")) { - return "schema"; - } - return "catalog"; + static boolean isDatabaseTermCatalog(DatabaseMetaData databaseMetaData, String driverVersion) + throws SQLException { + return isNewClickHouseDriver(driverVersion) && databaseMetaData.supportsCatalogsInDataManipulation(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index a8e331f9e468f6..067f27c664d784 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -49,7 +49,6 @@ import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; import org.apache.doris.thrift.TPaimonReaderType; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.annotations.VisibleForTesting; @@ -175,6 +174,7 @@ public PaimonScanNode(PlanNodeId id, @Override protected void doInitialize() throws UserException { super.doInitialize(); + long startTime = System.currentTimeMillis(); source = new PaimonSource(desc); serializedTable = PaimonUtil.encodeObjectToString(source.getPaimonTable()); // Todo: Get the current schema id of the table, instead of using -1. @@ -187,6 +187,9 @@ protected void doInitialize() throws UserException { ); backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); backendPaimonOptions = getBackendPaimonOptions(); + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetTableMetaTime(System.currentTimeMillis() - startTime); + } } @VisibleForTesting @@ -414,7 +417,9 @@ public List getSplits(int numBackends) throws UserException { ++paimonSplitNum; } - boolean applyCountPushdown = getPushDownAggNoGroupingOp() == TPushAggOp.COUNT; + // Merged row counts contain only COUNT(*) semantics. COUNT(col) must keep every DataSplit + // because BE will read the argument column to account for NULL and schema-mapping rules. + boolean applyCountPushdown = isTableLevelCountStarPushdown(); // Used to avoid repeatedly calculating partition info map for the same // partition data. // And for counting the number of selected partitions for this paimon table. @@ -612,26 +617,33 @@ public Map getIncrReadParams() throws UserException { @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { - Table paimonTable = getProcessedTable(); - List fieldNames = paimonTable.rowType().getFieldNames(); - int[] projected = desc.getSlots().stream().mapToInt( - slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) - .filter(i -> i >= 0) - .toArray(); - ReadBuilder readBuilder = paimonTable.newReadBuilder(); - TableScan scan = readBuilder.withFilter(predicates) - .withProjection(projected) - .newScan(); - PaimonMetricRegistry registry = new PaimonMetricRegistry(); - if (scan instanceof InnerTableScan) { - scan = ((InnerTableScan) scan).withMetricRegistry(registry); - } - List splits = scan.plan().splits(); - PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); - if (!registry.getAllGroups().isEmpty()) { - registry.clear(); + long startTime = System.currentTimeMillis(); + try { + Table paimonTable = getProcessedTable(); + List fieldNames = paimonTable.rowType().getFieldNames(); + int[] projected = desc.getSlots().stream().mapToInt( + slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) + .filter(i -> i >= 0) + .toArray(); + ReadBuilder readBuilder = paimonTable.newReadBuilder(); + TableScan scan = readBuilder.withFilter(predicates) + .withProjection(projected) + .newScan(); + PaimonMetricRegistry registry = new PaimonMetricRegistry(); + if (scan instanceof InnerTableScan) { + scan = ((InnerTableScan) scan).withMetricRegistry(registry); + } + List splits = scan.plan().splits(); + PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); + if (!registry.getAllGroups().isEmpty()) { + registry.clear(); + } + return splits; + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } - return splits; } @VisibleForTesting diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java index 72a739e7b9a64b..5115c2f9ec6f95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/systable/IcebergSysTable.java @@ -20,7 +20,6 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.iceberg.MetadataTableType; @@ -40,27 +39,20 @@ * @see org.apache.iceberg.MetadataTableType for all supported system table types */ public class IcebergSysTable extends NativeSysTable { - public static final String POSITION_DELETES = MetadataTableType.POSITION_DELETES.name().toLowerCase(Locale.ROOT); - /** * All supported Iceberg system tables. * Key is the system table name (e.g., "snapshots", "history"). */ public static final Map SUPPORTED_SYS_TABLES = Collections.unmodifiableMap( Arrays.stream(MetadataTableType.values()) - .filter(type -> type != MetadataTableType.POSITION_DELETES) - .map(type -> new IcebergSysTable(type.name().toLowerCase(Locale.ROOT), true)) + .map(type -> new IcebergSysTable(type.name().toLowerCase(Locale.ROOT))) .collect(Collectors.toMap(SysTable::getSysTableName, Function.identity()))); - public static final SysTable UNSUPPORTED_POSITION_DELETES_TABLE = - new IcebergSysTable(POSITION_DELETES, false); private final String tableName; - private final boolean supported; - private IcebergSysTable(String tableName, boolean supported) { + private IcebergSysTable(String tableName) { super(tableName); this.tableName = tableName; - this.supported = supported; } @Override @@ -70,9 +62,6 @@ public String getSysTableName() { @Override public ExternalTable createSysExternalTable(ExternalTable sourceTable) { - if (!supported) { - throw new AnalysisException("SysTable " + tableName + " is not supported yet"); - } if (!(sourceTable instanceof IcebergExternalTable)) { throw new IllegalArgumentException( "Expected IcebergExternalTable but got " + sourceTable.getClass().getSimpleName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 6ba1b5d4a32821..71fbb2fe53dff3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -45,7 +45,6 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileType; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.collect.Lists; @@ -141,9 +140,9 @@ public List getSplits(int numBackends) throws UserException { List fileStatuses = tableValuedFunction.getFileStatuses(); - // Push down count optimization. + // Avoid splitting only for table-level COUNT(*). COUNT(column) still reads column data. boolean needSplit = true; - if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT) { + if (isTableLevelCountStarPushdown()) { int parallelNum = sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()); int totalFileNum = fileStatuses.size(); needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index c835e1dfc71e8b..edfed3c249ec4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -878,6 +878,11 @@ private PlanFragment getPlanFragmentForPhysicalFileScan(PhysicalFileScan fileSca scanNode.setNereidsId(fileScan.getId()); context.getNereidsIdToPlanNodeIdMap().put(fileScan.getId(), scanNode.getId()); scanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(fileScan.getRelationId())); + scanNode.setPushDownCountSlotIds(context.getRelationPushCountArgumentExprIds(fileScan.getRelationId()) + .stream() + .map(exprId -> Objects.requireNonNull(context.findSlotRef(exprId), + "missing slot for pushed-down COUNT argument " + exprId).getSlotId()) + .collect(Collectors.toList())); TableNameInfo tableNameInfo = new TableNameInfo(null, "", ""); TableRefInfo ref = new TableRefInfo(tableNameInfo, null, null); @@ -1434,6 +1439,10 @@ public PlanFragment visitPhysicalStorageLayerAggregate( context.setRelationPushAggOp( storageLayerAggregate.getRelation().getRelationId(), pushAggOp); + context.setRelationPushCountArgumentExprIds( + storageLayerAggregate.getRelation().getRelationId(), + pushAggOp == TPushAggOp.COUNT + ? storageLayerAggregate.getCountArgumentExprIds() : ImmutableList.of()); PlanFragment planFragment = storageLayerAggregate.getRelation().accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 7420524239d465..55496fa2ac5c98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -55,6 +55,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -113,6 +114,7 @@ public class PlanTranslatorContext { private final Map cteScanNodeMap = Maps.newHashMap(); private final Map tablePushAggOp = Maps.newHashMap(); + private final Map> tablePushCountArgumentExprIds = Maps.newHashMap(); private final Map> statsUnknownColumnsMap = Maps.newHashMap(); private final RuntimeFilterContextV2 runtimeFilterV2Context; @@ -384,6 +386,14 @@ public TPushAggOp getRelationPushAggOp(RelationId relationId) { return tablePushAggOp.getOrDefault(relationId, TPushAggOp.NONE); } + public void setRelationPushCountArgumentExprIds(RelationId relationId, List exprIds) { + tablePushCountArgumentExprIds.put(relationId, Lists.newArrayList(exprIds)); + } + + public List getRelationPushCountArgumentExprIds(RelationId relationId) { + return tablePushCountArgumentExprIds.getOrDefault(relationId, Collections.emptyList()); + } + public boolean isTopMaterializeNode() { return isTopMaterializeNode; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 22356c98e760d6..462806e3294759 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Or; @@ -569,6 +570,8 @@ private LogicalAggregate storageLayerAggregate( Map, PushDownAggOp> supportedAgg = PushDownAggOp.supportedFunctions(); boolean containsCount = false; + boolean containsCountStar = false; + boolean countHasCastArgument = false; Set checkNullSlots = new HashSet<>(); Set expressionAfterProject = new HashSet<>(); @@ -584,12 +587,15 @@ private LogicalAggregate storageLayerAggregate( // Check if contains Count function if (functionClass.equals(Count.class)) { containsCount = true; - if (!function.getArguments().isEmpty()) { + if (function.getArguments().isEmpty()) { + containsCountStar = true; + } else { Expression arg0 = function.getArguments().get(0); if (arg0 instanceof SlotReference) { checkNullSlots.add((SlotReference) arg0); expressionAfterProject.add(arg0); } else if (arg0 instanceof Cast) { + countHasCastArgument = true; Expression child0 = arg0.child(0); if (child0 instanceof SlotReference) { checkNullSlots.add((SlotReference) child0); @@ -666,6 +672,7 @@ private LogicalAggregate storageLayerAggregate( return canNotPush; } else { if (needCheckSlotNull) { + countHasCastArgument = true; checkNullSlots.add((SlotReference) argument.child(0)); } } @@ -676,6 +683,16 @@ private LogicalAggregate storageLayerAggregate( argumentsOfAggregateFunction = processedExpressions; } + // File aggregate metadata can describe COUNT(*) or COUNT(file_column), but it cannot + // describe the CAST wrapped around a COUNT argument. Dropping that CAST is incorrect even + // when the source column is NOT NULL. For example, a non-null DOUBLE value outside the INT + // range becomes NULL for CAST(double_col AS INT), so COUNT(CAST(double_col AS INT)) must + // exclude it while a footer-level COUNT(double_col) would include it. Keep OLAP's existing + // storage-layer behavior unchanged, and make external files evaluate the CAST normally. + if (logicalScan instanceof LogicalFileScan && countHasCastArgument) { + return canNotPush; + } + Set pushDownAggOps = functionClasses.stream() .map(supportedAgg::get) .collect(Collectors.toSet()); @@ -689,6 +706,12 @@ private LogicalAggregate storageLayerAggregate( List usedSlotInTable = (List) Project.findProject(aggUsedSlots, logicalScan.getOutput()); + // COUNT(*) has no aggregate arguments, even though later column pruning retains one + // arbitrary scan slot. Preserve the semantic arguments here so the BE never needs to infer + // COUNT(col) from the post-pruning scan shape. + List countArgumentExprIds = mergeOp == PushDownAggOp.COUNT + ? usedSlotInTable.stream().map(SlotReference::getExprId).collect(Collectors.toList()) + : ImmutableList.of(); for (SlotReference slot : usedSlotInTable) { Optional optionalColumn = slot.getOriginalColumn(); @@ -714,10 +737,21 @@ private LogicalAggregate storageLayerAggregate( } } if (mergeOp == PushDownAggOp.COUNT || mergeOp == PushDownAggOp.MIX) { - // NULL value behavior in `count` function is zero, so - // we should not use row_count to speed up query. the col - // must be not null - if (column.isAllowNull() && checkNullSlots.contains(slot)) { + if (logicalScan instanceof LogicalFileScan && mergeOp == PushDownAggOp.COUNT + && containsCountStar && column.isAllowNull() && checkNullSlots.contains(slot)) { + // One metadata cardinality cannot represent both COUNT(*) and the smaller + // COUNT(nullable_col); synthetic rows would make one upper aggregate wrong. + return canNotPush; + } + // Nullable file COUNT is exact only when this query is routed to FileScannerV2, + // which carries the semantic argument and counts definition levels. Gating on the + // session switch keeps V1 on its original full-column evaluation path. + boolean supportsNullableFileCount = logicalScan instanceof LogicalFileScan + && mergeOp == PushDownAggOp.COUNT + && cascadesContext.getConnectContext() != null + && cascadesContext.getConnectContext().getSessionVariable().enableFileScannerV2; + if (column.isAllowNull() && checkNullSlots.contains(slot) + && !supportsNullableFileCount) { return canNotPush; } } @@ -732,11 +766,12 @@ private LogicalAggregate storageLayerAggregate( if (project != null) { return aggregate.withChildren(ImmutableList.of( project.withChildren( - ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp))) + ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalScan, mergeOp, countArgumentExprIds))) )); } else { return aggregate.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate(physicalScan, mergeOp) + new PhysicalStorageLayerAggregate(physicalScan, mergeOp, countArgumentExprIds) )); } @@ -748,11 +783,12 @@ private LogicalAggregate storageLayerAggregate( if (project != null) { return aggregate.withChildren(ImmutableList.of( project.withChildren( - ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp))) + ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalScan, mergeOp, countArgumentExprIds))) )); } else { return aggregate.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate(physicalScan, mergeOp) + new PhysicalStorageLayerAggregate(physicalScan, mergeOp, countArgumentExprIds) )); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index df88e84f5e6146..35d87e65279b59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.info.TableNameInfo; @@ -378,6 +379,9 @@ private void checkEngineWithCatalog() { + " Make sure 'engine' type is specified when use the catalog: " + ctlName); } } + if (Strings.isNullOrEmpty(ctlName)) { + return; + } CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); if (catalog instanceof HMSExternalCatalog && !engineName.equals(ENGINE_HIVE)) { throw new AnalysisException("Hms type catalog can only use `hive` engine."); @@ -788,6 +792,10 @@ public void validate(ConnectContext ctx) { + "and you can use 'bucket(num, column)' in 'PARTITIONED BY'."); } + if (engineName.equalsIgnoreCase(ENGINE_ICEBERG)) { + validateIcebergRowLineageColumns(); + } + // Validate Iceberg sort order columns if (sortOrderFields != null && !sortOrderFields.isEmpty()) { if (!engineName.equalsIgnoreCase(ENGINE_ICEBERG)) { @@ -1090,6 +1098,34 @@ private void validateKeyColumns() { } } + /** + * Validate that Iceberg v3 tables do not define row lineage reserved columns. + */ + public void validateIcebergRowLineageColumns(int formatVersion) { + if (formatVersion < IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION) { + return; + } + for (ColumnDefinition columnDef : columns) { + if (IcebergUtils.isIcebergRowLineageColumn(columnDef.getName())) { + throw new AnalysisException("Cannot create Iceberg v" + formatVersion + + " table with reserved row lineage column: " + columnDef.getName()); + } + } + } + + private void validateIcebergRowLineageColumns() { + validateIcebergRowLineageColumns(getEffectiveIcebergFormatVersion()); + } + + private int getEffectiveIcebergFormatVersion() { + CatalogIf catalog = Strings.isNullOrEmpty(ctlName) ? null + : Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); + if (catalog instanceof IcebergExternalCatalog) { + return IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalog.getProperties()); + } + return IcebergUtils.getEffectiveIcebergFormatVersion(properties, Collections.emptyMap()); + } + /** * analyzeEngine */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java index fad35834d90cf0..51a8bf054f5294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java @@ -170,7 +170,7 @@ public PhysicalLazyMaterialize(CHILD_TYPE child, Column originalColumn = materializeMap.get(lazySlot).baseSlot.getOriginalColumn().get(); outputBuilder.add(((SlotReference) lazySlot).withColumn(originalColumn)); lazyColumnForRel.add(originalColumn); - lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(lazySlot.getName())); + lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(originalColumn.getName())); lazySlotLocationForRel.add(loc); loc++; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 2385c0601e633f..ac2d77a4fce083 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; @@ -42,21 +43,30 @@ public class PhysicalStorageLayerAggregate extends PhysicalCatalogRelation { private final PhysicalCatalogRelation relation; private final PushDownAggOp aggOp; + private final List countArgumentExprIds; public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp) { + this(relation, aggOp, ImmutableList.of()); + } + + public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, + List countArgumentExprIds) { super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), Optional.empty(), relation.getLogicalProperties(), ImmutableList.of()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); + this.countArgumentExprIds = ImmutableList.copyOf(countArgumentExprIds); } public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, + List countArgumentExprIds, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), groupExpression, logicalProperties, physicalProperties, statistics, ImmutableList.of()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); + this.countArgumentExprIds = ImmutableList.copyOf(countArgumentExprIds); } public PhysicalRelation getRelation() { @@ -67,6 +77,10 @@ public PushDownAggOp getAggOp() { return aggOp; } + public List getCountArgumentExprIds() { + return countArgumentExprIds; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalStorageLayerAggregate(this, context); @@ -82,20 +96,21 @@ public String toString() { } public PhysicalStorageLayerAggregate withPhysicalOlapScan(PhysicalOlapScan physicalOlapScan) { - return new PhysicalStorageLayerAggregate(physicalOlapScan, aggOp); + // branch-4.1 predates copyWithSameId; its plan-copy contract creates a fresh node ID. + return new PhysicalStorageLayerAggregate(physicalOlapScan, aggOp, countArgumentExprIds); } @Override public PhysicalStorageLayerAggregate withGroupExpression(Optional groupExpression) { - return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression, - getLogicalProperties(), physicalProperties, statistics); + return new PhysicalStorageLayerAggregate(relation, aggOp, countArgumentExprIds, + groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression, - logicalProperties.get(), physicalProperties, statistics); + return new PhysicalStorageLayerAggregate(relation, aggOp, countArgumentExprIds, + groupExpression, logicalProperties.get(), physicalProperties, statistics); } @Override @@ -103,7 +118,7 @@ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalPr Statistics statistics) { return new PhysicalStorageLayerAggregate( (PhysicalCatalogRelation) relation.withPhysicalPropertiesAndStats(null, statistics), - aggOp, groupExpression, + aggOp, countArgumentExprIds, groupExpression, getLogicalProperties(), physicalProperties, statistics); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java index 4af4ba17e18578..9775f81c172fae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java @@ -131,6 +131,7 @@ public void bindDataSink(Optional insertCtx) } tSink.setFormatVersion(formatVersion); tSink.setSchemaJson(SchemaParser.toJson(schema)); + tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java index 0f3b1bb24d26bc..b7d3da47cb4d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java @@ -135,6 +135,7 @@ public void bindDataSink(Optional insertCtx) schema = IcebergUtils.appendRowLineageFieldsForV3(schema); } tSink.setSchemaJson(SchemaParser.toJson(schema)); + tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index d7e63e6e2bb266..bd03966d5576a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -59,6 +59,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -866,11 +867,18 @@ public void setCardinalityAfterFilter(long cardinalityAfterFilter) { } protected TPushAggOp pushDownAggNoGroupingOp = TPushAggOp.NONE; + // Explicit COUNT arguments. COUNT(*)/COUNT(1) intentionally keep this empty even though + // column pruning retains one placeholder scan slot. + protected List pushDownCountSlotIds = Collections.emptyList(); public void setPushDownAggNoGrouping(TPushAggOp pushDownAggNoGroupingOp) { this.pushDownAggNoGroupingOp = pushDownAggNoGroupingOp; } + public void setPushDownCountSlotIds(List pushDownCountSlotIds) { + this.pushDownCountSlotIds = Lists.newArrayList(pushDownCountSlotIds); + } + public void setChildrenDistributeExprLists(List> childrenDistributeExprLists) { this.childrenDistributeExprLists = childrenDistributeExprLists; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 6a0b7c71ffd057..c59992082ad54e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -730,12 +730,22 @@ public static Optional getIcebergColumnStats(String colName, or try (CloseableIterable fileScanTasks = tableScan.planFiles()) { for (FileScanTask task : fileScanTasks) { int colId = getColId(task.spec(), colName); - totalDataSize += task.file().columnSizes().get(colId); + Map columnSizes = task.file().columnSizes(); + Map nullValueCounts = task.file().nullValueCounts(); + Long columnSize = columnSizes == null ? null : columnSizes.get(colId); + Long nullValueCount = nullValueCounts == null ? null : nullValueCounts.get(colId); + // Iceberg can omit maps or entries for mode=none; partial aggregation would fabricate zero stats. + if (columnSize == null || nullValueCount == null) { + return Optional.empty(); + } + totalDataSize += columnSize; totalDataCount += task.file().recordCount(); - totalNumNull += task.file().nullValueCounts().get(colId); + totalNumNull += nullValueCount; } } catch (IOException e) { LOG.warn("Error to close FileScanTask.", e); + // A failed close can cancel an in-flight empty return, so accumulated stats are not reliable. + return Optional.empty(); } ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(totalDataCount); columnStatisticBuilder.setMaxValue(Double.POSITIVE_INFINITY); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java index 1b1742ece8a6a8..2bada7864d3642 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java @@ -905,8 +905,9 @@ public void testShouldWarmUpPerformance200kTables() { Assertions.assertEquals(200000, matched); System.out.println("[Perf] 200K tables, wildcard match-all: " + elapsedMs + " ms"); - Assertions.assertTrue(elapsedMs < 1500, - "200K regex matches should complete within 1.5s, took " + elapsedMs + " ms"); + // Shared FE UT workers need a broad guardrail; exact throughput belongs in dedicated benchmarks. + Assertions.assertTrue(elapsedMs < 3000, + "200K regex matches should complete within 3s, took " + elapsedMs + " ms"); } @Test @@ -1041,8 +1042,9 @@ public void testShouldWarmUpPerformanceRepeatedCycles200k() { Assertions.assertEquals(200000 * iterations, totalMatched); System.out.println("[Perf] 200K tables × 5 cycles: total=" + totalMs + " ms, avg=" + avgMs + " ms/cycle"); - Assertions.assertTrue(avgMs < 1000, - "Avg per refresh cycle for 200K tables should be < 1s, avg=" + avgMs + " ms"); + // Shared FE UT workers need a broad guardrail; exact throughput belongs in dedicated benchmarks. + Assertions.assertTrue(avgMs < 2000, + "Avg per refresh cycle for 200K tables should be < 2s, avg=" + avgMs + " ms"); } private static class RecordingAppender extends AbstractAppender { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java index 5475cfc4c98e8b..67209d224bd963 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/AutoProfileTest.java @@ -64,15 +64,16 @@ public void testAutoProfile() throws InterruptedException { result = System.currentTimeMillis(); } }; - profile.autoProfileDurationMs = 1000; - Thread.sleep(899); + // Keep ample margins because CI scheduling delays must not decide which threshold branch is exercised. + profile.autoProfileDurationMs = 10000; + Thread.sleep(100); profile.updateSummary(summaryInfo, true, null); Assertions.assertNull(ProfileManager.getInstance().queryIdToProfileMap.get(profile.getId())); profile = createProfile(); profile.setSummaryProfile(summaryProfile); - profile.autoProfileDurationMs = 500; - Thread.sleep(899); + profile.autoProfileDurationMs = 50; + Thread.sleep(200); profile.updateSummary(summaryInfo, true, null); Assertions.assertNotNull(ProfileManager.getInstance().queryIdToProfileMap.get(profile.getId())); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java index ef6fd3d63f3ce4..afe40a68dc8c16 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java @@ -77,4 +77,28 @@ public void testPreloadExternalMetadataTimeCounter() { Assertions.assertEquals(20, profile.getNereidsPreloadExternalMetadataTimeMs()); Assertions.assertEquals("20ms", profile.getPrettyNereidsPreloadExternalMetadataTime()); } + + @Test + public void testExternalTableMetaSummary() { + SummaryProfile profile = new SummaryProfile(); + profile.addExternalTableGetTableMetaTime(2); + profile.addExternalTableGetPartitionValuesTime(3); + profile.addExternalTableGetPartitionsTime(5); + profile.addExternalTableGetPartitionFilesTime(7); + profile.addExternalTableGetFileScanTasksTime(11); + + profile.update(ImmutableMap.of()); + + RuntimeProfile executionSummary = profile.getExecutionSummary(); + Assertions.assertEquals("28ms", executionSummary.getInfoString(SummaryProfile.EXTERNAL_TABLE_META_TIME)); + Assertions.assertEquals("2ms", executionSummary.getInfoString( + SummaryProfile.EXTERNAL_TABLE_GET_TABLE_META_TIME)); + Assertions.assertEquals("3ms", executionSummary.getInfoString( + SummaryProfile.EXTERNAL_TABLE_GET_PARTITION_VALUES_TIME)); + Assertions.assertEquals("5ms", executionSummary.getInfoString(SummaryProfile.GET_PARTITIONS_TIME)); + Assertions.assertEquals("7ms", executionSummary.getInfoString(SummaryProfile.GET_PARTITION_FILES_TIME)); + Assertions.assertEquals("11ms", executionSummary.getInfoString( + SummaryProfile.EXTERNAL_TABLE_GET_FILE_SCAN_TASKS_TIME)); + Assertions.assertEquals(28, profile.getExternalCatalogMetaTimeMs()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3ecb3a72d6b023..9422f08812de2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -28,6 +28,7 @@ import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.schema.external.TArrayField; import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TNestedField; import org.apache.doris.thrift.schema.external.TSchema; import org.apache.doris.thrift.schema.external.TStructField; @@ -254,6 +255,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertTrue(field1.isNameMappingIsAuthoritative()); Assert.assertEquals("7", field1.getInitialDefaultValue()); Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); @@ -262,7 +264,72 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertTrue(field2.isNameMappingIsAuthoritative()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + + @Test + public void testInitSchemaInfoForAllColumnSerializesNestedNonBinaryDefault() { + StructType structType = new StructType( + new StructField("added_int", Type.INT, "nested default", true)); + Column structColumn = new Column("s", structType, true); + structColumn.setUniqueId(10); + Column child = structColumn.getChildren().get(0); + child.setUniqueId(11); + child.setDefaultValueInfo(new Column("added_int", Type.INT, false, null, true, "7", "")); + TFileScanRangeParams params = new TFileScanRangeParams(); + + ExternalUtil.initSchemaInfoForAllColumn( + params, 1L, Collections.singletonList(structColumn), Collections.emptyMap()); + + TField childField = params.getHistorySchemaInfo().get(0).getRootField().getFields().get(0) + .getFieldPtr().getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assert.assertEquals("7", childField.getInitialDefaultValue()); + Assert.assertFalse(childField.isSetInitialDefaultValueIsBase64()); + } + + @Test + public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { + TFileScanRangeParams params = new TFileScanRangeParams(); + Column mappedColumn = new Column("a", Type.INT, true); + mappedColumn.setUniqueId(1); + Column unmappedColumn = new Column("b", Type.INT, true); + unmappedColumn.setUniqueId(2); + + Map> nameMapping = new HashMap<>(); + nameMapping.put(mappedColumn.getUniqueId(), Collections.singletonList("a")); + ExternalUtil.initSchemaInfoForAllColumn( + params, 600L, Arrays.asList(mappedColumn, unmappedColumn), nameMapping); + + List fields = params.getHistorySchemaInfo().get(0).getRootField().getFields(); + Assert.assertEquals(Collections.singletonList("a"), fields.get(0).getFieldPtr().getNameMapping()); + Assert.assertTrue(fields.get(0).getFieldPtr().isNameMappingIsAuthoritative()); + Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); + Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); + Assert.assertTrue(fields.get(1).getFieldPtr().isNameMappingIsAuthoritative()); + } + + @Test + public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping() { + Column column = new Column("a", Type.INT, true); + column.setUniqueId(1); + + TFileScanRangeParams absentParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn( + absentParams, 700L, Collections.singletonList(column), Collections.emptyMap()); + TField absentField = absentParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertFalse(absentField.isSetNameMapping()); + Assert.assertFalse(absentField.isSetNameMappingIsAuthoritative()); + + TFileScanRangeParams emptyParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn(emptyParams, 701L, + Collections.singletonList(column), Collections.emptyMap(), true, Collections.emptyMap()); + TField emptyField = emptyParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertTrue(emptyField.isSetNameMapping()); + Assert.assertTrue(emptyField.getNameMapping().isEmpty()); + Assert.assertTrue(emptyField.isNameMappingIsAuthoritative()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index aee814907bce61..21a899ac673f00 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -17,30 +17,64 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TColumnCategory; +import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TFileScanSlotInfo; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; public class FileQueryScanNodeTest { private static final long MB = 1024L * 1024L; + private static final Method UPDATE_REQUIRED_SLOTS_METHOD; + + static { + try { + UPDATE_REQUIRED_SLOTS_METHOD = FileQueryScanNode.class.getDeclaredMethod("updateRequiredSlots"); + UPDATE_REQUIRED_SLOTS_METHOD.setAccessible(true); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + private TableIf table; private static class TestFileQueryScanNode extends FileQueryScanNode { + private TableIf targetTable; + TestFileQueryScanNode(SessionVariable sv) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), "test", ScanContext.EMPTY, false, sv); } + TupleDescriptor getTupleDescriptor() { + return desc; + } + + void setTargetTable(TableIf targetTable) { + this.targetTable = targetTable; + } + @Override protected TFileFormatType getFileFormatType() throws UserException { return TFileFormatType.FORMAT_ORC; @@ -53,7 +87,7 @@ protected List getPathPartitionKeys() throws UserException { @Override protected TableIf getTargetTable() throws UserException { - return null; + return targetTable; } @Override @@ -62,6 +96,12 @@ protected Map getLocationProperties() throws UserException { } } + @Before + public void setUp() { + table = Mockito.mock(TableIf.class); + Mockito.when(table.getName()).thenReturn("test_table"); + } + @Test public void testApplyMaxFileSplitNumLimitRaisesTargetSize() { SessionVariable sv = new SessionVariable(); @@ -88,4 +128,38 @@ public void testApplyMaxFileSplitNumLimitDisabled() { long target = node.applyMaxFileSplitNumLimit(32 * MB, 10_000L * MB); Assert.assertEquals(32 * MB, target); } + + @Test + public void testUpdateRequiredSlotsPreservesInlineDefaultValueExpr() throws Exception { + SessionVariable sv = new SessionVariable(); + TestFileQueryScanNode node = new TestFileQueryScanNode(sv); + node.setTargetTable(table); + + TupleDescriptor desc = node.getTupleDescriptor(); + desc.setTable(table); + SlotDescriptor slot = new SlotDescriptor(new SlotId(1), desc); + slot.setColumn(new Column("c1", Type.INT)); + desc.addSlot(slot); + Mockito.when(table.getFullSchema()).thenReturn(Arrays.asList(slot.getColumn())); + + TExpr defaultExpr = new TExpr(); + defaultExpr.setNodes(Collections.emptyList()); + TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); + slotInfo.setSlotId(slot.getId().asInt()); + slotInfo.setCategory(TColumnCategory.REGULAR); + slotInfo.setIsFileSlot(true); + slotInfo.setDefaultValueExpr(defaultExpr); + + TFileScanRangeParams params = new TFileScanRangeParams(); + params.setRequiredSlots(Arrays.asList(slotInfo)); + node.params = params; + + UPDATE_REQUIRED_SLOTS_METHOD.invoke(node); + + TFileScanSlotInfo updatedSlotInfo = node.params.getRequiredSlots().get(0); + Assert.assertSame(slotInfo, updatedSlotInfo); + Assert.assertTrue(updatedSlotInfo.isSetDefaultValueExpr()); + Assert.assertSame(defaultExpr, updatedSlotInfo.getDefaultValueExpr()); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 69506a505cb21e..b4a1c6a5d1b235 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; @@ -187,6 +188,11 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + // The scan now resolves initial defaults from the statement-pinned schema id, so the + // mocked table must expose the same historical-schema lookup as a real Iceberg table. + Mockito.doAnswer(invocation -> ImmutableMap.of( + mockedIcebergTable.schema().schemaId(), mockedIcebergTable.schema())) + .when(mockedIcebergTable).schemas(); // Mock newScan() chain used by IcebergScanNode.createTableScan() TableScan mockedTableScan = Mockito.mock(TableScan.class, Mockito.RETURNS_DEEP_STUBS); @@ -277,6 +283,76 @@ public void testIcebergDeletePlanAddsRowIdProject() throws Exception { assertContainsPhysicalSink(physicalPlan, PhysicalIcebergDeleteSink.class); } + @Test + public void testCreateIcebergV3TableRejectsRowLineageReservedColumn() throws Exception { + useIceberg(); + String rowIdTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String rowIdSql = "create table " + rowIdTable + + " (_row_id bigint) properties('format-version'='3')"; + LogicalPlan rowIdPlan = parseStmt(rowIdSql); + Assertions.assertTrue(rowIdPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) rowIdPlan).getCreateTableInfo().validate(connectContext)); + + String lastUpdatedSequenceNumberTable = + "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String lastUpdatedSequenceNumberSql = "create table " + lastUpdatedSequenceNumberTable + + " (_last_updated_sequence_number bigint) properties('format-version'='3')"; + LogicalPlan lastUpdatedSequenceNumberPlan = parseStmt(lastUpdatedSequenceNumberSql); + Assertions.assertTrue(lastUpdatedSequenceNumberPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) lastUpdatedSequenceNumberPlan).getCreateTableInfo() + .validate(connectContext)); + + String formatV2Table = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String formatV2Sql = "create table " + formatV2Table + + " (_row_id bigint) properties('format-version'='2')"; + LogicalPlan formatV2Plan = parseStmt(formatV2Sql); + Assertions.assertTrue(formatV2Plan instanceof CreateTableCommand); + Assertions.assertDoesNotThrow( + () -> ((CreateTableCommand) formatV2Plan).getCreateTableInfo().validate(connectContext)); + } + + @Test + public void testCreateIcebergDefaultV3TableRejectsRowLineageReservedColumn() throws Exception { + useIceberg(); + IcebergExternalCatalog catalog = (IcebergExternalCatalog) Env.getCurrentEnv() + .getCatalogMgr().getCatalog(catalogName); + catalog.getCatalogProperty().addProperty("table-default.format-version", "3"); + try { + String rowIdTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String rowIdSql = "create table " + rowIdTable + " (_row_id bigint)"; + LogicalPlan rowIdPlan = parseStmt(rowIdSql); + Assertions.assertTrue(rowIdPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) rowIdPlan).getCreateTableInfo().validate(connectContext)); + Assertions.assertFalse(catalog.getCatalog().tableExists(TableIdentifier.of(dbName, rowIdTable))); + + String formatV2Table = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String formatV2Sql = "create table " + formatV2Table + + " (_row_id bigint) properties('format-version'='2')"; + LogicalPlan formatV2Plan = parseStmt(formatV2Sql); + Assertions.assertTrue(formatV2Plan instanceof CreateTableCommand); + Assertions.assertDoesNotThrow( + () -> ((CreateTableCommand) formatV2Plan).getCreateTableInfo().validate(connectContext)); + } finally { + catalog.getCatalogProperty().deleteProperty("table-default.format-version"); + } + } + + @Test + public void testIcebergV3CtasRejectsRowLineageReservedColumn() throws Exception { + useIceberg(); + String ctasTable = "row_lineage_reserved_" + UUID.randomUUID().toString().replace("-", ""); + String ctasSql = "create table " + ctasTable + + " properties('format-version'='3') as select 1 as _row_id"; + LogicalPlan ctasPlan = parseStmt(ctasSql); + Assertions.assertTrue(ctasPlan instanceof CreateTableCommand); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> ((CreateTableCommand) ctasPlan).validateCreateTableAsSelect( + connectContext, ((CreateTableCommand) ctasPlan).getCtasQuery().get())); + } + @Test public void testIcebergUpdatePlans() throws Exception { useIceberg(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java index 2bf1ef3deb4545..55a3a63a5fabfb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java @@ -17,9 +17,17 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -27,6 +35,7 @@ import org.apache.iceberg.catalog.ViewCatalog; import org.junit.Assert; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import java.util.Arrays; @@ -114,4 +123,50 @@ public void testListTableNamesFiltersViewsWhenRestViewEnabled() { Assert.assertEquals(Collections.singletonList("DORIS_HORIZON_T"), tableNames); } + + @Test + public void testPerformCreateTableRespectsCatalogDefaultFormatVersion() throws Exception { + Map catalogProps = new HashMap<>(); + catalogProps.put(CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.FORMAT_VERSION, "3"); + IcebergExternalCatalog dorisCatalog = mockHmsCatalog(catalogProps); + Catalog icebergCatalog = Mockito.mock(Catalog.class, + Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); + IcebergMetadataOps ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); + + ExternalDatabase dorisDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(dorisDb.getRemoteName()).thenReturn("db"); + Mockito.when(dorisDb.getTableNullable("tbl")).thenReturn(null); + Mockito.doReturn(dorisDb).when(dorisCatalog).getDbNullable("db"); + Mockito.when(dorisCatalog.getName()).thenReturn("iceberg_catalog"); + Mockito.when(icebergCatalog.tableExists(TableIdentifier.of("db", "tbl"))).thenReturn(false); + + CreateTableInfo createTableInfo = Mockito.mock(CreateTableInfo.class); + Map tableProps = new HashMap<>(); + Mockito.when(createTableInfo.getDbName()).thenReturn("db"); + Mockito.when(createTableInfo.getTableName()).thenReturn("tbl"); + Mockito.when(createTableInfo.isIfNotExists()).thenReturn(false); + Mockito.when(createTableInfo.getColumns()).thenReturn(Collections.singletonList( + new Column("id", Type.INT, true))); + Mockito.when(createTableInfo.getProperties()).thenReturn(tableProps); + + ops.performCreateTable(createTableInfo); + + Mockito.verify(createTableInfo).validateIcebergRowLineageColumns(3); + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + Mockito.verify(icebergCatalog).createTable(Mockito.eq(TableIdentifier.of("db", "tbl")), + Mockito.any(Schema.class), Mockito.any(PartitionSpec.class), propsCaptor.capture()); + Assert.assertFalse(propsCaptor.getValue().containsKey(TableProperties.FORMAT_VERSION)); + Assert.assertEquals(3, IcebergUtils.getEffectiveIcebergFormatVersion( + propsCaptor.getValue(), catalogProps)); + } + + private IcebergExternalCatalog mockHmsCatalog(Map catalogProperties) { + IcebergExternalCatalog dorisCatalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + }); + Mockito.when(dorisCatalog.getProperties()).thenReturn(catalogProperties); + Mockito.when(dorisCatalog.getIcebergCatalogType()).thenReturn(IcebergExternalCatalog.ICEBERG_HMS); + Mockito.when(dorisCatalog.getCatalogProperty()).thenReturn(new CatalogProperty(null, Collections.emptyMap())); + return dorisCatalog; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 47011cafc9473c..c9b5f13080c319 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -281,6 +281,20 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); } + @Test + public void testParseSchemaPreservesNestedNonBinaryInitialDefault() { + Schema schema = new Schema(Types.NestedField.optional(10, "s", Types.StructType.of( + Types.NestedField.optional("added_int") + .withId(11) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build()))); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getChildren().get(0).getDefaultValue()); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( @@ -373,6 +387,26 @@ public void testGetIdentityPartitionInfoMapSupportsFloatingPointPartitions() { Double.doubleToLongBits(Double.parseDouble(serializedDouble))); } + @Test + public void testParseFloatingPointPartitionValueSupportsSpecialValues() { + Assert.assertTrue(Float.isNaN( + (Float) IcebergUtils.parsePartitionValueFromString("NaN", Types.FloatType.get()))); + Assert.assertTrue(Float.isNaN( + (Float) IcebergUtils.parsePartitionValueFromString("nan", Types.FloatType.get()))); + Assert.assertEquals(Float.POSITIVE_INFINITY, + (Float) IcebergUtils.parsePartitionValueFromString("Infinity", Types.FloatType.get()), 0.0F); + Assert.assertEquals(Float.NEGATIVE_INFINITY, + (Float) IcebergUtils.parsePartitionValueFromString("-inf", Types.FloatType.get()), 0.0F); + Assert.assertTrue(Double.isNaN( + (Double) IcebergUtils.parsePartitionValueFromString("NaN", Types.DoubleType.get()))); + Assert.assertTrue(Double.isNaN( + (Double) IcebergUtils.parsePartitionValueFromString("nan", Types.DoubleType.get()))); + Assert.assertEquals(Double.POSITIVE_INFINITY, + (Double) IcebergUtils.parsePartitionValueFromString("Infinity", Types.DoubleType.get()), 0.0D); + Assert.assertEquals(Double.NEGATIVE_INFINITY, + (Double) IcebergUtils.parsePartitionValueFromString("-inf", Types.DoubleType.get()), 0.0D); + } + @Test public void testGetMatchingManifest() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java index 77d318518f326e..3c6aa2676803de 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java @@ -18,17 +18,33 @@ package org.apache.doris.datasource.iceberg.helper; import org.apache.doris.thrift.TFileContent; +import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -58,6 +74,242 @@ public void setUp() { } + @Test + public void testConvertToWriterResultRespectsNoneMetricsMode() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(ImmutableMap.of(2, 128L)); + columnStats.setValueCounts(ImmutableMap.of(2, 10L)); + columnStats.setNullValueCounts(ImmutableMap.of(2, 0L)); + columnStats.setLowerBounds(ImmutableMap.of(2, ByteBuffer.wrap(new byte[] {0x01}))); + columnStats.setUpperBounds(ImmutableMap.of(2, ByteBuffer.wrap(new byte[] {0x02}))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + WriteResult result = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)); + DataFile dataFile = result.dataFiles()[0]; + + Assertions.assertTrue(dataFile.columnSizes() == null || dataFile.columnSizes().isEmpty()); + Assertions.assertTrue(dataFile.valueCounts() == null || dataFile.valueCounts().isEmpty()); + Assertions.assertTrue(dataFile.nullValueCounts() == null || dataFile.nullValueCounts().isEmpty()); + Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); + Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); + } + + @Test + public void testConvertToWriterResultCountsModeOmitsBounds() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(ImmutableMap.of(2, 128L)); + columnStats.setValueCounts(ImmutableMap.of(2, 10L)); + columnStats.setNullValueCounts(ImmutableMap.of(2, 0L)); + columnStats.setLowerBounds(ImmutableMap.of( + 2, Conversions.toByteBuffer(Types.StringType.get(), "abcdefgh"))); + columnStats.setUpperBounds(ImmutableMap.of( + 2, Conversions.toByteBuffer(Types.StringType.get(), "ijklmnop"))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals(ImmutableMap.of(2, 128L), dataFile.columnSizes()); + Assertions.assertEquals(ImmutableMap.of(2, 10L), dataFile.valueCounts()); + Assertions.assertEquals(ImmutableMap.of(2, 0L), dataFile.nullValueCounts()); + Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); + Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); + } + + @Test + public void testConvertToWriterResultTruncatesStringAndBinaryBounds() { + Schema boundsSchema = new Schema( + Types.NestedField.optional(1, "text", Types.StringType.get()), + Types.NestedField.optional(2, "payload", Types.BinaryType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(boundsSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(3)")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setLowerBounds(ImmutableMap.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), "abcdef"), + 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); + columnStats.setUpperBounds(ImmutableMap.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), "uvwxyz"), + 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals("abc", Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.lowerBounds().get(1)).toString()); + Assertions.assertEquals("uvx", Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.upperBounds().get(1)).toString()); + Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 3}), dataFile.lowerBounds().get(2)); + Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 4}), dataFile.upperBounds().get(2)); + } + + @Test + public void testConvertToWriterResultPreservesOrcUpperBoundWithoutTruncatedSuccessor() { + Schema boundsSchema = new Schema( + Types.NestedField.optional(1, "text", Types.StringType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(boundsSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "orc", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(1)")); + + String maxWithoutSuccessor = new String(Character.toChars(Character.MAX_CODE_POINT)) + "tail"; + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setUpperBounds(ImmutableMap.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), maxWithoutSuccessor))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.orc"); + commitData.setRowCount(1); + commitData.setFileSize(128); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; + + Assertions.assertNotNull(dataFile.upperBounds()); + Assertions.assertEquals(maxWithoutSuccessor, Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.upperBounds().get(1)).toString()); + } + + @Test + public void testConvertToWriterResultBuildsMetricsPolicyOncePerBatch() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergCommitData firstCommit = new TIcebergCommitData(); + firstCommit.setFilePath("/path/to/first.parquet"); + firstCommit.setRowCount(10); + firstCommit.setFileSize(1024); + + TIcebergCommitData secondCommit = new TIcebergCommitData(); + secondCommit.setFilePath("/path/to/second.parquet"); + secondCommit.setRowCount(20); + secondCommit.setFileSize(2048); + + IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(firstCommit, secondCommit)); + + // One schema lookup is made by Iceberg's policy builder and one is captured for all files in the batch. + Mockito.verify(table, Mockito.times(2)).schema(); + } + + @Test + public void testConvertToWriterResultHandlesV3TransactionTableLineageMetrics(@TempDir Path tempDir) { + HadoopTables tables = new HadoopTables(new Configuration()); + Table baseTable = tables.create(schema, unpartitionedSpec, SortOrder.unsorted(), ImmutableMap.of( + TableProperties.FORMAT_VERSION, "3", + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(16)"), + tempDir.resolve("table").toUri().toString()); + Table transactionTable = baseTable.newTransaction().table(); + + int rowId = MetadataColumns.ROW_ID.fieldId(); + int sequenceNumberId = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(); + ByteBuffer rowIdBound = Conversions.toByteBuffer(MetadataColumns.ROW_ID.type(), 7L); + ByteBuffer sequenceNumberBound = Conversions.toByteBuffer( + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.type(), 3L); + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setLowerBounds(ImmutableMap.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + columnStats.setUpperBounds(ImmutableMap.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/v3-data.parquet"); + commitData.setRowCount(1); + commitData.setFileSize(128); + commitData.setColumnStats(columnStats); + + DataFile dataFile = Assertions.assertDoesNotThrow( + () -> IcebergWriterHelper.convertToWriterResult( + transactionTable, ImmutableList.of(commitData)).dataFiles()[0]); + Assertions.assertEquals(rowIdBound, dataFile.lowerBounds().get(rowId)); + Assertions.assertEquals(rowIdBound, dataFile.upperBounds().get(rowId)); + Assertions.assertEquals(sequenceNumberBound, dataFile.lowerBounds().get(sequenceNumberId)); + Assertions.assertEquals(sequenceNumberBound, dataFile.upperBounds().get(sequenceNumberId)); + } + + @Test + public void testConvertToWriterResultSuppressesLogicalMetricsBelowRepeatedFields() { + Schema repeatedSchema = new Schema( + Types.NestedField.optional(1, "items", + Types.ListType.ofOptional(2, Types.IntegerType.get())), + Types.NestedField.optional(3, "attributes", + Types.MapType.ofOptional(4, 5, Types.StringType.get(), Types.StringType.get())), + Types.NestedField.optional(6, "top_level", Types.IntegerType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(repeatedSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "full")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(ImmutableMap.of(2, 20L, 4, 40L, 5, 50L, 6, 60L)); + columnStats.setValueCounts(ImmutableMap.of(2, 2L, 4, 4L, 5, 5L, 6, 6L)); + columnStats.setNullValueCounts(ImmutableMap.of(2, 0L, 4, 0L, 5, 0L, 6, 0L)); + columnStats.setLowerBounds(ImmutableMap.of( + 2, Conversions.toByteBuffer(Types.IntegerType.get(), 2), + 4, Conversions.toByteBuffer(Types.StringType.get(), "key"), + 5, Conversions.toByteBuffer(Types.StringType.get(), "value"), + 6, Conversions.toByteBuffer(Types.IntegerType.get(), 6))); + columnStats.setUpperBounds(columnStats.getLowerBounds()); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/repeated.parquet"); + commitData.setRowCount(6); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals(ImmutableMap.of(2, 20L, 4, 40L, 5, 50L, 6, 60L), dataFile.columnSizes()); + Assertions.assertEquals(ImmutableMap.of(6, 6L), dataFile.valueCounts()); + Assertions.assertEquals(ImmutableMap.of(6, 0L), dataFile.nullValueCounts()); + Assertions.assertEquals(ImmutableMap.of(6, columnStats.getLowerBounds().get(6)), dataFile.lowerBounds()); + Assertions.assertEquals(ImmutableMap.of(6, columnStats.getUpperBounds().get(6)), dataFile.upperBounds()); + } + @Test public void testConvertToDeleteFiles_EmptyList() { List commitDataList = new ArrayList<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java new file mode 100644 index 00000000000000..6a714107a6e856 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.source; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IcebergDeleteFileFilterTest { + @Test + public void testValidateDeletionVectorMetadataAcceptsLongRange() { + Assertions.assertDoesNotThrow(() -> IcebergDeleteFileFilter.validateDeletionVectorMetadata( + "puffin.dv", 1L << 40, (1L << 32) + 17, (1L << 30) + 19)); + } + + @Test + public void testValidateDeletionVectorMetadataRejectsInvalidRange() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, null, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 1L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", -1, 1L, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, -1L, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 1L, -1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata( + "puffin.dv", Long.MAX_VALUE, Long.MAX_VALUE, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 90L, 11L)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 87e2fb62e52b96..504f0f48d090f8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -17,23 +17,49 @@ package org.apache.doris.datasource.iceberg.source; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccTableInfo; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.system.Backend; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TIcebergDeleteFileDesc; +import org.apache.doris.thrift.TPushAggOp; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ScanTaskUtil; @@ -42,20 +68,38 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.UUID; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; + @SuppressWarnings("unchecked") + private static Optional>> extractNameMapping( + IcebergScanNode node) throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("extractNameMapping"); + method.setAccessible(true); + return (Optional>>) method.invoke(node); + } + private static class TestIcebergScanNode extends IcebergScanNode { + private final boolean enableMappingVarbinary; private TableScan tableScan; TestIcebergScanNode(SessionVariable sv) { + this(sv, false); + } + + TestIcebergScanNode(SessionVariable sv, boolean enableMappingVarbinary) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); + this.enableMappingVarbinary = enableMappingVarbinary; } void setTableScan(TableScan tableScan) { @@ -71,10 +115,170 @@ public TableScan createTableScan() { public boolean isBatchMode() { return false; } + + @Override + protected boolean getEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + @Override + public List getPathPartitionKeys() { + return Collections.emptyList(); + } + + @Override + public TableSnapshot getQueryTableSnapshot() { + return tableSnapshot; + } + + @Override + public TableScanParams getScanParams() { + return scanParams; + } + + int enableAndGetIcebergScanSemanticsVersion() { + params = new TFileScanRangeParams(); + enableCurrentIcebergScanSemantics(); + return params.getIcebergScanSemanticsVersion(); + } + } + + @Test + public void testEmitsCurrentIcebergScanSemanticsCapability() { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + + Assert.assertEquals(IcebergScanNode.ICEBERG_SCAN_SEMANTICS_VERSION, + node.enableAndGetIcebergScanSemanticsVersion()); + } + + @Test + public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Table table = Mockito.mock(Table.class); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + setIcebergSource(node, source); + + Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); + Assert.assertFalse(extractNameMapping(node).isPresent()); + + Mockito.when(table.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + Optional>> emptyMapping = extractNameMapping(node); + Assert.assertTrue(emptyMapping.isPresent()); + Assert.assertTrue(emptyMapping.get().isEmpty()); + } + + @Test + public void testSnapshotCacheIgnoresIdlessNameMappingWrapper() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn(Collections.singletonMap( + TableProperties.DEFAULT_NAME_MAPPING, + "[{\"names\":[\"legacy_wrapper\"],\"fields\":[" + + "{\"field-id\":7,\"names\":[\"legacy_child\"]}]}]")); + + IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap()), + new IcebergSnapshot(1L, 1L), + IcebergUtils.getNameMapping(table)); + + Assert.assertTrue(snapshotCacheValue.getNameMapping().isPresent()); + Assert.assertEquals(Collections.singletonList("legacy_child"), + snapshotCacheValue.getNameMapping().get().get(7)); + Assert.assertEquals(Collections.singleton(7), + snapshotCacheValue.getNameMapping().get().keySet()); + } + + @Test + public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefresh() throws Exception { + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, 11L), Optional.empty()))); + try { + Assert.assertFalse(extractNameMapping(node).isPresent()); + } finally { + ConnectContext.remove(); + } + } + + private static class CountPlanningIcebergScanNode extends IcebergScanNode { + private final TableScan tableScan; + private final long snapshotCount; + private int snapshotCountCalls; + + CountPlanningIcebergScanNode(SessionVariable sv, TableScan tableScan, long snapshotCount) { + super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); + this.tableScan = tableScan; + this.snapshotCount = snapshotCount; + } + + @Override + public TableScan createTableScan() { + return tableScan; + } + + @Override + public long getCountFromSnapshot() { + ++snapshotCountCalls; + return snapshotCount; + } } @Test - public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + public void testTableLevelCountSplitPlanningRequiresCountStar() { + SessionVariable sv = Mockito.mock(SessionVariable.class); + Mockito.when(sv.getEnableExternalTableBatchMode()).thenReturn(false); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(tableScan.snapshot()).thenReturn(Mockito.mock(Snapshot.class)); + + // COUNT(required_col) carries a non-empty semantic argument list. Even though its result + // equals COUNT(*) for valid data, BE intentionally reads the column to enforce schema + // contracts. FE must therefore leave all real file tasks available to that fallback. + CountPlanningIcebergScanNode countColumnNode = + new CountPlanningIcebergScanNode(sv, tableScan, 30_000); + countColumnNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countColumnNode.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + Assert.assertFalse(countColumnNode.isBatchMode()); + Assert.assertEquals(0, countColumnNode.snapshotCountCalls); + + // COUNT(*) has an explicitly empty argument list, so snapshot row count remains eligible + // and doGetSplits may retain only representative tasks for parallel materialization. + CountPlanningIcebergScanNode countStarNode = + new CountPlanningIcebergScanNode(sv, tableScan, 30_000); + countStarNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countStarNode.setPushDownCountSlotIds(Collections.emptyList()); + Assert.assertFalse(countStarNode.isBatchMode()); + Assert.assertEquals(1, countStarNode.snapshotCountCalls); + } + + @Test + public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") .withId(7) .ofType(Types.BinaryType.get()) @@ -89,6 +293,7 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Mockito.when(snapshot.schemaId()).thenReturn(11); Table table = Mockito.mock(Table.class); Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + Mockito.when(table.schema()).thenReturn(currentSchema); TableScan snapshotScan = Mockito.mock(TableScan.class); Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); Mockito.when(snapshotScan.table()).thenReturn(table); @@ -96,9 +301,182 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); node.setTableScan(snapshotScan); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(TableIf.class)); + setIcebergSource(node, source); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertTrue(defaults.isEmpty()); + } + + @Test + public void testInitialDefaultMetadataUsesStatementPinnedSchemaAfterCacheInvalidation() throws Exception { + Schema pinnedSchema = new Schema(11, ImmutableList.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Schema refreshedSchema = new Schema(12, + ImmutableList.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.schema()).thenReturn(refreshedSchema); + Mockito.when(refreshedTable.schemas()).thenReturn(ImmutableMap.of( + pinnedSchema.schemaId(), pinnedSchema, + refreshedSchema.schemaId(), refreshedSchema)); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, pinnedSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + IcebergTableQueryInfo selectedSnapshot = Mockito.mock(IcebergTableQueryInfo.class); + Mockito.when(selectedSnapshot.getSchemaId()).thenReturn(11); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(snapshotScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(selectedSnapshot).when(node).getSpecifiedSnapshot(); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); + } + + @Test + public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws Exception { + Schema dataSnapshotSchema = new Schema(11, ImmutableList.of(Types.NestedField.optional("string_default") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build())); + Schema branchSchema = new Schema(12, ImmutableList.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Snapshot dataSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(dataSnapshot.schemaId()).thenReturn(dataSnapshotSchema.schemaId()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(ImmutableMap.of( + dataSnapshotSchema.schemaId(), dataSnapshotSchema, + branchSchema.schemaId(), branchSchema)); + TableScan branchScan = Mockito.mock(TableScan.class); + Mockito.when(branchScan.snapshot()).thenReturn(dataSnapshot); + Mockito.when(branchScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(branchScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(new IcebergTableQueryInfo(1L, "branch", branchSchema.schemaId())) + .when(node).getSpecifiedSnapshot(); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, branchSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() throws Exception { + Schema systemTableSchema = new Schema(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Table systemTable = Mockito.mock(Table.class); + Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable"); + icebergTableField.setAccessible(true); + icebergTableField.set(node, systemTable); + Field isSystemTableField = IcebergScanNode.class.getDeclaredField("isSystemTable"); + isSystemTableField.setAccessible(true); + isSystemTableField.setBoolean(node, true); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); + Mockito.verify(node, Mockito.never()).createTableScan(); + } + + private static void setIcebergTable(IcebergScanNode node, Table table) throws Exception { + Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable"); + icebergTableField.setAccessible(true); + icebergTableField.set(node, table); + } + + private static void setIcebergSource(IcebergScanNode node, IcebergSource source) throws Exception { + Field sourceField = IcebergScanNode.class.getDeclaredField("source"); + sourceField.setAccessible(true); + sourceField.set(node, source); } @Test @@ -177,9 +555,38 @@ public void testSetIcebergParamsUsesSplitFileFormat() throws Exception { TFileRangeDesc rangeDesc = new TFileRangeDesc(); method.invoke(node, rangeDesc, split); + // Iceberg tables may mix file formats, so each range must preserve its split format. Assert.assertEquals(TFileFormatType.FORMAT_ORC, rangeDesc.getFormatType()); } + @Test + public void testPositionDeleteSystemTableValidatesDeletionVectorMetadata() throws Exception { + DeleteFile deleteFile = Mockito.mock(DeleteFile.class); + Mockito.when(deleteFile.path()).thenReturn("file:///tmp/delete-shared.puffin"); + Mockito.when(deleteFile.format()).thenReturn(FileFormat.PUFFIN); + Mockito.when(deleteFile.fileSizeInBytes()).thenReturn(100L); + Mockito.when(deleteFile.contentOffset()).thenReturn(null); + Mockito.when(deleteFile.contentSizeInBytes()).thenReturn(10L); + + PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); + Mockito.when(task.file()).thenReturn(deleteFile); + Mockito.when(task.start()).thenReturn(0L); + Mockito.when(task.length()).thenReturn(100L); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Method method = IcebergScanNode.class.getDeclaredMethod( + "createIcebergPositionDeleteSysSplit", PositionDeletesScanTask.class); + method.setAccessible(true); + + try { + method.invoke(node, task); + Assert.fail("position_deletes planning should reject invalid deletion vector metadata"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof IllegalArgumentException); + Assert.assertTrue(e.getCause().getMessage().contains("delete-shared.puffin")); + } + } + @Test public void testSetIcebergParamsPropagatesPositionDeleteFileFormat() throws Exception { SessionVariable sv = new SessionVariable(); @@ -212,4 +619,98 @@ public void testSetIcebergParamsPropagatesPositionDeleteFileFormat() throws Exce .get(0); Assert.assertEquals(org.apache.doris.thrift.TFileFormatType.FORMAT_ORC, deleteFileDesc.getFileFormat()); } + + @Test + public void testPartitionDataJsonMatchesRenamedFieldById() throws Exception { + SessionVariable sv = new SessionVariable(); + TestIcebergScanNode node = new TestIcebergScanNode(sv); + + Schema schema = new Schema(Types.NestedField.required(1, "p", Types.IntegerType.get())); + PartitionSpec oldSpec = PartitionSpec.builderFor(schema).identity("p").build(); + PartitionData partitionData = new PartitionData(oldSpec.partitionType()); + partitionData.set(0, 10); + int partitionFieldId = oldSpec.fields().get(0).fieldId(); + List outputPartitionFields = Collections.singletonList( + Types.NestedField.optional(partitionFieldId, "p2", Types.IntegerType.get())); + + Method method = IcebergScanNode.class.getDeclaredMethod("getPartitionDataObjectJson", + PartitionData.class, PartitionSpec.class, List.class); + method.setAccessible(true); + + Assert.assertEquals("{\"p2\":10}", method.invoke(node, partitionData, oldSpec, outputPartitionFields)); + } + + @Test + public void testRejectBinaryPartitionValueWithoutBinarySafeTransport() throws Exception { + assertUnsupportedPositionDeletesPartitionValue( + Types.BinaryType.get(), ByteBuffer.wrap(new byte[] {0, (byte) 0xff}), false, "binary"); + assertUnsupportedPositionDeletesPartitionValue( + Types.FixedType.ofLength(2), ByteBuffer.wrap(new byte[] {0, (byte) 0xff}), false, "fixed[2]"); + } + + @Test + public void testRejectUuidPartitionValueWhenMappedToVarbinary() throws Exception { + assertUnsupportedPositionDeletesPartitionValue( + Types.UUIDType.get(), UUID.fromString("123e4567-e89b-12d3-a456-426614174000"), true, "uuid"); + } + + private void assertUnsupportedPositionDeletesPartitionValue( + org.apache.iceberg.types.Type type, Object value, boolean enableMappingVarbinary, + String expectedType) throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable(), enableMappingVarbinary); + Schema schema = new Schema(Types.NestedField.required(1, "p", type)); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("p").build(); + PartitionData partitionData = new PartitionData(spec.partitionType()); + partitionData.set(0, value); + + Method method = IcebergScanNode.class.getDeclaredMethod("getPartitionDataObjectJson", + PartitionData.class, PartitionSpec.class, List.class); + method.setAccessible(true); + try { + method.invoke(node, partitionData, spec, spec.partitionType().fields()); + Assert.fail("Binary partition values must not be silently materialized as NULL"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof UserException); + Assert.assertTrue(e.getCause().getMessage().contains("partition field 'p'")); + Assert.assertTrue(e.getCause().getMessage().contains(expectedType)); + } + } + + @Test + public void testRejectUnsupportedPositionDeleteFileFormat() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Method method = IcebergScanNode.class.getDeclaredMethod( + "getNativePositionDeleteFileFormat", FileFormat.class); + method.setAccessible(true); + + try { + method.invoke(node, FileFormat.AVRO); + Assert.fail("AVRO position delete files should be rejected explicitly"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof UnsupportedOperationException); + Assert.assertEquals("Unsupported Iceberg position delete file format: AVRO", + e.getCause().getMessage()); + } + } + + @Test + public void testRejectSmoothUpgradeSourceBackendForPositionDeletes() throws Exception { + Backend currentBackend = Mockito.mock(Backend.class); + Mockito.when(currentBackend.isSmoothUpgradeSrc()).thenReturn(false); + IcebergScanNode.checkPositionDeletesBackendCompatibility(Collections.singletonList(currentBackend)); + + Backend smoothUpgradeSource = Mockito.mock(Backend.class); + Mockito.when(smoothUpgradeSource.isSmoothUpgradeSrc()).thenReturn(true); + Mockito.when(smoothUpgradeSource.getId()).thenReturn(10001L); + List backends = new ArrayList<>(); + backends.add(currentBackend); + backends.add(smoothUpgradeSource); + + try { + IcebergScanNode.checkPositionDeletesBackendCompatibility(backends); + Assert.fail("smooth upgrade source backend should reject native position_deletes planning"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("backend 10001 is a smooth upgrade source")); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java index 99e4aa62dd574d..a76319d108f7cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java @@ -19,11 +19,36 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.Answers; +import org.mockito.Mockito; import java.lang.reflect.Method; +import java.sql.DatabaseMetaData; public class JdbcClickHouseClientTest { + @Test + public void testDatabaseTermFollowsDriverMetadata() throws Exception { + DatabaseMetaData databaseMetaData = Mockito.mock(DatabaseMetaData.class); + + Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(false); + Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.9.8")); + + Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(true); + Assert.assertTrue(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.7.1")); + + Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.4.2")); + } + + @Test + public void testClickHouseSpecificTableTypesAreVisible() { + JdbcClickHouseClient client = Mockito.mock(JdbcClickHouseClient.class, Answers.CALLS_REAL_METHODS); + + Assert.assertArrayEquals( + new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}, + client.getTableTypes()); + } + @Test public void testIsNewClickHouseDriver() { try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index d31ecb12ee45f1..9dcf91731bbee2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon.source; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.ExceptionChecker; @@ -34,6 +35,7 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TPushAggOp; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.io.DataFileMeta; @@ -68,6 +70,46 @@ public class PaimonScanNodeTest { @Mock private PaimonFileExternalCatalog paimonFileExternalCatalog; + private PaimonSource mockPaimonSourceWithPartitionKeys(List partitionKeys) { + PaimonSource source = Mockito.mock(PaimonSource.class); + Table paimonTable = Mockito.mock(Table.class); + Mockito.when(paimonTable.partitionKeys()).thenReturn(partitionKeys); + Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); + return source; + } + + @Test + public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() throws UserException { + PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv)); + node.setSource(mockPaimonSourceWithPartitionKeys(Collections.emptyList())); + List dataSplits = Arrays.asList( + mockCountDataSplit("f1.parquet", 4_000), + mockCountDataSplit("f2.parquet", 5_000), + mockCountDataSplit("f3.parquet", 6_000)); + Mockito.doReturn(dataSplits).when(node).getPaimonSplitFromAPI(); + Mockito.when(sv.isForceJniScanner()).thenReturn(true); + Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE"); + Mockito.when(sv.getParallelExecInstanceNum(ArgumentMatchers.nullable(String.class))).thenReturn(1); + + // Before the fix, the raw COUNT opcode made this path keep only parallel representative + // splits and attach the 15,000 metadata rows. BE rejects that shortcut for COUNT(col), so + // it would scan only those representatives and silently miss the discarded DataSplits. + node.setPushDownAggNoGrouping(TPushAggOp.COUNT); + node.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + List countColumnSplits = node.getSplits(1); + Assert.assertEquals(3, countColumnSplits.size()); + for (org.apache.doris.spi.Split split : countColumnSplits) { + Assert.assertFalse(((PaimonSplit) split).getRowCount().isPresent()); + } + + // COUNT(*) remains metadata-only. The 15,000 rows exceed the parallel threshold, so one + // configured execution instance retains one representative split carrying the full sum. + node.setPushDownCountSlotIds(Collections.emptyList()); + List countStarSplits = node.getSplits(1); + Assert.assertEquals(1, countStarSplits.size()); + Assert.assertEquals(Optional.of(15_000L), ((PaimonSplit) countStarSplits.get(0)).getRowCount()); + } + @Test public void testSplitWeight() throws UserException { @@ -744,4 +786,20 @@ private DataSplit createDataSplit(String fileName) { .withDataFiles(Collections.singletonList(dataFileMeta)) .build(); } + + private DataSplit mockCountDataSplit(String fileName, long rowCount) { + DataFileMeta dataFileMeta = DataFileMeta.forAppend(fileName, 64L * 1024 * 1024, rowCount, + SimpleStats.EMPTY_STATS, 1L, 1L, 1L, Collections.emptyList(), null, + FileSource.APPEND, Collections.emptyList(), null, null, + Collections.emptyList()); + DataSplit dataSplit = Mockito.mock(DataSplit.class); + Mockito.when(dataSplit.rowCount()).thenReturn(rowCount); + Mockito.when(dataSplit.mergedRowCountAvailable()).thenReturn(true); + Mockito.when(dataSplit.mergedRowCount()).thenReturn(rowCount); + Mockito.when(dataSplit.partition()).thenReturn(BinaryRow.singleColumn(1)); + Mockito.when(dataSplit.dataFiles()).thenReturn(Collections.singletonList(dataFileMeta)); + Mockito.when(dataSplit.convertToRawFiles()).thenReturn(Optional.empty()); + Mockito.when(dataSplit.deletionFiles()).thenReturn(Optional.empty()); + return dataSplit; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java index e6c5b0be328f99..d2627befd3734c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java @@ -21,7 +21,6 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; -import org.apache.doris.nereids.exceptions.AnalysisException; import mockit.Expectations; import mockit.Mocked; @@ -37,8 +36,8 @@ public class IcebergSysTableResolverTest { private IcebergExternalDatabase db; @Test - public void testSupportedSysTablesExcludePositionDeletes() { - Assertions.assertFalse(IcebergSysTable.SUPPORTED_SYS_TABLES.containsKey(IcebergSysTable.POSITION_DELETES)); + public void testSupportedSysTablesIncludePositionDeletes() { + Assertions.assertTrue(IcebergSysTable.SUPPORTED_SYS_TABLES.containsKey("position_deletes")); } @Test @@ -61,10 +60,14 @@ public void testResolveForPlanAndDescribeUseNativePath() throws Exception { } @Test - public void testPositionDeletesKeepsUnsupportedError() throws Exception { + public void testPositionDeletesUseNativePath() throws Exception { IcebergExternalTable sourceTable = newIcebergTable(); - Assertions.assertThrows(AnalysisException.class, () -> - SysTableResolver.resolveForPlan(sourceTable, "test_ctl", "test_db", "tbl$position_deletes")); + Optional plan = SysTableResolver.resolveForPlan( + sourceTable, "test_ctl", "test_db", "tbl$position_deletes"); + Assertions.assertTrue(plan.isPresent()); + Assertions.assertTrue(plan.get().isNative()); + Assertions.assertTrue(plan.get().getSysExternalTable() instanceof IcebergSysExternalTable); + Assertions.assertEquals("tbl$position_deletes", plan.get().getSysExternalTable().getName()); } private IcebergExternalTable newIcebergTable() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index 8cf98daea94c59..7645f0675074ce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -17,15 +17,22 @@ package org.apache.doris.datasource.tvf.source; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.FunctionGenTable; +import org.apache.doris.datasource.FileQueryScanNode; +import org.apache.doris.datasource.FileSplitter; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.spi.Split; import org.apache.doris.tablefunction.ExternalFileTableValuedFunction; import org.apache.doris.thrift.TBrokerFileStatus; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TPushAggOp; +import com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -37,6 +44,40 @@ public class TVFScanNodeTest { private static final long MB = 1024L * 1024L; + @Test + public void testCountColumnKeepsNormalFileSplitting() throws Exception { + SessionVariable sv = new SessionVariable(); + sv.parallelExecInstanceNum = 1; + sv.setFileSplitSize(32 * MB); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + FunctionGenTable table = Mockito.mock(FunctionGenTable.class); + ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); + Mockito.when(table.getTvf()).thenReturn(tvf); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_LOCAL); + Mockito.when(tvf.getFileStatuses()).thenReturn(ImmutableList.of( + splittableFile("file:///tmp/count_col_1.parquet", 128 * MB), + splittableFile("file:///tmp/count_col_2.parquet", 128 * MB))); + desc.setTable(table); + + TVFScanNode countColumnNode = new TVFScanNode( + new PlanNodeId(0), desc, false, sv, ScanContext.EMPTY); + setFileSplitter(countColumnNode, new FileSplitter(32 * MB, 32 * MB, 0)); + countColumnNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countColumnNode.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + + List countColumnSplits = countColumnNode.getSplits(1); + Assert.assertEquals(8, countColumnSplits.size()); + + TVFScanNode countStarNode = new TVFScanNode( + new PlanNodeId(1), desc, false, sv, ScanContext.EMPTY); + setFileSplitter(countStarNode, new FileSplitter(32 * MB, 32 * MB, 0)); + countStarNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countStarNode.setPushDownCountSlotIds(Collections.emptyList()); + + List countStarSplits = countStarNode.getSplits(1); + Assert.assertEquals(2, countStarSplits.size()); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); @@ -57,4 +98,19 @@ public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Excep long target = (long) method.invoke(node, statuses); Assert.assertEquals(100 * MB, target); } + + private static TBrokerFileStatus splittableFile(String path, long size) { + TBrokerFileStatus status = new TBrokerFileStatus(); + status.setPath(path); + status.setSize(size); + status.setModificationTime(0); + status.setIsSplitable(true); + return status; + } + + private static void setFileSplitter(TVFScanNode node, FileSplitter splitter) throws Exception { + java.lang.reflect.Field field = FileQueryScanNode.class.getDeclaredField("fileSplitter"); + field.setAccessible(true); + field.set(node, splitter); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java index 159cd611676086..b49f8c8a19e84d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java @@ -22,12 +22,15 @@ import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.processor.post.PlanPostProcessors; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.MaterializationNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -80,6 +83,29 @@ public void test2() throws Exception { Assertions.assertEquals("k2", slots.get(0).getColumn().getName()); } + @Test + public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Exception { + this.createTable("create table lazy_materialize_alias_tbl(" + + "sort_col int, lazy_col int) " + + "duplicate key(sort_col) distributed by hash(sort_col) buckets 1 " + + "properties('replication_num' = '1')"); + String sql = "select lazy_col as lazy_alias from lazy_materialize_alias_tbl " + + "order by sort_col limit 1"; + + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + List> materializeNodes = plan.collectToList( + node -> node instanceof PhysicalLazyMaterialize); + Assertions.assertEquals(1, materializeNodes.size(), plan.treeString()); + Assertions.assertEquals(ImmutableList.of(ImmutableList.of(1)), + materializeNodes.get(0).getLazyBaseColumnIndices()); + } + @Test public void testLightSchemaChangeFalse() throws Exception { this.createTable("create table tm_lsc_false (k int, v int) duplicate key(k) " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java index bf5b74f2ea6f25..d492100e867f29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java @@ -17,6 +17,12 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; @@ -27,7 +33,10 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ln; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate.PushDownAggOp; @@ -38,6 +47,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.Optional; @@ -75,7 +85,26 @@ public void testWithoutProject() { .applyImplementation(storageLayerAggregateWithoutProject()) .matches( logicalAggregate( - physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT) + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(olapScan.getOutput().get(0).getExprId()))) + ) + ); + + // COUNT(*) still keeps a placeholder scan slot after column pruning, so its semantic + // argument list must remain empty instead of being inferred from the physical scan shape. + aggregate = new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(), "count_star")), + true, Optional.empty(), olapScan); + context = MemoTestUtils.createCascadesContext(aggregate); + + PlanChecker.from(context) + .applyImplementation(storageLayerAggregateWithoutProject()) + .matches( + logicalAggregate( + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().isEmpty()) ) ); @@ -96,6 +125,67 @@ public void testWithoutProject() { ); } + @Test + public void testNullableFileCountUsesStorageLayerAggregate() { + LogicalAggregate aggregate = newNullableFileCountAggregate(); + LogicalFileScan fileScan = aggregate.child(); + + PlanChecker.from(MemoTestUtils.createCascadesContext(aggregate)) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .matches(logicalAggregate( + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(fileScan.getOutput().get(0).getExprId()))))); + } + + @Test + public void testNullableFileCountDoesNotUseV1StorageLayerAggregate() { + LogicalAggregate aggregate = newNullableFileCountAggregate(); + CascadesContext context = MemoTestUtils.createCascadesContext(aggregate); + context.getConnectContext().getSessionVariable().enableFileScannerV2 = false; + + PlanChecker.from(context) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .nonMatch(physicalStorageLayerAggregate()); + } + + @Test + public void testMixedCountStarAndNullableFileCountDoesNotUseStorageLayerAggregate() { + LogicalAggregate nullableCount = newNullableFileCountAggregate(); + LogicalFileScan fileScan = nullableCount.child(); + LogicalAggregate mixedCount = new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(), "count_star"), + new Alias(new Count(fileScan.getOutput().get(0)), "count_nullable")), + true, Optional.empty(), fileScan); + + PlanChecker.from(MemoTestUtils.createCascadesContext(mixedCount)) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .nonMatch(physicalStorageLayerAggregate()); + } + + private LogicalAggregate newNullableFileCountAggregate() { + Column nullableColumn = new Column("value", Type.INT, true); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema()).thenReturn(ImmutableList.of(nullableColumn)); + Mockito.when(table.getName()).thenReturn("nullable_file_table"); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getName()).thenReturn("catalog"); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(table.getDatabase()).thenReturn(database); + LogicalFileScan fileScan = new LogicalFileScan(new RelationId(1), table, + ImmutableList.of("catalog", "db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + return new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(fileScan.getOutput().get(0)), "count")), + true, Optional.empty(), fileScan); + } + @Override public RulePromise defaultPromise() { return RulePromise.IMPLEMENT; @@ -138,7 +228,9 @@ public void testWithProject() { .matches( logicalAggregate( logicalProject( - physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT) + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(olapScan.getOutput().get(0).getExprId()))) ) ) ); @@ -197,6 +289,15 @@ private Rule storageLayerAggregateWithoutProject() { .get(); } + private Rule storageLayerAggregateWithoutProjectForFileScan() { + return new AggregateStrategies().buildRules() + .stream() + .filter(rule -> rule.getRuleType() + == RuleType.STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT_FOR_FILE_SCAN) + .findFirst() + .get(); + } + private Rule storageLayerAggregateWithProject() { return new AggregateStrategies().buildRules() .stream() diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java index 23dcb4403ce731..74d231d3154a25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java @@ -27,6 +27,7 @@ import org.apache.doris.thrift.TIcebergMergeSink; import org.apache.doris.thrift.TIcebergRewritableDeleteFileSet; +import com.google.common.collect.ImmutableMap; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; @@ -74,6 +75,63 @@ public void testBindDataSinkSkipsRewritableDeleteFileSetsAndRowLineageSchemaForV IcebergUtils.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL)); } + @Test + public void testBindDataSinkDisablesColumnStatsWhenAllMetricsAreNone() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, ImmutableMap.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")), new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertFalse(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForMetricsOverride() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, ImmutableMap.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "counts")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForV3LineageFields() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3, ImmutableMap.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForOrcTopLevelComplexField() throws Exception { + Schema schema = new Schema(Types.NestedField.optional(1, "items", + Types.ListType.ofOptional(2, Types.IntegerType.get()))); + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, schema, ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "orc", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "items", "counts")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + private static TIcebergRewritableDeleteFileSet buildDeleteFileSet() { TIcebergDeleteFileDesc deleteFileDesc = new TIcebergDeleteFileDesc(); deleteFileDesc.setPath("file:///tmp/delete.puffin"); @@ -84,13 +142,24 @@ private static TIcebergRewritableDeleteFileSet buildDeleteFileSet() { } private static IcebergExternalTable mockIcebergExternalTable(int formatVersion) { - Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + return mockIcebergExternalTable(formatVersion, Collections.emptyMap()); + } + + private static IcebergExternalTable mockIcebergExternalTable( + int formatVersion, Map metricsProperties) { + return mockIcebergExternalTable(formatVersion, + new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())), metricsProperties); + } + + private static IcebergExternalTable mockIcebergExternalTable( + int formatVersion, Schema schema, Map metricsProperties) { PartitionSpec spec = PartitionSpec.unpartitioned(); Map properties = new HashMap<>(); properties.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)); properties.put(TableProperties.DEFAULT_FILE_FORMAT, "parquet"); properties.put(TableProperties.PARQUET_COMPRESSION, "snappy"); properties.put(TableProperties.WRITE_DATA_LOCATION, "file:///tmp/iceberg_tbl/data"); + properties.putAll(metricsProperties); Table icebergTable = Mockito.mock(Table.class); Mockito.when(icebergTable.properties()).thenReturn(properties); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 2de5d19e7fb738..02356bb094efa6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -39,6 +39,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergHadoopExternalCatalog; +import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; @@ -49,17 +50,31 @@ import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.ResultRow; import org.apache.doris.statistics.TableStatsMeta; +import org.apache.doris.thrift.TIcebergColumnStats; +import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TStorageType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import mockit.Mock; import mockit.MockUp; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalTime; import java.time.format.DateTimeFormatter; @@ -70,6 +85,63 @@ import java.util.Map; class StatisticsUtilTest { + @Test + void testGetIcebergColumnStatsReturnsEmptyForDisabledMetrics() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).build(); + org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(spec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(ImmutableMap.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(ImmutableMap.of(1, 128L)); + columnStats.setValueCounts(ImmutableMap.of(1, 10L)); + columnStats.setNullValueCounts(ImmutableMap.of(1, 0L)); + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, ImmutableList.of(commitData)).dataFiles()[0]; + + TableScan tableScan = Mockito.mock(TableScan.class); + FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); + Mockito.when(table.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); + Mockito.when(tableScan.planFiles()) + .thenReturn(CloseableIterable.withNoopClose(ImmutableList.of(fileScanTask))); + Mockito.when(fileScanTask.spec()).thenReturn(spec); + Mockito.when(fileScanTask.file()).thenReturn(dataFile); + + Assertions.assertTrue(!StatisticsUtil.getIcebergColumnStats("id", table).isPresent()); + } + + @Test + void testGetIcebergColumnStatsReturnsEmptyWhenCloseFails() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).build(); + org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class); + TableScan tableScan = Mockito.mock(TableScan.class); + FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); + DataFile dataFile = Mockito.mock(DataFile.class); + Mockito.when(table.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); + Mockito.when(tableScan.planFiles()).thenReturn(CloseableIterable.combine( + ImmutableList.of(fileScanTask), () -> { + throw new IOException("close failed"); + })); + Mockito.when(fileScanTask.spec()).thenReturn(spec); + Mockito.when(fileScanTask.file()).thenReturn(dataFile); + Mockito.when(dataFile.columnSizes()).thenReturn(ImmutableMap.of()); + Mockito.when(dataFile.nullValueCounts()).thenReturn(ImmutableMap.of()); + + Assertions.assertTrue(!StatisticsUtil.getIcebergColumnStats("id", table).isPresent()); + } + @Test void testConvertToDouble() { try { diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 3640117def87ab..a0cb46f11669d9 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -486,6 +486,8 @@ struct TIcebergTableSink { 15: optional map static_partition_values; 16: optional PlanNodes.TSortInfo sort_info; 17: optional TIcebergWriteType write_type = TIcebergWriteType.INSERT; + // Unset keeps collection enabled for rolling upgrades with older FEs. + 18: optional bool collect_column_stats; } struct TIcebergRewritableDeleteFileSet { @@ -532,6 +534,8 @@ struct TIcebergMergeSink { 11: optional map hadoop_config 12: optional Types.TFileType file_type 13: optional list broker_addresses; + // Unset keeps collection enabled for rolling upgrades with older FEs. + 14: optional bool collect_column_stats; // delete side (position delete only) 20: optional TFileContent delete_type diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 46ae54781ae825..86915e46d28bfb 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -58,7 +58,10 @@ struct TField { // True when initial_default_value is Base64 and must be decoded before constructing the Doris // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. - 8: optional bool initial_default_value_is_base64 + 8: optional bool initial_default_value_is_base64, + // Version marker for authoritative Iceberg mapping semantics. Its absence preserves the + // legacy name fallback when a new BE executes a plan produced by an older FE during rollout. + 9: optional bool name_mapping_is_authoritative } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 47bc948f2291c1..8ae188a73118f0 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -253,9 +253,21 @@ struct TFileTextScanRangeParams { 8: optional bool empty_field_as_null } +enum TColumnCategory { + REGULAR = 0, + PARTITION_KEY = 1, + SYNTHESIZED = 2, + GENERATED = 3, +} + struct TFileScanSlotInfo { 1: optional Types.TSlotId slot_id; 2: optional bool is_file_slot; + 3: optional TColumnCategory category; + // Default value expression for this column when it is missing from the data file. + // Populated by FE from Column.getDefaultValue() or NULL literal. + // This replaces the separate default_value_of_src_slot map in TFileScanRangeParams. + 4: optional Exprs.TExpr default_value_expr; } // descirbe how to read file @@ -297,6 +309,10 @@ struct TIcebergDeleteFileDesc { 6: optional i64 content_offset; 7: optional i64 content_size_in_bytes; 8: optional TFileFormatType file_format; + // Original Iceberg delete file path before Doris storage path normalization. + 9: optional string original_path; + // Referenced data file path. Required to materialize rows from deletion vectors. + 10: optional string referenced_data_file_path; } struct TIcebergFileDesc { @@ -432,6 +448,8 @@ struct TTableFormatFileDesc { 8: optional TLakeSoulFileDesc lakesoul_params 9: optional i64 table_level_row_count = -1 10: optional TRemoteDorisFileDesc remote_doris_params + // JDBC connection parameters (used when table_format_type == "jdbc") + 11: optional map jdbc_params } // Deprecated, hive text talbe is a special format, not a serde type @@ -508,6 +526,10 @@ struct TFileScanRangeParams { // Paimon options from FE, used for jni/native scanner // Set at ScanNode level to avoid redundant serialization in each split 30: optional map paimon_options + // Versioned Iceberg scan semantics negotiated by FE. Absence/zero preserves legacy BE + // behavior during a BE-first rolling upgrade; version 1 enables file-wide ID projection and + // logical initial-default materialization. + 34: optional i32 iceberg_scan_semantics_version } struct TFileRangeDesc { @@ -1548,6 +1570,10 @@ struct TPlanNode { 50: optional list> distribute_expr_lists 51: optional bool is_serial_operator 52: optional TRecCTEScanNode rec_cte_scan_node + // COUNT(*) and COUNT(col) share push_down_agg_type_opt=COUNT, but file readers need to know + // whether a projected scan slot is the aggregate argument or merely the placeholder retained by + // column pruning. Empty means row-count semantics; non-empty identifies explicit COUNT columns. + 55: optional list push_down_count_slot_ids // projections is final projections, which means projecting into results and materializing them into the output block. 101: optional list projections diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out new file mode 100644 index 00000000000000..d7f3395f293c2a --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partition_rename_field -- +0 10 +1 10 diff --git a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out new file mode 100644 index 00000000000000..0ad208e4be8889 --- /dev/null +++ b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !clickhouse_v2_schema -- +2 + +-- !clickhouse_v2_distributed -- +2 + +-- !clickhouse_v2_materialized_view -- +2 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv b/regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet new file mode 100644 index 00000000000000..22eff58b7a2389 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet differ diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet new file mode 100644 index 00000000000000..5a9ca0baa47e89 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet differ diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet new file mode 100644 index 00000000000000..5b973181a56148 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet differ diff --git a/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out b/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out new file mode 100644 index 00000000000000..c8312571c56e36 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !empty_csv -- + +-- !struct_leaf_promotion_cold -- +1 10 100 +3 10 300 + +-- !struct_leaf_promotion_warm -- +1 10 100 +3 10 300 + +-- !count_evolved_struct -- +4 + +-- !unprojected_unsupported_time -- +1 +2 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index 4fe42d7fcdcc77..bb74c79d5a6907 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -808,10 +808,10 @@ true -- !test_98 -- \N \N \N -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -866,10 +866,10 @@ abcDeFGhijkLmnOp \N 1212 -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml index 9efd3b6fe97cd4..c5b81b39ce38db 100644 --- a/regression-test/framework/pom.xml +++ b/regression-test/framework/pom.xml @@ -210,7 +210,7 @@ under the License. org.junit.jupiter junit-jupiter-api - 5.8.2 + 5.10.2 mysql diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 0075e501545a3d..70e5c6e11a4d7d 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -3655,35 +3655,35 @@ class Suite implements GroovyInterceptable { throw new IllegalArgumentException("invalid caseElapseSeconds, ${caseElapseSeconds}") } - long sleepSeconds = 0 - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now() + LocalDateTime boundary switch (caseSpanConstraint) { case "NOT_CROSS_HOUR_BOUNDARY": - LocalDateTime nextHour = now.withMinute(0).withSecond(0).withNano(0).plusHours(1); - long secondsToNextHour = ChronoUnit.SECONDS.between(now, nextHour) - - if (secondsToNextHour < caseElapseSeconds) { - sleepSeconds = secondsToNextHour - } + boundary = now.withMinute(0).withSecond(0).withNano(0).plusHours(1) break case "NOT_CROSS_DAY_BOUNDARY": - LocalDateTime startOfNextDay = now.toLocalDate().plusDays(1).atStartOfDay(); - long secondsToNextDay = ChronoUnit.SECONDS.between(now, startOfNextDay) - - if (secondsToNextDay < caseElapseSeconds) { - sleepSeconds = secondsToNextDay - } + boundary = now.toLocalDate().plusDays(1).atStartOfDay() break default: throw new IllegalArgumentException("invalid caseSpanConstraint:${caseSpanConstraint}") } - if (sleepSeconds > 0) { - logger.info("test sleeps ${sleepSeconds} to satisfy ${caseSpanConstraint}") - Thread.sleep(sleepSeconds * 1000) + long sleepMillis = calculateBoundarySleepMillis(now, boundary, caseElapseSeconds) + if (sleepMillis > 0) { + logger.info("test sleeps ${sleepMillis} ms to satisfy ${caseSpanConstraint}") + Thread.sleep(sleepMillis) + } + } + + static long calculateBoundarySleepMillis(LocalDateTime now, LocalDateTime boundary, int caseElapseSeconds) { + long millisToBoundary = ChronoUnit.MILLIS.between(now, boundary) + if (millisToBoundary <= TimeUnit.SECONDS.toMillis(caseElapseSeconds)) { + // Cross the boundary by a full millisecond; truncated fractional milliseconds must not resume early. + return millisToBoundary + 1 } + return 0 } void retryUntilHasSqlCache(String sql) { diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy new file mode 100644 index 00000000000000..4b8693e7678d6a --- /dev/null +++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import org.junit.jupiter.api.Test + +import java.time.LocalDateTime +import java.util.concurrent.TimeUnit + +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertTrue + +class SuiteBoundaryWaitTest { + @Test + void waitCrossesBoundaryWhenRemainingTimeHasFractionalSecond() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:40.285999999") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + long waitMillis = Suite.calculateBoundarySleepMillis(now, nextHour, 45) + + assertTrue(now.plusNanos(TimeUnit.MILLISECONDS.toNanos(waitMillis)).isAfter(nextHour)) + } + + @Test + void waitCrossesBoundaryWhenExpectedDurationEndsExactlyAtBoundary() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:15") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + long waitMillis = Suite.calculateBoundarySleepMillis(now, nextHour, 45) + + assertTrue(now.plusNanos(TimeUnit.MILLISECONDS.toNanos(waitMillis)).isAfter(nextHour)) + } + + @Test + void noWaitWhenExpectedDurationFinishesBeforeBoundary() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:14.999") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + assertEquals(0, Suite.calculateBoundarySleepMillis(now, nextHour, 45)) + } +} diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index 189634a4134704..222d3a393b215e 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -223,6 +223,15 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") } + // Output aliases must not be used to resolve physical Hive column indices. + explain { + sql "select name as lazy_alias from orc_topn_lazy_mat_table order by id limit 10;" + contains("VMaterializeNode") + contains("column_descs_lists[[`name` text NULL]]") + contains("column_idxs_lists: [[1]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") + } + explain { sql """ select a.name,length(a.name),a.value,b.*,a.* from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name limit 10 """ diff --git a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy index fbb0e343e09039..52cd0c60ad36f0 100644 --- a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy @@ -177,25 +177,25 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("2", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("2", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("1", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("1", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -208,25 +208,25 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -239,25 +239,25 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -271,26 +271,26 @@ suite("test_orc_lazy_mat_profile", "p0,external,hive,external_docker,external_do def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("8", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("8", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("7", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("7", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("6", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("6", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("9", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("9", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy new file mode 100644 index 00000000000000..dae16a9f3c7b66 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy @@ -0,0 +1,659 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_position_deletes_sys_table", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_position_deletes_sys_table" + String dbName = "position_deletes_sys_table_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + );""" + + spark_iceberg_multi """ + SET spark.sql.shuffle.partitions=1; + CREATE DATABASE IF NOT EXISTS demo.${dbName}; + DROP TABLE IF EXISTS demo.${dbName}.pd_unpartitioned; + CREATE TABLE demo.${dbName}.pd_unpartitioned ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_unpartitioned + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd') AS t(id, name); + DELETE FROM demo.${dbName}.pd_unpartitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_orc_unpartitioned; + CREATE TABLE demo.${dbName}.pd_orc_unpartitioned ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.format.default'='orc', + 'write.delete.format.default'='orc', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_orc_unpartitioned + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c') AS t(id, name); + DELETE FROM demo.${dbName}.pd_orc_unpartitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_partitioned; + CREATE TABLE demo.${dbName}.pd_partitioned ( + id INT, + name STRING, + dt STRING + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_partitioned + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (1, 'a', '2026-06-26'), + (2, 'b', '2026-06-26'), + (3, 'c', '2026-06-27'), + (4, 'd', '2026-06-27') AS t(id, name, dt); + DELETE FROM demo.${dbName}.pd_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_int_partitioned; + CREATE TABLE demo.${dbName}.pd_int_partitioned ( + id INT, + name STRING, + p INT + ) USING iceberg + PARTITIONED BY (p) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_int_partitioned + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (1, 'a', 10), + (2, 'b', 10), + (3, 'c', 20) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_int_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_date_partitioned; + CREATE TABLE demo.${dbName}.pd_date_partitioned ( + id INT, + name STRING, + dt DATE + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_date_partitioned + SELECT /*+ COALESCE(1) */ id, name, CAST(dt AS DATE) FROM VALUES + (1, 'a', '2026-01-02'), + (2, 'b', '2026-01-02'), + (3, 'c', '2026-03-04') AS t(id, name, dt); + DELETE FROM demo.${dbName}.pd_date_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_partition_evolution; + CREATE TABLE demo.${dbName}.pd_partition_evolution ( + id INT, + name STRING, + p INT + ) USING iceberg + PARTITIONED BY (p) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_partition_evolution + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (1, 'a', 10), + (2, 'b', 10), + (3, 'c', 20), + (4, 'd', 20) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_evolution WHERE id = 2; + ALTER TABLE demo.${dbName}.pd_partition_evolution ADD PARTITION FIELD bucket(1, id); + INSERT INTO demo.${dbName}.pd_partition_evolution + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (5, 'e', 10), + (6, 'f', 10) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_evolution WHERE id = 6; + DROP TABLE IF EXISTS demo.${dbName}.pd_partition_rename; + CREATE TABLE demo.${dbName}.pd_partition_rename ( + id INT, + name STRING, + p INT + ) USING iceberg + PARTITIONED BY (p) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_partition_rename + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (1, 'a', 10), + (2, 'b', 10), + (3, 'c', 20), + (4, 'd', 20) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_rename WHERE id = 2; + -- Replacing an identity transform with the same transform and an alias renames the + -- partition field while preserving its Iceberg field ID. + ALTER TABLE demo.${dbName}.pd_partition_rename REPLACE PARTITION FIELD p WITH p AS p2; + INSERT INTO demo.${dbName}.pd_partition_rename + SELECT /*+ COALESCE(1) */ id, name, p FROM VALUES + (5, 'e', 10), + (6, 'f', 10) AS t(id, name, p); + DELETE FROM demo.${dbName}.pd_partition_rename WHERE id = 6; + DROP TABLE IF EXISTS demo.${dbName}.pd_v3_unpartitioned; + CREATE TABLE demo.${dbName}.pd_v3_unpartitioned ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_v3_unpartitioned + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c') AS t(id, name); + DELETE FROM demo.${dbName}.pd_v3_unpartitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_v3_partitioned; + CREATE TABLE demo.${dbName}.pd_v3_partitioned ( + id INT, + name STRING, + dt STRING + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='3', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_v3_partitioned + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (1, 'a', '2026-07-01'), + (2, 'b', '2026-07-01'), + (3, 'c', '2026-07-02') AS t(id, name, dt); + DELETE FROM demo.${dbName}.pd_v3_partitioned WHERE id = 2; + DROP TABLE IF EXISTS demo.${dbName}.pd_schema_time_travel; + CREATE TABLE demo.${dbName}.pd_schema_time_travel ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_schema_time_travel + SELECT /*+ COALESCE(1) */ id, name FROM VALUES + (1, 'a'), (2, 'b'), (3, 'c') AS t(id, name); + DELETE FROM demo.${dbName}.pd_schema_time_travel WHERE id = 2; + ALTER TABLE demo.${dbName}.pd_schema_time_travel ADD COLUMN note STRING; + INSERT INTO demo.${dbName}.pd_schema_time_travel + SELECT /*+ COALESCE(1) */ id, name, note FROM VALUES + (4, 'd', 'new'), (5, 'e', 'new') AS t(id, name, note); + DELETE FROM demo.${dbName}.pd_schema_time_travel WHERE id = 4; + DROP TABLE IF EXISTS demo.${dbName}.pd_no_deletes; + CREATE TABLE demo.${dbName}.pd_no_deletes ( + id INT, + name STRING + ) USING iceberg + TBLPROPERTIES ('format-version'='2'); + INSERT INTO demo.${dbName}.pd_no_deletes VALUES (1, 'a'), (2, 'b'); + """ + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + def assertPositionDeletesSchema = { String tableName, boolean partitioned, List extraColumns = [] -> + List> descRows = sql """desc ${tableName}\$position_deletes""" + List columns = descRows.collect { it[0].toString() } + List expectedColumns = ["file_path", "pos", "row"] + if (partitioned) { + expectedColumns.add("partition") + } + expectedColumns.addAll(["spec_id", "delete_file_path"]) + expectedColumns.addAll(extraColumns) + assertEquals(expectedColumns, columns) + } + + def countRows = { String query -> + List> rows = sql query + assertEquals(1, rows.size()) + return ((Number) rows[0][0]).longValue() + } + + def assertSparkDorisPositionDeletes = { String tableName, List columns -> + String columnList = columns.join(", ") + String orderBy = "file_path, pos, delete_file_path" + List> sparkRows = spark_iceberg """ + select ${columnList} + from demo.${dbName}.${tableName}.position_deletes + order by ${orderBy} + """ + List> dorisRows = sql """ + select ${columnList} + from ${tableName}\$position_deletes + order by ${orderBy} + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertSparkDorisTableRows = { String tableName, List> expectedRows -> + spark_iceberg """REFRESH TABLE demo.${dbName}.${tableName}""" + List> sparkRows = spark_iceberg """ + select id, name, dt + from demo.${dbName}.${tableName} + order by id + """ + List> dorisRows = sql """ + select id, name, dt + from ${tableName} + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + assertEquals(expectedRows, dorisRows) + } + + List v3ExtraColumns = ["content_offset", "content_size_in_bytes"] + List commonCompareColumns = ["file_path", "pos", "spec_id", "delete_file_path"] + List v3CompareColumns = commonCompareColumns + v3ExtraColumns + + assertPositionDeletesSchema("pd_unpartitioned", false) + long unpartitionedCount = countRows("""select count(*) from pd_unpartitioned\$position_deletes""") + assertTrue(unpartitionedCount > 0) + assertSparkDorisPositionDeletes("pd_unpartitioned", commonCompareColumns) + assertEquals(unpartitionedCount, countRows("""select count(pos) from pd_unpartitioned\$position_deletes""")) + assertEquals(unpartitionedCount, (long) sql( + """select * from pd_unpartitioned\$position_deletes""").size()) + assertEquals(unpartitionedCount, (long) sql( + """select pos from pd_unpartitioned\$position_deletes""").size()) + assertEquals(unpartitionedCount, (long) sql( + """select delete_file_path, pos from pd_unpartitioned\$position_deletes""").size()) + assertEquals(unpartitionedCount, (long) sql( + """select file_path, pos, delete_file_path from pd_unpartitioned\$position_deletes where pos >= 0""").size()) + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where spec_id = 0""")) + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where delete_file_path is not null""")) + assertEquals(0L, countRows("""select count(*) from pd_unpartitioned\$position_deletes where pos < 0""")) + List> unpartitionedRows = sql """select `row` from pd_unpartitioned\$position_deletes""" + assertEquals(unpartitionedCount, (long) unpartitionedRows.size()) + + assertPositionDeletesSchema("pd_orc_unpartitioned", false) + long orcUnpartitionedCount = countRows( + """select count(*) from pd_orc_unpartitioned\$position_deletes""") + assertTrue(orcUnpartitionedCount > 0) + assertSparkDorisPositionDeletes("pd_orc_unpartitioned", commonCompareColumns) + List> orcUnpartitionedRows = sql """select `row` from pd_orc_unpartitioned\$position_deletes""" + assertEquals(orcUnpartitionedCount, (long) orcUnpartitionedRows.size()) + assertTrue(orcUnpartitionedRows.every { it[0] == null }) + try { + sql """set file_split_size=1""" + assertSparkDorisPositionDeletes("pd_unpartitioned", commonCompareColumns) + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where pos >= 0""")) + assertEquals(orcUnpartitionedCount, countRows( + """select count(*) from pd_orc_unpartitioned\$position_deletes where pos >= 0""")) + } finally { + sql """unset variable file_split_size""" + } + assertEquals([[1, "a"], [3, "c"], [4, "d"]], sql("""select * from pd_unpartitioned order by id""")) + + assertPositionDeletesSchema("pd_partitioned", true) + long partitionedCount = countRows("""select count(*) from pd_partitioned\$position_deletes""") + assertTrue(partitionedCount > 0) + assertSparkDorisPositionDeletes("pd_partitioned", commonCompareColumns) + assertEquals(partitionedCount, countRows("""select count(pos) from pd_partitioned\$position_deletes""")) + assertEquals(partitionedCount, (long) sql( + """select * from pd_partitioned\$position_deletes""").size()) + assertEquals(partitionedCount, (long) sql( + """select file_path, pos, delete_file_path from pd_partitioned\$position_deletes where pos >= 0""").size()) + assertEquals(partitionedCount, countRows( + """select count(*) from pd_partitioned\$position_deletes where `partition` is not null""")) + assertEquals(0L, countRows("""select count(*) from pd_partitioned\$position_deletes where pos < 0""")) + List> partitionedRows = sql """select `row`, `partition` from pd_partitioned\$position_deletes""" + assertEquals(partitionedCount, (long) partitionedRows.size()) + assertTrue(partitionedRows.every { it[0] == null && it[1] != null }) + assertEquals([[1, "a", "2026-06-26"], [3, "c", "2026-06-27"], [4, "d", "2026-06-27"]], + sql("""select * from pd_partitioned order by id""")) + + assertPositionDeletesSchema("pd_int_partitioned", true) + long intPartitionedCount = countRows("""select count(*) from pd_int_partitioned\$position_deletes""") + assertTrue(intPartitionedCount > 0) + assertEquals(intPartitionedCount, countRows("""select count(pos) from pd_int_partitioned\$position_deletes""")) + assertEquals(intPartitionedCount, countRows( + """select count(*) from pd_int_partitioned\$position_deletes where `partition` is not null""")) + List> intPartitionedRows = sql """select `row`, `partition` from pd_int_partitioned\$position_deletes""" + assertEquals(intPartitionedCount, (long) intPartitionedRows.size()) + assertTrue(intPartitionedRows.every { it[0] == null && it[1] != null }) + assertTrue(intPartitionedRows.every { + String partitionValue = it[1].toString() + partitionValue.contains("\"p\":10") && !partitionValue.contains("\"p\":\"10\"") + }) + assertEquals([[1, "a", 10], [3, "c", 20]], sql("""select * from pd_int_partitioned order by id""")) + + // DATE-typed partition column. Iceberg serializes the partition value to an ISO date string + // (`2026-01-02`) which the BE parses back into the metadata table `partition` struct DATE field. + // Verifies the typed partition round-trip is not limited to STRING/INT columns. + assertPositionDeletesSchema("pd_date_partitioned", true) + long datePartitionedCount = countRows("""select count(*) from pd_date_partitioned\$position_deletes""") + assertEquals(1L, datePartitionedCount) + assertSparkDorisPositionDeletes("pd_date_partitioned", commonCompareColumns) + assertEquals(datePartitionedCount, countRows( + """select count(*) from pd_date_partitioned\$position_deletes where `partition` is not null""")) + List> datePartitionedRows = sql """ + select `row`, cast(`partition` as string) from pd_date_partitioned\$position_deletes + """ + assertEquals(datePartitionedCount, (long) datePartitionedRows.size()) + assertTrue(datePartitionedRows.every { it[0] == null && it[1] != null }) + // The deleted row (id=2) lives in partition dt=2026-01-02, so the partition value must render + // that exact date rather than a null / days-since-epoch integer / mangled string. + assertTrue(datePartitionedRows.every { it[1].toString().contains("2026-01-02") }) + assertEquals([[1, "a", "2026-01-02"], [3, "c", "2026-03-04"]], + sql("""select id, name, cast(dt as string) from pd_date_partitioned order by id""")) + + assertPositionDeletesSchema("pd_partition_evolution", true) + long partitionEvolutionCount = countRows("""select count(*) from pd_partition_evolution\$position_deletes""") + assertEquals(2L, partitionEvolutionCount) + assertSparkDorisPositionDeletes("pd_partition_evolution", commonCompareColumns) + assertEquals(partitionEvolutionCount, countRows( + """select count(pos) from pd_partition_evolution\$position_deletes""")) + assertEquals(partitionEvolutionCount, countRows( + """select count(*) from pd_partition_evolution\$position_deletes where `partition` is not null""")) + List> partitionEvolutionRows = sql """ + select spec_id, cast(`partition` as string) + from pd_partition_evolution\$position_deletes order by spec_id + """ + assertEquals(2, partitionEvolutionRows.size()) + assertEquals(0, ((Number) partitionEvolutionRows[0][0]).intValue()) + assertEquals(1, ((Number) partitionEvolutionRows[1][0]).intValue()) + String oldSpecPartition = partitionEvolutionRows[0][1].toString() + String newSpecPartition = partitionEvolutionRows[1][1].toString() + assertTrue(oldSpecPartition.contains("\"p\":10")) + assertTrue(oldSpecPartition.contains(":null")) + assertTrue(newSpecPartition.contains("\"p\":10")) + assertFalse(newSpecPartition.contains(":null")) + assertEquals([[1, "a", 10], [3, "c", 20], [4, "d", 20], [5, "e", 10]], + sql("""select * from pd_partition_evolution order by id""")) + + assertPositionDeletesSchema("pd_partition_rename", true) + long partitionRenameCount = countRows("""select count(*) from pd_partition_rename\$position_deletes""") + order_qt_partition_rename_field """ + select spec_id, struct_element(`partition`, 'p2') + from pd_partition_rename\$position_deletes + order by spec_id + """ + List> sparkPartitionRenameRows = spark_iceberg """ + select spec_id, partition.p2 + from demo.${dbName}.pd_partition_rename.position_deletes + order by spec_id + """ + List> dorisPartitionRenameRows = sql """ + select spec_id, struct_element(`partition`, 'p2') + from pd_partition_rename\$position_deletes + order by spec_id + """ + assertSparkDorisResultEquals(sparkPartitionRenameRows, dorisPartitionRenameRows) + assertEquals(partitionRenameCount, countRows(""" + select count(*) from pd_partition_rename\$position_deletes + where struct_element(`partition`, 'p2') = 10 + """)) + + assertPositionDeletesSchema("pd_v3_unpartitioned", false, v3ExtraColumns) + long v3UnpartitionedCount = countRows("""select count(*) from pd_v3_unpartitioned\$position_deletes""") + assertEquals(1L, v3UnpartitionedCount) + assertSparkDorisPositionDeletes("pd_v3_unpartitioned", v3CompareColumns) + assertEquals(v3UnpartitionedCount, countRows( + """select count(pos) from pd_v3_unpartitioned\$position_deletes""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(content_offset) from pd_v3_unpartitioned\$position_deletes""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(content_size_in_bytes) from pd_v3_unpartitioned\$position_deletes""")) + assertEquals(v3UnpartitionedCount, (long) sql( + """select file_path, pos, delete_file_path, content_offset, content_size_in_bytes + from pd_v3_unpartitioned\$position_deletes""").size()) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes + where content_offset >= 0 and content_size_in_bytes > 0""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes where spec_id = 0""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes where delete_file_path is not null""")) + List> v3UnpartitionedRows = sql """select `row` from pd_v3_unpartitioned\$position_deletes""" + assertEquals(v3UnpartitionedCount, (long) v3UnpartitionedRows.size()) + assertTrue(v3UnpartitionedRows.every { it[0] == null }) + assertEquals([[1, "a"], [3, "c"]], sql("""select * from pd_v3_unpartitioned order by id""")) + + assertPositionDeletesSchema("pd_v3_partitioned", true, v3ExtraColumns) + long v3PartitionedCount = countRows("""select count(*) from pd_v3_partitioned\$position_deletes""") + assertEquals(1L, v3PartitionedCount) + assertSparkDorisPositionDeletes("pd_v3_partitioned", v3CompareColumns) + assertEquals(v3PartitionedCount, countRows( + """select count(pos) from pd_v3_partitioned\$position_deletes""")) + assertEquals(v3PartitionedCount, countRows( + """select count(content_offset) from pd_v3_partitioned\$position_deletes""")) + assertEquals(v3PartitionedCount, countRows( + """select count(content_size_in_bytes) from pd_v3_partitioned\$position_deletes""")) + assertEquals(v3PartitionedCount, (long) sql( + """select file_path, pos, delete_file_path, content_offset, content_size_in_bytes + from pd_v3_partitioned\$position_deletes""").size()) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes + where content_offset >= 0 and content_size_in_bytes > 0""")) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes where `partition` is not null""")) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes where spec_id = 0""")) + assertEquals(v3PartitionedCount, countRows( + """select count(*) from pd_v3_partitioned\$position_deletes where delete_file_path is not null""")) + List> v3PartitionedRows = sql """select `row`, `partition` from pd_v3_partitioned\$position_deletes""" + assertEquals(v3PartitionedCount, (long) v3PartitionedRows.size()) + assertTrue(v3PartitionedRows.every { it[0] == null && it[1] != null }) + assertEquals([[1, "a", "2026-07-01"], [3, "c", "2026-07-02"]], + sql("""select * from pd_v3_partitioned order by id""")) + + assertPositionDeletesSchema("pd_schema_time_travel", false) + long schemaChangeCount = countRows("""select count(*) from pd_schema_time_travel\$position_deletes""") + assertEquals(2L, schemaChangeCount) + assertEquals(schemaChangeCount, countRows( + """select count(*) from pd_schema_time_travel\$position_deletes where pos >= 0""")) + assertEquals(schemaChangeCount, countRows( + """select count(*) from pd_schema_time_travel\$position_deletes where spec_id = 0""")) + assertEquals(schemaChangeCount, countRows( + """select count(*) from pd_schema_time_travel\$position_deletes where delete_file_path is not null""")) + assertEquals(0L, countRows("""select count(*) from pd_schema_time_travel\$position_deletes where pos < 0""")) + List> schemaChangeRows = sql """select `row` from pd_schema_time_travel\$position_deletes""" + assertEquals(schemaChangeCount, (long) schemaChangeRows.size()) + assertTrue(schemaChangeRows.every { it[0] == null }) + assertEquals([[1, "a", null], [3, "c", null], [5, "e", "new"]], + sql("""select * from pd_schema_time_travel order by id""")) + + List> schemaChangeSnapshots = sql """ + select snapshot_id from pd_schema_time_travel\$snapshots order by committed_at + """ + assertEquals(4, schemaChangeSnapshots.size()) + Object afterInsertSnapshot = schemaChangeSnapshots.get(0)[0] + Object afterFirstDeleteSnapshot = schemaChangeSnapshots.get(1)[0] + Object afterSecondDeleteSnapshot = schemaChangeSnapshots.get(3)[0] + assertEquals(0L, countRows(""" + select count(*) from pd_schema_time_travel\$position_deletes + for version as of ${afterInsertSnapshot} + """)) + assertEquals(1L, countRows(""" + select count(*) from pd_schema_time_travel\$position_deletes + for version as of ${afterFirstDeleteSnapshot} + """)) + assertEquals(2L, countRows(""" + select count(*) from pd_schema_time_travel\$position_deletes + for version as of ${afterSecondDeleteSnapshot} + """)) + assertEquals([[1, "a"], [2, "b"], [3, "c"]], + sql("""select * from pd_schema_time_travel for version as of ${afterInsertSnapshot} order by id""")) + assertEquals([[1, "a"], [3, "c"]], + sql("""select * from pd_schema_time_travel for version as of ${afterFirstDeleteSnapshot} order by id""")) + assertEquals([[1, "a", null], [3, "c", null], [5, "e", "new"]], + sql("""select * from pd_schema_time_travel for version as of ${afterSecondDeleteSnapshot} order by id""")) + + assertPositionDeletesSchema("pd_no_deletes", false) + assertSparkDorisPositionDeletes("pd_no_deletes", commonCompareColumns) + assertEquals(0L, countRows("""select count(*) from pd_no_deletes\$position_deletes""")) + + def assertPositionDeletesScannerPath = { boolean enableFileScannerV2 -> + sql """set enable_file_scanner_v2 = ${enableFileScannerV2}""" + assertEquals(unpartitionedCount, countRows( + """select count(*) from pd_unpartitioned\$position_deletes where pos >= 0""")) + assertEquals(partitionedCount, countRows( + """select count(*) from pd_partitioned\$position_deletes where `partition` is not null""")) + assertEquals(partitionRenameCount, countRows( + """select count(*) from pd_partition_rename\$position_deletes + where struct_element(`partition`, 'p2') is not null""")) + assertEquals(v3UnpartitionedCount, countRows( + """select count(*) from pd_v3_unpartitioned\$position_deletes + where content_offset >= 0 and content_size_in_bytes > 0""")) + assertSparkDorisPositionDeletes("pd_unpartitioned", commonCompareColumns) + assertSparkDorisPositionDeletes("pd_orc_unpartitioned", commonCompareColumns) + assertSparkDorisPositionDeletes("pd_v3_unpartitioned", v3CompareColumns) + } + assertPositionDeletesScannerPath(false) + assertPositionDeletesScannerPath(true) + sql """set enable_file_scanner_v2 = true""" + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.pd_doris_spark_interop; + CREATE TABLE demo.${dbName}.pd_doris_spark_interop ( + id INT, + name STRING, + dt STRING + ) USING iceberg + PARTITIONED BY (dt) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.pd_doris_spark_interop + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (1, 'spark_a', '2026-07-03'), + (2, 'spark_b', '2026-07-03') AS t(id, name, dt); + """ + + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [2, "spark_b", "2026-07-03"] + ]) + + sql """insert into pd_doris_spark_interop values (3, 'doris_c', '2026-07-04')""" + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [2, "spark_b", "2026-07-03"], + [3, "doris_c", "2026-07-04"] + ]) + + spark_iceberg """ + INSERT INTO demo.${dbName}.pd_doris_spark_interop + SELECT /*+ COALESCE(1) */ id, name, dt FROM VALUES + (4, 'spark_d', '2026-07-04') AS t(id, name, dt) + """ + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [2, "spark_b", "2026-07-03"], + [3, "doris_c", "2026-07-04"], + [4, "spark_d", "2026-07-04"] + ]) + + sql """delete from pd_doris_spark_interop where id = 2""" + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [1, "spark_a", "2026-07-03"], + [3, "doris_c", "2026-07-04"], + [4, "spark_d", "2026-07-04"] + ]) + + spark_iceberg """DELETE FROM demo.${dbName}.pd_doris_spark_interop WHERE id = 1""" + assertSparkDorisTableRows("pd_doris_spark_interop", [ + [3, "doris_c", "2026-07-04"], + [4, "spark_d", "2026-07-04"] + ]) + + assertPositionDeletesSchema("pd_doris_spark_interop", true) + long interopPositionDeleteCount = countRows( + """select count(*) from pd_doris_spark_interop\$position_deletes""") + assertEquals(2L, interopPositionDeleteCount) + assertEquals(interopPositionDeleteCount, countRows( + """select count(*) from pd_doris_spark_interop\$position_deletes + where `partition` is not null and delete_file_path is not null""")) + assertSparkDorisPositionDeletes("pd_doris_spark_interop", commonCompareColumns) +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy index 3e541af65d7818..cb01ba95bb72ac 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_sys_table.groovy @@ -262,7 +262,7 @@ suite("test_iceberg_sys_table", "p0,external,doris,external_docker,external_dock systableName) } - def test_table_systables = { table -> + def test_table_systables = { table, boolean partitioned -> test_systable_entries(table, "entries") test_systable_entries(table, "all_entries") test_systable_files(table, "files") @@ -278,17 +278,21 @@ suite("test_iceberg_sys_table", "p0,external,doris,external_docker,external_dock test_systable_manifests(table, "manifests") test_systable_manifests(table, "all_manifests") test_systable_partitions(table) - // TODO: these table will be supportted in future - // test_systable_position_deletes(table) - test { - sql """select * from ${table}\$position_deletes""" - exception "SysTable position_deletes is not supported yet" + List> positionDeleteDesc = sql """desc ${table}\$position_deletes""" + List positionDeleteColumns = positionDeleteDesc.collect { it[0].toString() } + List expectedPositionDeleteColumns = ["file_path", "pos", "row"] + if (partitioned) { + expectedPositionDeleteColumns.add("partition") } + expectedPositionDeleteColumns.addAll(["spec_id", "delete_file_path"]) + assertEquals(expectedPositionDeleteColumns, positionDeleteColumns) + List> positionDeleteCount = sql """select count(*) from ${table}\$position_deletes""" + assertEquals(1, positionDeleteCount.size()) } - test_table_systables("test_iceberg_systable_unpartitioned") - test_table_systables("test_iceberg_systable_partitioned") + test_table_systables("test_iceberg_systable_unpartitioned", false) + test_table_systables("test_iceberg_systable_partitioned", true) sql """drop table if exists test_iceberg_systable_tbl1;""" sql """create table test_iceberg_systable_tbl1 (id int);""" @@ -319,10 +323,17 @@ suite("test_iceberg_sys_table", "p0,external,doris,external_docker,external_dock """ exception "denied" } + test { + sql """ + select file_path, pos from ${catalog_name}.${db_name}.test_iceberg_systable_tbl1\$position_deletes + """ + exception "denied" + } } sql """grant select_priv on ${catalog_name}.${db_name}.test_iceberg_systable_tbl1 to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { sql """select committed_at, snapshot_id, parent_id, operation from ${catalog_name}.${db_name}.test_iceberg_systable_tbl1\$snapshots""" + sql """select file_path, pos from ${catalog_name}.${db_name}.test_iceberg_systable_tbl1\$position_deletes""" } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy new file mode 100644 index 00000000000000..4d986201fbeea4 --- /dev/null +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_clickhouse_jdbc_v2", "p0,external") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String clickhousePort = context.config.otherConfigs.get("clickhouse_22_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + // Keep large drivers in the regression bucket so isolated workers do not require public egress. + String driverUrl = "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.9.8-all.jar" + + sql """ drop catalog if exists clickhouse_v2_schema """ + sql """ create catalog clickhouse_v2_schema properties( + "type"="jdbc", + "user"="default", + "password"="123456", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhousePort}/doris_test?jdbc_schema_term=schema", + "driver_url" = "${driverUrl}", + "driver_class" = "com.clickhouse.jdbc.Driver" + );""" + + order_qt_clickhouse_v2_schema """ select count(*) from clickhouse_v2_schema.doris_test.type """ + order_qt_clickhouse_v2_distributed """ select count(*) from clickhouse_v2_schema.doris_test.distributed_type """ + order_qt_clickhouse_v2_materialized_view """ select count(*) from clickhouse_v2_schema.doris_test.materialized_view_type """ + sql """ drop catalog clickhouse_v2_schema """ + } +} diff --git a/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy new file mode 100644 index 00000000000000..6fa29c321c8e72 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_file_scanner_v2_review_fixes", "p0,external") { + def backends = sql "show backends" + assertTrue(backends.size() > 0) + def backendId = backends[0][0] + def dataPath = context.config.dataPath + "/external_table_p0/tvf" + // A developer may point this at fixtures already visible to every BE (for example, a shared + // checkout mounted at /tmp). CI leaves it unset and exercises the normal per-BE distribution. + def predeployedFixturePath = context.config.otherConfigs.get("fileScannerV2FixturePath") + def remotePath = predeployedFixturePath ?: "/tmp/file_scanner_v2_review_fixes" + def fixtureNames = [ + "file_scanner_v2_empty.csv", + "file_scanner_v2_struct_0_bigint.parquet", + "file_scanner_v2_struct_1_int.parquet", + "file_scanner_v2_unsupported_time.parquet" + ] + + if (predeployedFixturePath == null) { + mkdirRemotePathOnAllBE("root", remotePath) + for (def backend : backends) { + for (def fixtureName : fixtureNames) { + scpFiles("root", backend[1], "${dataPath}/${fixtureName}", remotePath, false) + } + } + } + + sql "set enable_file_scanner_v2 = true" + + // A zero-byte CSV is a valid zero-row split when the schema is supplied explicitly. The EOF + // raised during reader preparation must skip this split instead of failing the query. + order_qt_empty_csv """ + SELECT id, name + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_empty.csv", + "backend_id" = "${backendId}", + "format" = "csv", + "csv_schema" = "id:int;name:string") + ORDER BY id + """ + + // The first file defines STRUCT, while the second stores a as INT. Localization is + // split-specific: the BIGINT file compares BIGINT values, while the old INT file safely rewrites + // the exactly representable BIGINT literal 10 to INT and compares in the physical file type. + // If a literal cannot round-trip through INT, the mapper instead casts the INT data to BIGINT. + // Repeating the multi-file read also verifies that neither predicate rewrites nor page-cache + // range state leak from one file schema into the next file or the warm scan. + def evolvedStructQuery = """ + SELECT id, col.a, col.b + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_struct_*.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + WHERE col.a = CAST(10 AS BIGINT) + ORDER BY id + """ + order_qt_struct_leaf_promotion_cold evolvedStructQuery + order_qt_struct_leaf_promotion_warm evolvedStructQuery + + // COUNT(col) sees a non-trivial mapping for the file whose STRUCT leaf is INT while the merged + // table type is BIGINT. It must fall back to the normal scan so schema-evolution casts and + // nullability checks run instead of counting footer values directly. + order_qt_count_evolved_struct """ + SELECT COUNT(col.a) + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_struct_*.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + """ + + // TIME_MILLIS is intentionally unsupported by the record reader. It must not prevent schema + // construction when only the supported id column is projected. + order_qt_unprojected_unsupported_time """ + SELECT id + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_unsupported_time.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + ORDER BY id + """ + + // A requested unsupported leaf must fail before metadata pruning. The fixture stores positive + // TIME_MILLIS values, so `unsupported_time < 0` can be eliminated from INT32 row-group + // statistics. Without the early validation, this query incorrectly returns an empty result + // instead of preserving the unsupported logical-type contract. + test { + sql """ + SELECT unsupported_time + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_unsupported_time.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + WHERE unsupported_time < 0 + """ + exception "Parquet TIME with isAdjustedToUTC=true is not supported" + } + +}