diff --git a/src/duckdb/extension/json/json_common.cpp b/src/duckdb/extension/json/json_common.cpp index 87a65ae4e..98d7f3237 100644 --- a/src/duckdb/extension/json/json_common.cpp +++ b/src/duckdb/extension/json/json_common.cpp @@ -134,14 +134,14 @@ static inline idx_t ReadInteger(const char *ptr, const char *const end, idx_t &i static inline JSONKeyReadResult ReadKey(const char *ptr, const char *const end) { D_ASSERT(ptr != end); if (*ptr == '*') { // Wildcard - if (*(ptr + 1) == '*') { + if (ptr + 1 != end && *(ptr + 1) == '*') { return JSONKeyReadResult::RecWildCard(); } return JSONKeyReadResult::WildCard(); } bool recursive = false; if (*ptr == '.') { - char next = *(ptr + 1); + const char next = ptr + 1 == end ? '\0' : *(ptr + 1); if (next == '*') { return JSONKeyReadResult::RecWildCard(); } @@ -151,6 +151,10 @@ static inline JSONKeyReadResult ReadKey(const char *ptr, const char *const end) ptr++; recursive = true; } + if (ptr == end) { + // recursive '.' with no key following it + return JSONKeyReadResult::Empty(); + } bool escaped = false; if (*ptr == '"') { ptr++; // Skip past opening '"' diff --git a/src/duckdb/extension/json/json_functions/json_create.cpp b/src/duckdb/extension/json/json_functions/json_create.cpp index 2f1ae608a..1f1ed9856 100644 --- a/src/duckdb/extension/json/json_functions/json_create.cpp +++ b/src/duckdb/extension/json/json_functions/json_create.cpp @@ -85,6 +85,12 @@ struct JSONCopyFormatOptions { optional_ptr timestamptz_ns_format_expression; }; +static JSONCopyFormatOptions ClientFormatOptions(ClientContext &context) { + JSONCopyFormatOptions result; + result.context = context; + return result; +} + struct JSONCopyToJSONFunctionData : public FunctionData { public: JSONCopyToJSONFunctionData(StructNames const_struct_names_p, bool has_date_format_p, string date_format_string_p, @@ -792,7 +798,18 @@ static void CreateValuesTimestampNS(yyjson_mut_doc *doc, yyjson_mut_val *vals[], static void CreateValuesTimestampTZ(yyjson_mut_doc *doc, yyjson_mut_val *vals[], Vector &value_v, idx_t count, const JSONCopyFormatOptions &options) { - CreateValuesFormatted(doc, vals, value_v, count, bool(options.timestamp_format), [&](Vector &string_vector) { + if (!options.timestamp_format) { + // no COPY format was given - use the full cast set so that the session time zone is applied + Vector string_vector(LogicalType::VARCHAR, count); + if (options.context) { + VectorOperations::Cast(*options.context.get_mutable(), value_v, string_vector, count, true); + } else { + VectorOperations::DefaultCast(value_v, string_vector, count); + } + TemplatedCreateValues(doc, vals, string_vector, count); + return; + } + CreateValuesFormatted(doc, vals, value_v, count, true, [&](Vector &string_vector) { auto format_expression = value_v.GetType().id() == LogicalTypeId::TIMESTAMP_TZ_NS ? options.timestamptz_ns_format_expression : options.timestamptz_format_expression; @@ -973,6 +990,7 @@ static void ObjectFunction(DataChunk &args, ExpressionState &state, Vector &resu const auto &info = func_expr.BindInfo()->Cast(); auto &lstate = JSONFunctionLocalState::ResetAndGet(state); auto alc = lstate.json_allocator->GetYYAlc(); + auto options = ClientFormatOptions(state.GetContext()); // Initialize values const idx_t count = args.size(); @@ -987,7 +1005,7 @@ static void ObjectFunction(DataChunk &args, ExpressionState &state, Vector &resu for (idx_t pair_idx = 0; pair_idx < args.data.size() / 2; pair_idx++) { Vector &key_v = args.data[pair_idx * 2]; Vector &value_v = args.data[pair_idx * 2 + 1]; - CreateKeyValuePairs(info.const_struct_names, doc, objs, vals, key_v, value_v, count); + CreateKeyValuePairs(info.const_struct_names, doc, objs, vals, key_v, value_v, count, options); } // Write JSON values to string auto objects = FlatVector::GetDataMutable(result); @@ -1002,6 +1020,7 @@ static void ArrayFunction(DataChunk &args, ExpressionState &state, Vector &resul const auto &info = func_expr.BindInfo()->Cast(); auto &lstate = JSONFunctionLocalState::ResetAndGet(state); auto alc = lstate.json_allocator->GetYYAlc(); + auto options = ClientFormatOptions(state.GetContext()); // Initialize arrays const idx_t count = args.size(); @@ -1014,7 +1033,7 @@ static void ArrayFunction(DataChunk &args, ExpressionState &state, Vector &resul auto vals = JSONCommon::AllocateArray(doc, count); // Loop through args for (auto &v : args.data) { - CreateValues(info.const_struct_names, doc, vals, v, count); + CreateValues(info.const_struct_names, doc, vals, v, count, options); for (idx_t i = 0; i < count; i++) { yyjson_mut_arr_append(arrs[i], vals[i]); } @@ -1060,8 +1079,9 @@ static void ToJSONFunction(DataChunk &args, ExpressionState &state, Vector &resu const auto &info = func_expr.BindInfo()->Cast(); auto &lstate = JSONFunctionLocalState::ResetAndGet(state); auto alc = lstate.json_allocator->GetYYAlc(); + auto options = ClientFormatOptions(state.GetContext()); - ToJSONFunctionInternal(info.const_struct_names, args.data[0], args.size(), result, alc); + ToJSONFunctionInternal(info.const_struct_names, args.data[0], args.size(), result, alc, options); } static void JSONCopyToJSONFunction(DataChunk &args, ExpressionState &state, Vector &result) { @@ -1184,16 +1204,17 @@ ScalarFunctionSet JSONFunctions::GetRowToJSONFunction() { struct NestedToJSONCastData : public BoundCastData { public: - NestedToJSONCastData() { + explicit NestedToJSONCastData(optional_ptr client) : client(client) { } unique_ptr Copy() const override { - auto result = make_uniq(); + auto result = make_uniq(client); result->const_struct_names = const_struct_names.Copy(); return std::move(result); } public: + optional_ptr client; StructNames const_struct_names; }; @@ -1201,14 +1222,17 @@ static bool AnyToJSONCast(Vector &source, Vector &result, idx_t count, CastParam auto &lstate = parameters.local_state->Cast(); lstate.json_allocator->Reset(); auto alc = lstate.json_allocator->GetYYAlc(); - const auto &names = parameters.cast_data->Cast().const_struct_names; + auto &cast_data = parameters.cast_data->Cast(); + const auto &names = cast_data.const_struct_names; - ToJSONFunctionInternal(names, source, count, result, alc); + JSONCopyFormatOptions options; + options.context = cast_data.client; + ToJSONFunctionInternal(names, source, count, result, alc, options); return true; } static BoundCastInfo AnyToJSONCastBind(BindCastInput &input, const LogicalType &source, const LogicalType &target) { - auto cast_data = make_uniq(); + auto cast_data = make_uniq(input.context); GetJSONType(cast_data->const_struct_names, source); return BoundCastInfo(AnyToJSONCast, std::move(cast_data), JSONFunctionLocalState::InitCastLocalState); } diff --git a/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp b/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp index d6fa8986e..4b6b69fe7 100644 --- a/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +++ b/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp @@ -18,6 +18,10 @@ class ParquetDecimalUtils { template static PHYSICAL_TYPE ReadDecimalValue(const_data_ptr_t pointer, idx_t size, const ParquetColumnSchema &) { PHYSICAL_TYPE res = 0; + if (size == 0) { + // empty byte array - value is zero, and there is no sign byte to read + return res; + } auto res_ptr = (uint8_t *)&res; bool positive = (*pointer & 0x80) == 0; diff --git a/src/duckdb/extension/parquet/parquet_reader.cpp b/src/duckdb/extension/parquet/parquet_reader.cpp index e4afefc8d..ea47aa207 100644 --- a/src/duckdb/extension/parquet/parquet_reader.cpp +++ b/src/duckdb/extension/parquet/parquet_reader.cpp @@ -48,6 +48,7 @@ #include "duckdb/common/types/data_chunk.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/common/vector_size.hpp" +#include "duckdb/function/scalar/struct_utils.hpp" #include "duckdb/logging/log_type.hpp" #include "duckdb/logging/logger.hpp" #include "duckdb/main/client_context.hpp" @@ -1483,8 +1484,8 @@ static FilterPropagateResult CheckParquetFloatFilter(ClientContext &context, Col return FilterPropagateResult::NO_PRUNING_POSSIBLE; } -static bool TryGetListBloomFilterLeaf(ColumnReader &column_reader, const Expression &expr, - optional_ptr &leaf_reader) { +static bool TryGetNestedBloomFilterLeaf(ColumnReader &column_reader, const Expression &expr, + optional_ptr &leaf_reader) { if (expr.GetExpressionClass() == ExpressionClass::BOUND_REF) { if (expr.Cast().Index() != 0) { return false; @@ -1498,14 +1499,31 @@ static bool TryGetListBloomFilterLeaf(ColumnReader &column_reader, const Express auto &function = expr.Cast(); if (function.GetChildren().empty() || - !TryGetListBloomFilterLeaf(column_reader, *function.GetChildren()[0], leaf_reader)) { + !TryGetNestedBloomFilterLeaf(column_reader, *function.GetChildren()[0], leaf_reader)) { return false; } + + // Handle LIST type. if (leaf_reader->Type().id() == LogicalTypeId::LIST && (function.Function().GetName() == "list_extract" || function.Function().GetName() == "array_extract")) { leaf_reader = &leaf_reader->Cast().GetChildReader(); return true; } + + // Handle STRUCT type. + if (leaf_reader->Type().id() == LogicalTypeId::STRUCT) { + idx_t child_idx; + if (!TryGetStructExtractChildIndex(function, child_idx)) { + return false; + } + auto &struct_reader = leaf_reader->Cast(); + if (child_idx >= struct_reader.child_readers.size() || !struct_reader.child_readers[child_idx]) { + return false; + } + leaf_reader = struct_reader.child_readers[child_idx].get(); + return true; + } + return false; } @@ -1553,7 +1571,7 @@ static bool TryGetBloomFilterLeaf(ColumnReader &column_reader, const TableFilter return false; } - if (!TryGetListBloomFilterLeaf(column_reader, *column_expr, leaf_reader) || + if (!TryGetNestedBloomFilterLeaf(column_reader, *column_expr, leaf_reader) || leaf_reader->Type() != constant->GetValue().type()) { return false; } diff --git a/src/duckdb/extension/parquet/parquet_writer.cpp b/src/duckdb/extension/parquet/parquet_writer.cpp index 0b99281f4..c254ac93f 100644 --- a/src/duckdb/extension/parquet/parquet_writer.cpp +++ b/src/duckdb/extension/parquet/parquet_writer.cpp @@ -913,16 +913,43 @@ struct DecimalStatsUnifier : public NumericStatsUnifier { if (stats.empty()) { return string(); } - auto stats_data = const_data_ptr_cast(stats.data()); - if (sizeof(T) == sizeof(hugeint_t)) { - auto _schema = ParquetColumnSchema(); - auto numeric_val = ParquetDecimalUtils::ReadDecimalValue(stats_data, stats.size(), _schema); + auto schema = ParquetColumnSchema(); // schema unused for FLBA/hugeint_t + auto numeric_val = ParquetDecimalUtils::ReadDecimalValue(stats_data, stats.size(), schema); return Value::DECIMAL(numeric_val, width, scale).ToString(); } else { - auto numeric_val = Load(stats_data); - return Value::DECIMAL(numeric_val, width, scale).ToString(); + return Value::DECIMAL(Load(stats_data), width, scale).ToString(); + } + } + + void UnifyMinMax(const string &new_min, const string &new_max) override { + if (sizeof(T) != sizeof(hugeint_t)) { + // INT32/INT64-backed decimals are little-endian; the base compare is correct. + BaseNumericStatsUnifier::UnifyMinMax(new_min, new_max); + } else { + // FLBA decimal stats are big-endian two's complement (most significant byte + // first), so a native little-endian Load would compare the wrong value; decode + // the bytes into a hugeint_t before comparing. + auto decode = [](const string &stats) { + auto schema = ParquetColumnSchema(); // schema unused for FLBA/hugeint_t + return ParquetDecimalUtils::ReadDecimalValue(const_data_ptr_cast(stats.data()), stats.size(), + schema); + }; + + if (!this->min_is_set) { + this->global_min = new_min; + this->min_is_set = true; + } else if (LessThan::Operation(decode(new_min), decode(this->global_min))) { + this->global_min = new_min; + } + + if (!this->max_is_set) { + this->global_max = new_max; + this->max_is_set = true; + } else if (GreaterThan::Operation(decode(new_max), decode(this->global_max))) { + this->global_max = new_max; + } } } }; diff --git a/src/duckdb/extension/parquet/reader/decimal_column_reader.cpp b/src/duckdb/extension/parquet/reader/decimal_column_reader.cpp index b645ef626..ef58e9c5e 100644 --- a/src/duckdb/extension/parquet/reader/decimal_column_reader.cpp +++ b/src/duckdb/extension/parquet/reader/decimal_column_reader.cpp @@ -37,6 +37,10 @@ template <> double ParquetDecimalUtils::ReadDecimalValue(const_data_ptr_t pointer, idx_t size, const ParquetColumnSchema &schema_ele) { double res = 0; + if (size == 0) { + // empty byte array - value is zero, and there is no sign byte to read + return res; + } bool positive = (*pointer & 0x80) == 0; for (idx_t i = 0; i < size; i += 8) { auto byte_size = MinValue(sizeof(uint64_t), size - i); diff --git a/src/duckdb/extension/parquet/writer/variant/convert_variant.cpp b/src/duckdb/extension/parquet/writer/variant/convert_variant.cpp index a759e44b2..6a9114084 100644 --- a/src/duckdb/extension/parquet/writer/variant/convert_variant.cpp +++ b/src/duckdb/extension/parquet/writer/variant/convert_variant.cpp @@ -71,8 +71,7 @@ static uint8_t EncodeMetadataHeader(idx_t byte_length) { uint8_t header_byte = 0; //! Set 'version' to 1 header_byte |= static_cast(1); - //! Set 'sorted_strings' to 1 - header_byte |= static_cast(1) << 4; + //! NOTE: keep 'sorted_strings' at 0, we don't always sort the key strings //! Set 'offset_size_minus_one' to byte_length-1 header_byte |= (static_cast(byte_length) - 1) << 6; diff --git a/src/duckdb/src/catalog/catalog.cpp b/src/duckdb/src/catalog/catalog.cpp index dcc9ed18a..59c3966e3 100644 --- a/src/duckdb/src/catalog/catalog.cpp +++ b/src/duckdb/src/catalog/catalog.cpp @@ -556,13 +556,19 @@ vector Catalog::SimilarEntriesInSchemas(ClientContext &cont } vector GetCatalogEntries(CatalogEntryRetriever &retriever, const Identifier &catalog, - const Identifier &schema) { + const Identifier &schema, CatalogType lookup_type = CatalogType::INVALID) { auto &context = retriever.GetContext(); vector entries; auto &search_path = retriever.GetSearchPath(); if (IsInvalidCatalog(catalog) && IsInvalidSchema(schema)) { - // no catalog or schema provided - scan the entire search path - entries = search_path.Get(); + // no catalog or schema provided - scan the entire search path. + if (lookup_type == CatalogType::INVALID || lookup_type == CatalogType::TABLE_ENTRY) { + entries = search_path.Get(); + } else { + // for non-table lookups, resolve implicit catalogs that requested precedence to their default schema in + // place + entries = search_path.GetWithPrecedenceSchemas(context); + } } else if (IsInvalidCatalog(catalog)) { auto catalogs = search_path.GetCatalogsForSchema(schema); for (auto &catalog_name : catalogs) { @@ -971,7 +977,7 @@ CatalogEntryLookup Catalog::TryLookupEntryAcrossCatalogs(CatalogEntryRetriever & lookups.emplace_back(*catalog_entry, lookup_info); return TryLookupEntry(retriever, lookups, lookup_info, if_not_found, false); } - auto entries = GetCatalogEntries(retriever, catalog, schema); + auto entries = GetCatalogEntries(retriever, catalog, schema, lookup_info.GetCatalogType()); vector final_lookups; lookups.reserve(entries.size()); for (auto &entry : entries) { @@ -997,9 +1003,9 @@ CatalogEntryLookup Catalog::TryLookupEntryAcrossCatalogs(CatalogEntryRetriever & lookups.emplace_back(std::move(lookup)); } - bool allow_default_table_lookup = catalog.empty() && schema.empty(); + bool allow_default_lookup = catalog.empty() && schema.empty(); - return TryLookupEntry(retriever, lookups, lookup_info, if_not_found, allow_default_table_lookup); + return TryLookupEntry(retriever, lookups, lookup_info, if_not_found, allow_default_lookup); } CatalogEntryLookup Catalog::LookupEntry(CatalogEntryRetriever &retriever, const EntryLookupInfo &lookup_info, @@ -1041,7 +1047,7 @@ static void ThrowDefaultTableAmbiguityException(CatalogEntryLookup &base_lookup, CatalogEntryLookup Catalog::TryLookupEntry(CatalogEntryRetriever &retriever, const vector &lookups, const EntryLookupInfo &lookup_info, OnEntryNotFound if_not_found, - bool allow_default_table_lookup) { + bool allow_default_lookup) { auto &context = retriever.GetContext(); reference_set_t schemas; bool all_errors = true; @@ -1065,7 +1071,7 @@ CatalogEntryLookup Catalog::TryLookupEntry(CatalogEntryRetriever &retriever, con // Special case for tables: we do a second lookup searching for catalogs with default tables that also match this // lookup - if (lookup_info.GetCatalogType() == CatalogType::TABLE_ENTRY && allow_default_table_lookup) { + if (lookup_info.GetCatalogType() == CatalogType::TABLE_ENTRY && allow_default_lookup) { if (!result.Found()) { result = TryLookupDefaultTable(retriever, lookup_info, false); if (result.error.HasError()) { @@ -1081,6 +1087,15 @@ CatalogEntryLookup Catalog::TryLookupEntry(CatalogEntryRetriever &retriever, con } } + // Special case for non-table entries (functions, macros, types): Fall back to the default schema of any + // catalog flagged as an implicit search catalog. Ignored if the schema already had precedence. + if (lookup_info.GetCatalogType() != CatalogType::TABLE_ENTRY && allow_default_lookup && !result.Found()) { + result = TryLookupDefaultSchema(retriever, lookup_info); + if (result.error.HasError()) { + error_data = std::move(result.error); + } + } + if (result.Found()) { return result; } @@ -1156,6 +1171,28 @@ CatalogEntryLookup Catalog::TryLookupDefaultTable(CatalogEntryRetriever &retriev return {nullptr, nullptr, ErrorData()}; } +CatalogEntryLookup Catalog::TryLookupDefaultSchema(CatalogEntryRetriever &retriever, + const EntryLookupInfo &lookup_info) { + // look for the entry in the default schema of every catalog that was flagged as an implicit search catalog + auto &search_path = retriever.GetSearchPath(); + for (auto &implicit_catalog : search_path.GetImplicitSearchCatalogs()) { + auto catalog_entry = GetCatalogEntry(retriever, implicit_catalog.GetCatalog()); + if (!catalog_entry) { + continue; + } + auto transaction = catalog_entry->GetCatalogTransaction(retriever.GetContext()); + EntryLookupInfo default_schema_lookup(lookup_info, QualifiedName(catalog_entry->GetName(), + Identifier(catalog_entry->GetDefaultSchema()), + lookup_info.GetEntryIdentifier())); + auto result = catalog_entry->TryLookupEntryInternal(transaction, default_schema_lookup); + if (result.Found() || result.error.HasError()) { + return result; + } + } + + return {nullptr, nullptr, ErrorData()}; +} + optional_ptr Catalog::GetEntry(CatalogEntryRetriever &retriever, const EntryLookupInfo &lookup_info, OnEntryNotFound if_not_found) { auto result = TryLookupEntryAcrossCatalogs(retriever, lookup_info, if_not_found); diff --git a/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp b/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp index 4483474c0..e7785e7c2 100644 --- a/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp +++ b/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp @@ -189,6 +189,7 @@ optional_ptr DuckSchemaEntry::AddEntryInternal(CatalogTransaction optional_ptr DuckSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { auto table = make_uniq(catalog, *this, info); + auto &dependencies = info.Base().dependencies; // add a foreign key constraint in main key table if there is a foreign key constraint vector> fk_arrays; @@ -200,13 +201,13 @@ optional_ptr DuckSchemaEntry::CreateTable(CatalogTransaction trans // make a dependency between this table and referenced table auto &set = GetCatalogSet(CatalogType::TABLE_ENTRY); - info.dependencies.AddDependency(*set.GetEntry(transaction, fk_info.GetQualifiedName().Name())); + dependencies.AddDependency(*set.GetEntry(transaction, fk_info.GetQualifiedName().Name())); } - for (auto &dep : info.dependencies.Set()) { + for (auto &dep : dependencies.Set()) { table->dependencies.AddDependency(dep); } - auto entry = AddEntryInternal(transaction, std::move(table), info.Base().on_conflict, info.dependencies); + auto entry = AddEntryInternal(transaction, std::move(table), info.Base().on_conflict, dependencies); if (!entry) { return nullptr; } diff --git a/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp b/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp index 202f588ff..d6b0fb105 100644 --- a/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp @@ -133,6 +133,10 @@ static void CheckTypeIsSupported(const LogicalType &logical_type, AttachedDataba }); } +static void SetAlterDependencies(BoundCreateTableInfo &info, AlterInfo &alter_info) { + alter_info.new_dependencies = make_uniq(info.Base().dependencies); +} + virtual_column_map_t DuckTableEntry::GetVirtualColumns() const { virtual_column_map_t virtual_columns; virtual_columns.insert(make_pair(COLUMN_IDENTIFIER_ROW_ID, TableColumn("rowid", LogicalType::ROW_TYPE))); @@ -455,6 +459,7 @@ unique_ptr DuckTableEntry::RenameColumn(ClientContext &context, Re } auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); + SetAlterDependencies(*bound_create_info, info); return make_uniq(catalog, schema, *bound_create_info, storage, triggers); } @@ -501,6 +506,7 @@ unique_ptr DuckTableEntry::AddColumn(ClientContext &context, AddCo binder->BindDefaultValue(info.new_column, bound_defaults, catalog_name.GetIdentifierName(), schema_name.GetIdentifierName()); } + SetAlterDependencies(*bound_create_info, info); auto new_storage = make_shared_ptr(context, *storage, info.new_column, *bound_defaults.back()); return make_uniq(catalog, schema, *bound_create_info, new_storage, triggers); } @@ -815,7 +821,7 @@ unique_ptr DuckTableEntry::RemoveColumn(ClientContext &context, Re dropped_column_is_generated); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); - info.new_dependencies = make_uniq(std::move(bound_create_info->dependencies)); + SetAlterDependencies(*bound_create_info, info); if (columns.GetColumn(LogicalIndex(removed_index)).Generated()) { return make_uniq(catalog, schema, *bound_create_info, storage, triggers); } @@ -1039,7 +1045,7 @@ unique_ptr DuckTableEntry::SetDefault(ClientContext &context, SetD auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); - info.new_dependencies = make_uniq(std::move(bound_create_info->dependencies)); + SetAlterDependencies(*bound_create_info, info); return make_uniq(catalog, schema, *bound_create_info, storage, triggers); } @@ -1068,6 +1074,7 @@ unique_ptr DuckTableEntry::SetNotNull(ClientContext &context, SetN auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); + SetAlterDependencies(*bound_create_info, info); // Early return if (has_not_null) { @@ -1101,6 +1108,7 @@ unique_ptr DuckTableEntry::DropNotNull(ClientContext &context, Dro auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); + SetAlterDependencies(*bound_create_info, info); return make_uniq(catalog, schema, *bound_create_info, storage, triggers); } @@ -1192,6 +1200,7 @@ unique_ptr DuckTableEntry::ChangeColumnType(ClientContext &context } auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); + SetAlterDependencies(*bound_create_info, info); vector storage_oids; for (idx_t i = 0; i < bound_columns.size(); i++) { @@ -1272,6 +1281,7 @@ unique_ptr DuckTableEntry::DropForeignKeyConstraint(ClientContext auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); + SetAlterDependencies(*bound_create_info, info); return make_uniq(catalog, schema, *bound_create_info, storage, triggers); } @@ -1346,6 +1356,7 @@ unique_ptr DuckTableEntry::AddConstraint(ClientContext &context, A const auto bound_constraint = binder->BindConstraint(*info.constraint, table_info.GetTableName(), table_info.columns); const auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, info.bind_mode); + SetAlterDependencies(*bound_create_info, info); auto new_storage = make_shared_ptr(context, *storage, *bound_constraint); auto new_entry = make_uniq(catalog, schema, *bound_create_info, new_storage, triggers); diff --git a/src/duckdb/src/catalog/catalog_search_path.cpp b/src/duckdb/src/catalog/catalog_search_path.cpp index 0fefce0b6..cce8dab81 100644 --- a/src/duckdb/src/catalog/catalog_search_path.cpp +++ b/src/duckdb/src/catalog/catalog_search_path.cpp @@ -4,6 +4,7 @@ #include "duckdb/catalog/catalog.hpp" #include "duckdb/common/constants.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/common/sql_identifier.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/main/database_manager.hpp" @@ -13,8 +14,9 @@ namespace duckdb { -CatalogSearchEntry::CatalogSearchEntry(Identifier catalog_p, Identifier schema_p) - : catalog(std::move(catalog_p)), schema(std::move(schema_p)) { +CatalogSearchEntry::CatalogSearchEntry(Identifier catalog_p, Identifier schema_p, bool default_schema_precedence_p) + : catalog(std::move(catalog_p)), schema(std::move(schema_p)), + default_schema_precedence(default_schema_precedence_p) { } string CatalogSearchEntry::ToString() const { @@ -29,7 +31,7 @@ string CatalogSearchEntry::WriteOptionallyQuoted(const Identifier &input) { auto &name = input.GetIdentifierName(); for (idx_t i = 0; i < name.size(); i++) { if (name[i] == '.' || name[i] == ',' || name[i] == '"') { - return "\"" + StringUtil::Replace(name, "\"", "\"\"") + "\""; + return SQLQuotedIdentifier::ToString(name); } } return name; @@ -283,6 +285,37 @@ vector CatalogSearchPath::GetSchemasForCatalog(const Identifier &cat return schemas; } +vector CatalogSearchPath::GetImplicitSearchCatalogs() const { + // Get the implicit entries that were not already resolved by the precedence flag. + vector catalogs; + for (auto &path : paths) { + if (path.GetSchema().empty() && !path.GetCatalog().empty() && !path.DefaultSchemaPrecedence()) { + catalogs.push_back(path); + } + } + return catalogs; +} + +vector CatalogSearchPath::GetWithPrecedenceSchemas(ClientContext &context) const { + vector res; + for (auto &path : paths) { + // Resolve implicit entries with default_schema_precedence to the catalog's default schema. + if (path.GetSchema().empty()) { + if (path.GetCatalog().empty() || !path.DefaultSchemaPrecedence()) { + continue; + } + auto catalog_entry = Catalog::GetCatalogEntry(context, path.GetCatalog()); + if (!catalog_entry) { + continue; + } + res.emplace_back(path.GetCatalog(), Identifier(catalog_entry->GetDefaultSchema())); + } else { + res.emplace_back(path); + } + } + return res; +} + const CatalogSearchEntry &CatalogSearchPath::GetDefault() const { D_ASSERT(paths.size() >= 2); D_ASSERT(!paths[1].GetSchema().empty()); diff --git a/src/duckdb/src/catalog/dependency_manager.cpp b/src/duckdb/src/catalog/dependency_manager.cpp index ab18e6c9c..5760e99b0 100644 --- a/src/duckdb/src/catalog/dependency_manager.cpp +++ b/src/duckdb/src/catalog/dependency_manager.cpp @@ -18,6 +18,7 @@ #include "duckdb/parser/qualified_name.hpp" #include "duckdb/common/printer.hpp" +#include "duckdb/common/sql_identifier.hpp" namespace duckdb { @@ -245,9 +246,20 @@ void DependencyManager::CreateDependent(CatalogTransaction transaction, const De set.CreateEntry(transaction, entry_name, std::move(dep)); } +static string CatalogEntryInfoToString(const CatalogEntryInfo &entry) { + auto schema = StringUtil::Join(entry.schema_path, entry.schema_path.size(), ".", [](const Identifier &id) { + return SQLIdentifier::ToString(id.GetIdentifierName()); + }); + return schema + "." + SQLIdentifier::ToString(entry.name.GetIdentifierName()) + + StringUtil::Format("(%s)", CatalogTypeToString(entry.type)); +} + void DependencyManager::CreateDependency(CatalogTransaction transaction, DependencyInfo &info) { auto subject_entry = LookupEntry(transaction, info.subject.entry); info.subject.oid = subject_entry ? subject_entry->oid : optional_idx(); + if (!subject_entry) { + throw InternalException("Couldn't locate entry: '%s'", CatalogEntryInfoToString(info.subject.entry)); + } DependencyCatalogSet subjects(Subjects(), info.dependent.entry); DependencyCatalogSet dependents(Dependents(), info.subject.entry); diff --git a/src/duckdb/src/common/adbc/adbc.cpp b/src/duckdb/src/common/adbc/adbc.cpp index ac0a8ff26..517ebfa7c 100644 --- a/src/duckdb/src/common/adbc/adbc.cpp +++ b/src/duckdb/src/common/adbc/adbc.cpp @@ -30,7 +30,7 @@ struct DuckDBErrorDetails { #include -#include "duckdb/main/prepared_statement_data.hpp" +#include "duckdb/main/materialized_query_result.hpp" #include "duckdb/parser/keyword_helper.hpp" @@ -93,8 +93,8 @@ AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *err adbc_driver->ConnectionGetOptionBytes = duckdb_adbc::ConnectionGetOptionBytes; adbc_driver->ConnectionGetOptionDouble = duckdb_adbc::ConnectionGetOptionDouble; adbc_driver->ConnectionGetOptionInt = duckdb_adbc::ConnectionGetOptionInt; - adbc_driver->ConnectionGetStatistics = nullptr; - adbc_driver->ConnectionGetStatisticNames = nullptr; + adbc_driver->ConnectionGetStatistics = duckdb_adbc::ConnectionGetStatistics; + adbc_driver->ConnectionGetStatisticNames = duckdb_adbc::ConnectionGetStatisticNames; adbc_driver->ConnectionSetOptionBytes = duckdb_adbc::ConnectionSetOptionBytes; adbc_driver->ConnectionSetOptionInt = duckdb_adbc::ConnectionSetOptionInt; adbc_driver->ConnectionSetOptionDouble = duckdb_adbc::ConnectionSetOptionDouble; @@ -314,12 +314,16 @@ AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, return ADBC_STATUS_NOT_IMPLEMENTED; } if (strcmp(key, "uri") == 0) { - if (strncmp(value, "file:", 5) != 0) { + std::string file_path; + if (strncmp(value, "file:", 5) == 0) { + file_path = std::string(value + 5); + } else if (strncmp(value, "duckdb:", 7) == 0) { + file_path = std::string(value + 7); + } else { wrapper->uri_path = value; wrapper->uri_set = true; return ADBC_STATUS_OK; } - std::string file_path(value + 5); auto suffix_pos = file_path.find_first_of("?#"); if (suffix_pos != std::string::npos) { file_path.erase(suffix_pos); @@ -334,7 +338,7 @@ AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, file_path = (authority_lc.empty() || authority_lc == "localhost") ? std::string() : authority; } else { if (!authority_lc.empty() && authority_lc != "localhost") { - SetError(error, "file: URI with a non-empty authority is not supported"); + SetError(error, "URI with a non-empty authority is not supported"); return ADBC_STATUS_INVALID_ARGUMENT; } file_path = file_path.substr(path_start); @@ -1625,11 +1629,11 @@ AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, stru // would be worth the extra management auto prepared_wrapper = reinterpret_cast(wrapper->statement); - if (!prepared_wrapper || !prepared_wrapper->statement || !prepared_wrapper->statement->data) { + if (!prepared_wrapper || !prepared_wrapper->statement) { SetError(error, "Invalid prepared statement wrapper"); return ADBC_STATUS_INVALID_ARGUMENT; } - auto count = prepared_wrapper->statement->data->properties.parameter_count; + auto count = prepared_wrapper->statement->GetParameterCount(); std::vector types(count); std::vector owned_names; owned_names.reserve(count); @@ -1742,8 +1746,8 @@ AdbcStatusCode StatementExecuteQuery(struct AdbcStatement *statement, struct Arr std::memset(&raw_stream_wrapper->result, 0, sizeof(raw_stream_wrapper->result)); DuckDBAdbcStreamWrapperGuard stream_wrapper(raw_stream_wrapper); // Only process the stream if there are parameters to bind - auto prepared_statement_params = reinterpret_cast(wrapper->statement) - ->statement->data->properties.parameter_count; + auto prepared_statement_params = + reinterpret_cast(wrapper->statement)->statement->GetParameterCount(); if (has_stream && prepared_statement_params > 0) { // A stream was bound to the statement, use that to bind parameters ArrowArrayStream stream = wrapper->ingestion_stream; @@ -2626,6 +2630,602 @@ AdbcStatusCode ConnectionGetTableTypes(struct AdbcConnection *connection, struct return QueryInternal(connection, out, q, error); } +//===--------------------------------------------------------------------===// +// Statistics (ADBC 1.1.0) +//===--------------------------------------------------------------------===// +// The `statistic_value` column of GetStatistics must be a dense union per +// the ADBC spec. DuckDB's Arrow export only produces sparse unions, +// so the result batch cannot be produced with QueryInternal; +// instead it is built manually with malloc'd buffers and released through a +// recursive release callback. + +// Recursively frees an ArrowArray tree built by InitStatisticsArrayNode. +static void ReleaseStatisticsArray(struct ArrowArray *array) { + if (!array || !array->release) { + return; + } + if (array->buffers) { + for (int64_t i = 0; i < array->n_buffers; i++) { + if (array->buffers[i]) { + free(const_cast(array->buffers[i])); + } + } + free(array->buffers); + } + if (array->children) { + for (int64_t i = 0; i < array->n_children; i++) { + auto child = array->children[i]; + if (child) { + if (child->release) { + child->release(child); + } + free(child); + } + } + free(array->children); + } + array->release = nullptr; +} + +// Zero-initializes an ArrowArray node, allocates its buffer/child pointer +// arrays and installs ReleaseStatisticsArray. Children are zero-initialized; +// releasing the root frees a partially built tree, so callers only need to +// release the root on failure. +static AdbcStatusCode InitStatisticsArrayNode(struct ArrowArray *array, int64_t n_buffers, int64_t n_children, + struct AdbcError *error) { + memset(array, 0, sizeof(*array)); + array->release = ReleaseStatisticsArray; + if (n_buffers > 0) { + array->buffers = static_cast(malloc(sizeof(void *) * static_cast(n_buffers))); + if (!array->buffers) { + SetError(error, "Failed to allocate buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + memset(array->buffers, 0, sizeof(void *) * static_cast(n_buffers)); + array->n_buffers = n_buffers; + } + if (n_children > 0) { + array->children = + static_cast(malloc(sizeof(struct ArrowArray *) * static_cast(n_children))); + if (!array->children) { + SetError(error, "Failed to allocate children for statistics result"); + return ADBC_STATUS_INTERNAL; + } + memset(array->children, 0, sizeof(struct ArrowArray *) * static_cast(n_children)); + array->n_children = n_children; + for (int64_t i = 0; i < n_children; i++) { + array->children[i] = static_cast(malloc(sizeof(struct ArrowArray))); + if (!array->children[i]) { + SetError(error, "Failed to allocate child array for statistics result"); + return ADBC_STATUS_INTERNAL; + } + memset(array->children[i], 0, sizeof(struct ArrowArray)); + } + } + return ADBC_STATUS_OK; +} + +// Allocates a zeroed buffer owned by the array node (freed by the release +// callback). Returns nullptr on allocation failure. +static void *AllocStatisticsBuffer(struct ArrowArray *array, idx_t buffer_idx, size_t size) { + auto alloc_size = size == 0 ? 1 : size; + auto buffer = malloc(alloc_size); + if (buffer) { + memset(buffer, 0, alloc_size); + array->buffers[buffer_idx] = buffer; + } + return buffer; +} + +// Fills a utf8 array node from `values` (no NULL entries). +static AdbcStatusCode BuildStatisticsVarcharArray(struct ArrowArray *array, + const duckdb::vector &values, + struct AdbcError *error) { + auto count = values.size(); + auto status = InitStatisticsArrayNode(array, 3, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + array->length = static_cast(count); + auto offsets = static_cast(AllocStatisticsBuffer(array, 1, sizeof(int32_t) * (count + 1))); + size_t total_size = 0; + for (auto &value : values) { + total_size += value.size(); + } + auto data = static_cast(AllocStatisticsBuffer(array, 2, total_size)); + if (!offsets || !data) { + SetError(error, "Failed to allocate string buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + int32_t offset = 0; + for (idx_t i = 0; i < count; i++) { + offsets[i] = offset; + memcpy(data + offset, values[i].data(), values[i].size()); + offset += static_cast(values[i].size()); + } + offsets[count] = offset; + return ADBC_STATUS_OK; +} + +// Fills a utf8 array node where every entry is NULL (used for the table-level +// `column_name` column). +static AdbcStatusCode BuildStatisticsAllNullVarcharArray(struct ArrowArray *array, idx_t count, + struct AdbcError *error) { + auto status = InitStatisticsArrayNode(array, 3, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + array->length = static_cast(count); + array->null_count = static_cast(count); + // All-zero validity bitmap (all NULL), all-zero offsets and an empty data buffer. + auto validity = AllocStatisticsBuffer(array, 0, (count + 7) / 8); + auto offsets = AllocStatisticsBuffer(array, 1, sizeof(int32_t) * (count + 1)); + auto data = AllocStatisticsBuffer(array, 2, 0); + if (!validity || !offsets || !data) { + SetError(error, "Failed to allocate string buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + return ADBC_STATUS_OK; +} + +#define CHECK_STATISTICS_NANOARROW(EXPR) \ + do { \ + if ((EXPR) != 0) { \ + SetError(error, "Failed to build Arrow schema for statistics result"); \ + return ADBC_STATUS_INTERNAL; \ + } \ + } while (0) + +// Builds the GetStatistics result schema defined in adbc.h. The caller is +// responsible for releasing `schema` on failure. +static AdbcStatusCode BuildGetStatisticsSchema(struct ArrowSchema &schema, struct AdbcError *error) { + using duckdb_nanoarrow::ArrowSchemaAllocateChildren; + using duckdb_nanoarrow::ArrowSchemaInit; + using duckdb_nanoarrow::ArrowSchemaSetFormat; + using duckdb_nanoarrow::ArrowSchemaSetName; + + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(&schema, duckdb_nanoarrow::NANOARROW_TYPE_STRUCT)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(&schema, 2)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(schema.children[0], duckdb_nanoarrow::NANOARROW_TYPE_STRING)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(schema.children[0], "catalog_name")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(schema.children[1], duckdb_nanoarrow::NANOARROW_TYPE_LIST)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(schema.children[1], "catalog_db_schemas")); + schema.children[1]->flags &= ~ARROW_FLAG_NULLABLE; + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(schema.children[1], 1)); + + auto db_schema_schema = schema.children[1]->children[0]; + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(db_schema_schema, duckdb_nanoarrow::NANOARROW_TYPE_STRUCT)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(db_schema_schema, "item")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(db_schema_schema, 2)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(db_schema_schema->children[0], duckdb_nanoarrow::NANOARROW_TYPE_STRING)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(db_schema_schema->children[0], "db_schema_name")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(db_schema_schema->children[1], duckdb_nanoarrow::NANOARROW_TYPE_LIST)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(db_schema_schema->children[1], "db_schema_statistics")); + db_schema_schema->children[1]->flags &= ~ARROW_FLAG_NULLABLE; + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(db_schema_schema->children[1], 1)); + + auto statistics_schema = db_schema_schema->children[1]->children[0]; + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(statistics_schema, duckdb_nanoarrow::NANOARROW_TYPE_STRUCT)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(statistics_schema, "item")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(statistics_schema, 5)); + CHECK_STATISTICS_NANOARROW( + ArrowSchemaInit(statistics_schema->children[0], duckdb_nanoarrow::NANOARROW_TYPE_STRING)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(statistics_schema->children[0], "table_name")); + statistics_schema->children[0]->flags &= ~ARROW_FLAG_NULLABLE; + CHECK_STATISTICS_NANOARROW( + ArrowSchemaInit(statistics_schema->children[1], duckdb_nanoarrow::NANOARROW_TYPE_STRING)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(statistics_schema->children[1], "column_name")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(statistics_schema->children[2], duckdb_nanoarrow::NANOARROW_TYPE_INT16)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(statistics_schema->children[2], "statistic_key")); + statistics_schema->children[2]->flags &= ~ARROW_FLAG_NULLABLE; + + // statistic_value: dense union (the bundled nanoarrow has no union type + // template, so the format string is set directly). + auto value_schema = statistics_schema->children[3]; + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(value_schema, duckdb_nanoarrow::NANOARROW_TYPE_UNINITIALIZED)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetFormat(value_schema, "+ud:0,1,2,3")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(value_schema, "statistic_value")); + value_schema->flags &= ~ARROW_FLAG_NULLABLE; + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(value_schema, 4)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(value_schema->children[0], duckdb_nanoarrow::NANOARROW_TYPE_INT64)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(value_schema->children[0], "int64")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(value_schema->children[1], duckdb_nanoarrow::NANOARROW_TYPE_UINT64)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(value_schema->children[1], "uint64")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(value_schema->children[2], duckdb_nanoarrow::NANOARROW_TYPE_DOUBLE)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(value_schema->children[2], "float64")); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(value_schema->children[3], duckdb_nanoarrow::NANOARROW_TYPE_BINARY)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(value_schema->children[3], "binary")); + + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(statistics_schema->children[4], duckdb_nanoarrow::NANOARROW_TYPE_BOOL)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(statistics_schema->children[4], "statistic_is_approximate")); + statistics_schema->children[4]->flags &= ~ARROW_FLAG_NULLABLE; + return ADBC_STATUS_OK; +} + +// Builds the GetStatisticNames result schema (statistic_name utf8 not null, +// statistic_key int16 not null). The caller releases `schema` on failure. +static AdbcStatusCode BuildGetStatisticNamesSchema(struct ArrowSchema &schema, struct AdbcError *error) { + using duckdb_nanoarrow::ArrowSchemaAllocateChildren; + using duckdb_nanoarrow::ArrowSchemaInit; + using duckdb_nanoarrow::ArrowSchemaSetName; + + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(&schema, duckdb_nanoarrow::NANOARROW_TYPE_STRUCT)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaAllocateChildren(&schema, 2)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(schema.children[0], duckdb_nanoarrow::NANOARROW_TYPE_STRING)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(schema.children[0], "statistic_name")); + schema.children[0]->flags &= ~ARROW_FLAG_NULLABLE; + CHECK_STATISTICS_NANOARROW(ArrowSchemaInit(schema.children[1], duckdb_nanoarrow::NANOARROW_TYPE_INT16)); + CHECK_STATISTICS_NANOARROW(ArrowSchemaSetName(schema.children[1], "statistic_key")); + schema.children[1]->flags &= ~ARROW_FLAG_NULLABLE; + return ADBC_STATUS_OK; +} + +#undef CHECK_STATISTICS_NANOARROW + +namespace { +// Row-count statistics grouped as catalog -> schema -> table, mirroring the +// nesting of the GetStatistics result schema. +struct StatisticsTableEntry { + duckdb::string table_name; + uint64_t row_count; +}; +struct StatisticsSchemaGroup { + duckdb::string name; + duckdb::vector tables; +}; +struct StatisticsCatalogGroup { + duckdb::string name; + duckdb::vector schemas; +}; + +// Releases a manually built ArrowSchema/ArrowArray pair unless ownership was +// transferred to a stream (BatchToArrayStream zeroes its inputs on success, +// which makes this a no-op). +struct StatisticsBatchGuard { + StatisticsBatchGuard(struct ArrowSchema &schema_p, struct ArrowArray &array_p) : schema(schema_p), array(array_p) { + } + ~StatisticsBatchGuard() { + if (schema.release) { + schema.release(&schema); + } + if (array.release) { + array.release(&array); + } + } + struct ArrowSchema &schema; + struct ArrowArray &array; +}; +} // namespace + +// Assembles the nested GetStatistics batch. The caller releases `array` on +// failure. +static AdbcStatusCode BuildGetStatisticsArray(struct ArrowArray &array, + const duckdb::vector &catalogs, + idx_t schema_count, idx_t stat_count, struct AdbcError *error) { + auto catalog_count = catalogs.size(); + + // Root struct: one row per catalog. + auto status = InitStatisticsArrayNode(&array, 1, 2, error); + if (status != ADBC_STATUS_OK) { + return status; + } + array.length = static_cast(catalog_count); + + duckdb::vector catalog_names; + duckdb::vector schema_names; + duckdb::vector table_names; + for (auto &catalog : catalogs) { + catalog_names.push_back(catalog.name); + for (auto &schema_group : catalog.schemas) { + schema_names.push_back(schema_group.name); + for (auto &table : schema_group.tables) { + table_names.push_back(table.table_name); + } + } + } + + status = BuildStatisticsVarcharArray(array.children[0], catalog_names, error); + if (status != ADBC_STATUS_OK) { + return status; + } + + // catalog_db_schemas: list over catalogs. + auto schemas_list = array.children[1]; + status = InitStatisticsArrayNode(schemas_list, 2, 1, error); + if (status != ADBC_STATUS_OK) { + return status; + } + schemas_list->length = static_cast(catalog_count); + auto schema_list_offsets = + static_cast(AllocStatisticsBuffer(schemas_list, 1, sizeof(int32_t) * (catalog_count + 1))); + if (!schema_list_offsets) { + SetError(error, "Failed to allocate offsets for statistics result"); + return ADBC_STATUS_INTERNAL; + } + int32_t schema_offset = 0; + for (idx_t i = 0; i < catalog_count; i++) { + schema_list_offsets[i] = schema_offset; + schema_offset += static_cast(catalogs[i].schemas.size()); + } + schema_list_offsets[catalog_count] = schema_offset; + + // DB_SCHEMA_SCHEMA struct: one row per (catalog, schema) pair. + auto schema_struct = schemas_list->children[0]; + status = InitStatisticsArrayNode(schema_struct, 1, 2, error); + if (status != ADBC_STATUS_OK) { + return status; + } + schema_struct->length = static_cast(schema_count); + + status = BuildStatisticsVarcharArray(schema_struct->children[0], schema_names, error); + if (status != ADBC_STATUS_OK) { + return status; + } + + // db_schema_statistics: list over schemas. + auto stats_list = schema_struct->children[1]; + status = InitStatisticsArrayNode(stats_list, 2, 1, error); + if (status != ADBC_STATUS_OK) { + return status; + } + stats_list->length = static_cast(schema_count); + auto stats_list_offsets = + static_cast(AllocStatisticsBuffer(stats_list, 1, sizeof(int32_t) * (schema_count + 1))); + if (!stats_list_offsets) { + SetError(error, "Failed to allocate offsets for statistics result"); + return ADBC_STATUS_INTERNAL; + } + int32_t stat_offset = 0; + idx_t schema_idx = 0; + for (auto &catalog : catalogs) { + for (auto &schema_group : catalog.schemas) { + stats_list_offsets[schema_idx++] = stat_offset; + stat_offset += static_cast(schema_group.tables.size()); + } + } + stats_list_offsets[schema_count] = stat_offset; + + // STATISTICS_SCHEMA struct: one row per statistic (one ROW_COUNT per table). + auto stats_struct = stats_list->children[0]; + status = InitStatisticsArrayNode(stats_struct, 1, 5, error); + if (status != ADBC_STATUS_OK) { + return status; + } + stats_struct->length = static_cast(stat_count); + + status = BuildStatisticsVarcharArray(stats_struct->children[0], table_names, error); + if (status != ADBC_STATUS_OK) { + return status; + } + // column_name: all NULL (row counts are table-level statistics). + status = BuildStatisticsAllNullVarcharArray(stats_struct->children[1], stat_count, error); + if (status != ADBC_STATUS_OK) { + return status; + } + + // statistic_key: always ADBC_STATISTIC_ROW_COUNT_KEY. + auto key_array = stats_struct->children[2]; + status = InitStatisticsArrayNode(key_array, 2, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + key_array->length = static_cast(stat_count); + auto keys = static_cast(AllocStatisticsBuffer(key_array, 1, sizeof(int16_t) * stat_count)); + if (!keys) { + SetError(error, "Failed to allocate keys for statistics result"); + return ADBC_STATUS_INTERNAL; + } + for (idx_t i = 0; i < stat_count; i++) { + keys[i] = ADBC_STATISTIC_ROW_COUNT_KEY; + } + + // statistic_value: dense union. The ADBC spec types ROW_COUNT as int64 + // when exact and float64 when approximate; DuckDB row counts are always + // approximate estimates, so every value is stored in the float64 child + // (type id 2) and offsets are 0..n-1. + auto value_array = stats_struct->children[3]; + status = InitStatisticsArrayNode(value_array, 2, 4, error); + if (status != ADBC_STATUS_OK) { + return status; + } + value_array->length = static_cast(stat_count); + auto type_ids = static_cast(AllocStatisticsBuffer(value_array, 0, stat_count)); + auto value_offsets = static_cast(AllocStatisticsBuffer(value_array, 1, sizeof(int32_t) * stat_count)); + if (!type_ids || !value_offsets) { + SetError(error, "Failed to allocate union buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + for (idx_t i = 0; i < stat_count; i++) { + type_ids[i] = 2; + value_offsets[i] = static_cast(i); + } + // int64 / uint64 children: empty. + status = InitStatisticsArrayNode(value_array->children[0], 2, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + status = InitStatisticsArrayNode(value_array->children[1], 2, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + // binary child: empty, but variable-size layouts need an offsets buffer + // with a single zero entry. + auto binary_child = value_array->children[3]; + status = InitStatisticsArrayNode(binary_child, 3, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + if (!AllocStatisticsBuffer(binary_child, 1, sizeof(int32_t))) { + SetError(error, "Failed to allocate union buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + // float64 child: carries all row counts. + auto float64_child = value_array->children[2]; + status = InitStatisticsArrayNode(float64_child, 2, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + float64_child->length = static_cast(stat_count); + auto row_counts = static_cast(AllocStatisticsBuffer(float64_child, 1, sizeof(double) * stat_count)); + if (!row_counts) { + SetError(error, "Failed to allocate union buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + idx_t stat_idx = 0; + for (auto &catalog : catalogs) { + for (auto &schema_group : catalog.schemas) { + for (auto &table : schema_group.tables) { + row_counts[stat_idx++] = static_cast(table.row_count); + } + } + } + + // statistic_is_approximate: all true (row counts come from estimates). + auto approximate_array = stats_struct->children[4]; + status = InitStatisticsArrayNode(approximate_array, 2, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + approximate_array->length = static_cast(stat_count); + auto approximate_bits = static_cast(AllocStatisticsBuffer(approximate_array, 1, (stat_count + 7) / 8)); + if (!approximate_bits) { + SetError(error, "Failed to allocate buffers for statistics result"); + return ADBC_STATUS_INTERNAL; + } + memset(approximate_bits, 0xFF, (stat_count + 7) / 8); + return ADBC_STATUS_OK; +} + +AdbcStatusCode ConnectionGetStatistics(struct AdbcConnection *connection, const char *catalog, const char *db_schema, + const char *table_name, char approximate, struct ArrowArrayStream *out, + struct AdbcError *error) { + if (!connection || !connection->private_data) { + SetError(error, "Connection is invalid"); + return ADBC_STATUS_INVALID_ARGUMENT; + } + if (!out) { + SetError(error, "Output parameter was not provided"); + return ADBC_STATUS_INVALID_ARGUMENT; + } + auto conn_wrapper = static_cast(connection->private_data); + if (!conn_wrapper->connection) { + SetError(error, "Connection is not initialized"); + return ADBC_STATUS_INVALID_STATE; + } + // Running a query on this connection invalidates any open streaming results. + conn_wrapper->MaterializeStreams(); + auto conn = reinterpret_cast(conn_wrapper->connection); + + // DuckDB only exposes cheap row-count estimates, so statistics are always + // returned with statistic_is_approximate = true; the spec allows returning + // approximate values regardless of the `approximate` argument. + auto query = duckdb::StringUtil::Format(R"( + SELECT database_name, schema_name, table_name, estimated_size + FROM duckdb_tables() + WHERE NOT internal + AND database_name LIKE %s + AND schema_name LIKE %s + AND table_name LIKE %s + ORDER BY database_name, schema_name, table_name + )", + createFilter(catalog), createFilter(db_schema), createFilter(table_name)); + auto result = conn->Query(query); + if (result->HasError()) { + SetError(error, result->GetError()); + return ADBC_STATUS_INTERNAL; + } + + duckdb::vector catalogs; + idx_t schema_count = 0; + idx_t stat_count = 0; + for (idx_t row_idx = 0; row_idx < result->RowCount(); row_idx++) { + auto size_value = result->GetValue(3, row_idx); + if (size_value.IsNull()) { + continue; + } + auto estimated_size = size_value.GetValue(); + if (estimated_size < 0) { + continue; + } + auto catalog_name = result->GetValue(0, row_idx).GetValue(); + auto schema_name = result->GetValue(1, row_idx).GetValue(); + auto current_table = result->GetValue(2, row_idx).GetValue(); + if (catalogs.empty() || catalogs.back().name != catalog_name) { + catalogs.push_back({catalog_name, {}}); + } + auto &catalog_group = catalogs.back(); + if (catalog_group.schemas.empty() || catalog_group.schemas.back().name != schema_name) { + catalog_group.schemas.push_back({schema_name, {}}); + schema_count++; + } + catalog_group.schemas.back().tables.push_back({current_table, static_cast(estimated_size)}); + stat_count++; + } + + struct ArrowSchema schema; + memset(&schema, 0, sizeof(schema)); + struct ArrowArray array; + memset(&array, 0, sizeof(array)); + StatisticsBatchGuard guard(schema, array); + + auto status = BuildGetStatisticsSchema(schema, error); + if (status != ADBC_STATUS_OK) { + return status; + } + status = BuildGetStatisticsArray(array, catalogs, schema_count, stat_count, error); + if (status != ADBC_STATUS_OK) { + return status; + } + // Per ADBC semantics `out` is a pure output parameter that may contain + // uninitialized stack garbage, so it must be zeroed rather than letting + // BatchToArrayStream inspect `out->release` (its initialization check is + // only meaningful for internally managed, zero-initialized streams). + memset(out, 0, sizeof(*out)); + // On success ownership of schema/array moves into the stream and the guard + // becomes a no-op. + return BatchToArrayStream(&array, &schema, out, error); +} + +AdbcStatusCode ConnectionGetStatisticNames(struct AdbcConnection *connection, struct ArrowArrayStream *out, + struct AdbcError *error) { + if (!connection || !connection->private_data) { + SetError(error, "Connection is invalid"); + return ADBC_STATUS_INVALID_ARGUMENT; + } + if (!out) { + SetError(error, "Output parameter was not provided"); + return ADBC_STATUS_INVALID_ARGUMENT; + } + // DuckDB defines no driver-specific statistics (keys >= 1024), so the + // result is always empty. It is built manually so the NOT NULL fields + // match the spec exactly. + struct ArrowSchema schema; + memset(&schema, 0, sizeof(schema)); + struct ArrowArray array; + memset(&array, 0, sizeof(array)); + StatisticsBatchGuard guard(schema, array); + + auto status = BuildGetStatisticNamesSchema(schema, error); + if (status != ADBC_STATUS_OK) { + return status; + } + status = InitStatisticsArrayNode(&array, 1, 2, error); + if (status != ADBC_STATUS_OK) { + return status; + } + status = BuildStatisticsVarcharArray(array.children[0], duckdb::vector(), error); + if (status != ADBC_STATUS_OK) { + return status; + } + status = InitStatisticsArrayNode(array.children[1], 2, 0, error); + if (status != ADBC_STATUS_OK) { + return status; + } + // `out` may contain uninitialized stack garbage; see ConnectionGetStatistics. + memset(out, 0, sizeof(*out)); + return BatchToArrayStream(&array, &schema, out, error); +} + int ErrorGetDetailCount(const struct AdbcError *error) { if (!error || error->vendor_code != ADBC_ERROR_VENDOR_CODE_PRIVATE_DATA || !error->private_data) { return 0; diff --git a/src/duckdb/src/common/adbc/nanoarrow/schema.cpp b/src/duckdb/src/common/adbc/nanoarrow/schema.cpp index 38d1b314f..29e164dc0 100644 --- a/src/duckdb/src/common/adbc/nanoarrow/schema.cpp +++ b/src/duckdb/src/common/adbc/nanoarrow/schema.cpp @@ -429,6 +429,8 @@ int ArrowSchemaDeepCopy(struct ArrowSchema *schema, struct ArrowSchema *schema_o return result; } + schema_out->flags = schema->flags; + result = ArrowSchemaSetName(schema_out, schema->name); if (result != NANOARROW_OK) { schema_out->release(schema_out); diff --git a/src/duckdb/src/common/arrow/arrow_converter.cpp b/src/duckdb/src/common/arrow/arrow_converter.cpp index 96d98350e..32192b016 100644 --- a/src/duckdb/src/common/arrow/arrow_converter.cpp +++ b/src/duckdb/src/common/arrow/arrow_converter.cpp @@ -177,17 +177,11 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co case LogicalTypeId::UUID: { if (options.arrow_lossless_conversion) { SetArrowExtension(root_holder, child, type, context); + } else if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { + // UUID is cast to a regular (non-view) string by the appender, so never declare "vu". + child.format = "U"; } else { - if (options.produce_arrow_string_view && options.arrow_output_version >= ArrowFormatVersion::V1_4) { - // List views are only introduced in arrow format v1.4 - child.format = "vu"; - } else { - if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { - child.format = "U"; - } else { - child.format = "u"; - } - } + child.format = "u"; } break; } @@ -409,6 +403,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]); InitializeChild(root_holder.nested_children.back()[0], root_holder); child.dictionary = root_holder.nested_children_ptr.back()[0]; + // dict values are always written in the regular int32 layout (see ArrowEnumData). child.dictionary->format = "u"; break; } diff --git a/src/duckdb/src/common/arrow/arrow_type_extension.cpp b/src/duckdb/src/common/arrow/arrow_type_extension.cpp index 6c362e671..e7baa48ec 100644 --- a/src/duckdb/src/common/arrow/arrow_type_extension.cpp +++ b/src/duckdb/src/common/arrow/arrow_type_extension.cpp @@ -280,7 +280,8 @@ struct ArrowJson { root_holder.metadata_info.emplace_back(schema_metadata.SerializeMetadata()); schema.metadata = root_holder.metadata_info.back().get(); const auto options = context.GetClientProperties(); - if (options.produce_arrow_string_view) { + // view layout only when string_view + >= 1.4; declare it to match. + if (options.produce_arrow_string_view && options.arrow_output_version >= ArrowFormatVersion::V1_4) { schema.format = "vu"; } else { if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { @@ -301,6 +302,8 @@ struct ArrowBit { } else if (format == "Z") { return make_uniq(LogicalType::BIT, make_uniq(ArrowVariableSizeType::SUPER_SIZE)); + } else if (format == "vz") { + return make_uniq(LogicalType::BIT, make_uniq(ArrowVariableSizeType::VIEW)); } throw InvalidInputException("Arrow extension type \"%s\" not supported for BIT type", format.c_str()); } @@ -312,7 +315,10 @@ struct ArrowBit { root_holder.metadata_info.emplace_back(schema_metadata.SerializeMetadata()); schema.metadata = root_holder.metadata_info.back().get(); const auto options = context.GetClientProperties(); - if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { + if (options.arrow_output_version >= ArrowFormatVersion::V1_4) { + // >= 1.4 appends the binary view (4-buffer) layout; declare it to match. + schema.format = "vz"; + } else if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { schema.format = "Z"; } else { schema.format = "z"; @@ -329,6 +335,8 @@ struct ArrowBignum { } else if (format == "Z") { return make_uniq(LogicalType::BIGNUM, make_uniq(ArrowVariableSizeType::SUPER_SIZE)); + } else if (format == "vz") { + return make_uniq(LogicalType::BIGNUM, make_uniq(ArrowVariableSizeType::VIEW)); } throw InvalidInputException("Arrow extension type \"%s\" not supported for Bignum", format.c_str()); } @@ -340,7 +348,10 @@ struct ArrowBignum { root_holder.metadata_info.emplace_back(schema_metadata.SerializeMetadata()); schema.metadata = root_holder.metadata_info.back().get(); const auto options = context.GetClientProperties(); - if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { + if (options.arrow_output_version >= ArrowFormatVersion::V1_4) { + // >= 1.4 appends the binary view (4-buffer) layout; declare it to match. + schema.format = "vz"; + } else if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { schema.format = "Z"; } else { schema.format = "z"; @@ -493,7 +504,10 @@ struct ArrowGeometry { schema.metadata = root_holder.metadata_info.back().get(); const auto options = context.GetClientProperties(); - if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { + if (options.arrow_output_version >= ArrowFormatVersion::V1_4) { + // >= 1.4 appends the binary view (4-buffer) layout; declare it to match. + schema.format = "vz"; + } else if (options.arrow_offset_size == ArrowOffsetSize::LARGE) { schema.format = "Z"; } else { schema.format = "z"; diff --git a/src/duckdb/src/common/arrow/physical_arrow_collector.cpp b/src/duckdb/src/common/arrow/physical_arrow_collector.cpp index 3db92b58f..8cae2f497 100644 --- a/src/duckdb/src/common/arrow/physical_arrow_collector.cpp +++ b/src/duckdb/src/common/arrow/physical_arrow_collector.cpp @@ -2,6 +2,7 @@ #include "duckdb/common/arrow/physical_arrow_collector.hpp" #include "duckdb/common/arrow/physical_arrow_batch_collector.hpp" #include "duckdb/common/arrow/arrow_query_result.hpp" +#include "duckdb/common/assert.hpp" #include "duckdb/main/prepared_statement_data.hpp" #include "duckdb/execution/physical_plan_generator.hpp" #include "duckdb/main/client_context.hpp" diff --git a/src/duckdb/src/common/box_renderer.cpp b/src/duckdb/src/common/box_renderer.cpp index b2b839a7b..3f1e71d8e 100644 --- a/src/duckdb/src/common/box_renderer.cpp +++ b/src/duckdb/src/common/box_renderer.cpp @@ -1082,6 +1082,7 @@ bool JSONParser::Process(const string &value) { char quote_char = '"'; bool can_parse_value = false; bool in_unquoted_value = false; + success = true; pos = 0; for (; success && pos < value.size(); pos++) { auto c = value[pos]; @@ -1110,7 +1111,8 @@ bool JSONParser::Process(const string &value) { case ']': { // closing bracket - move to next line and pop back the separator if (separators.empty() || !SeparatorIsMatching(separators.back(), c)) { - throw InternalException("Failed to parse JSON string %s - invalid JSON", value); + success = false; + break; } separators.pop_back(); HandleBracketClose(c); @@ -1171,7 +1173,7 @@ bool JSONParser::Process(const string &value) { state = JSONState::IN_QUOTE; HandleCharacter(c); } else { - throw InternalException("Invalid json state"); + success = false; } } if (!success) { @@ -1536,6 +1538,20 @@ struct JSONHighlighter : public JSONParser { explicit JSONHighlighter(BoxRenderValue &render_value) : render_value(render_value) { } +public: + bool Highlight(const string &value) { + const auto annotation_count = render_value.annotations.size(); + if (JSONParser::Process(value)) { + return true; + } + + while (render_value.annotations.size() > annotation_count) { + render_value.annotations.pop_back(); + } + + return false; + } + protected: void HandleNull() override { render_value.annotations.emplace_back(ResultRenderType::NULL_VALUE, pos); @@ -1582,7 +1598,7 @@ void BoxRendererImplementation::HighlightValue(BoxRenderValue &render_value) { return; } JSONHighlighter highlighter(render_value); - highlighter.Process(render_value.text); + highlighter.Highlight(render_value.text); } void BoxRendererImplementation::PotentiallyExpandRow(BoxRenderRow &row, vector &rows, diff --git a/src/duckdb/src/common/enum_util.cpp b/src/duckdb/src/common/enum_util.cpp index 253a0af39..4aa8b5017 100644 --- a/src/duckdb/src/common/enum_util.cpp +++ b/src/duckdb/src/common/enum_util.cpp @@ -5134,10 +5134,13 @@ const StringUtil::EnumStringLiteral *GetSerializationVersionDeprecatedValues() { { static_cast(SerializationVersionDeprecated::V1_4_2), "V1_4_2" }, { static_cast(SerializationVersionDeprecated::V1_4_3), "V1_4_3" }, { static_cast(SerializationVersionDeprecated::V1_4_4), "V1_4_4" }, + { static_cast(SerializationVersionDeprecated::V1_4_5), "V1_4_5" }, { static_cast(SerializationVersionDeprecated::V1_5_0), "V1_5_0" }, { static_cast(SerializationVersionDeprecated::V1_5_1), "V1_5_1" }, { static_cast(SerializationVersionDeprecated::V1_5_2), "V1_5_2" }, { static_cast(SerializationVersionDeprecated::V1_5_3), "V1_5_3" }, + { static_cast(SerializationVersionDeprecated::V1_5_4), "V1_5_4" }, + { static_cast(SerializationVersionDeprecated::V1_5_5), "V1_5_5" }, { static_cast(SerializationVersionDeprecated::V2_0_0), "V2_0_0" }, { static_cast(SerializationVersionDeprecated::LATEST), "LATEST" }, { static_cast(SerializationVersionDeprecated::INVALID), "INVALID" } @@ -5147,12 +5150,12 @@ const StringUtil::EnumStringLiteral *GetSerializationVersionDeprecatedValues() { template<> const char* EnumUtil::ToChars(SerializationVersionDeprecated value) { - return StringUtil::EnumToString(GetSerializationVersionDeprecatedValues(), 27, "SerializationVersionDeprecated", static_cast(value)); + return StringUtil::EnumToString(GetSerializationVersionDeprecatedValues(), 30, "SerializationVersionDeprecated", static_cast(value)); } template<> SerializationVersionDeprecated EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetSerializationVersionDeprecatedValues(), 27, "SerializationVersionDeprecated", value)); + return static_cast(StringUtil::StringToEnum(GetSerializationVersionDeprecatedValues(), 30, "SerializationVersionDeprecated", value)); } const StringUtil::EnumStringLiteral *GetSetOperationTypeValues() { @@ -5632,10 +5635,13 @@ const StringUtil::EnumStringLiteral *GetStorageVersionValues() { { static_cast(StorageVersion::V1_4_2), "V1_4_2" }, { static_cast(StorageVersion::V1_4_3), "V1_4_3" }, { static_cast(StorageVersion::V1_4_4), "V1_4_4" }, + { static_cast(StorageVersion::V1_4_5), "V1_4_5" }, { static_cast(StorageVersion::V1_5_0), "V1_5_0" }, { static_cast(StorageVersion::V1_5_1), "V1_5_1" }, { static_cast(StorageVersion::V1_5_2), "V1_5_2" }, { static_cast(StorageVersion::V1_5_3), "V1_5_3" }, + { static_cast(StorageVersion::V1_5_4), "V1_5_4" }, + { static_cast(StorageVersion::V1_5_5), "V1_5_5" }, { static_cast(StorageVersion::V2_0_0), "V2_0_0" }, { static_cast(StorageVersion::LATEST), "LATEST" }, { static_cast(StorageVersion::DEPRECATED), "DEPRECATED" }, @@ -5646,12 +5652,12 @@ const StringUtil::EnumStringLiteral *GetStorageVersionValues() { template<> const char* EnumUtil::ToChars(StorageVersion value) { - return StringUtil::EnumToString(GetStorageVersionValues(), 67, "StorageVersion", static_cast(value)); + return StringUtil::EnumToString(GetStorageVersionValues(), 70, "StorageVersion", static_cast(value)); } template<> StorageVersion EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetStorageVersionValues(), 67, "StorageVersion", value)); + return static_cast(StringUtil::StringToEnum(GetStorageVersionValues(), 70, "StorageVersion", value)); } const StringUtil::EnumStringLiteral *GetStrTimeSpecifierValues() { diff --git a/src/duckdb/src/common/identifier.cpp b/src/duckdb/src/common/identifier.cpp index 84be4fad6..bf6e03780 100644 --- a/src/duckdb/src/common/identifier.cpp +++ b/src/duckdb/src/common/identifier.cpp @@ -6,6 +6,10 @@ namespace duckdb { +bool Identifier::StartsWith(const string &prefix) const { + return StringUtil::CIStartsWith(value, prefix); +} + hash_t Identifier::Hash() const { return StringUtil::CIHash(value); } diff --git a/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp b/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp index 257f1be75..7f19b7609 100644 --- a/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +++ b/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp @@ -333,6 +333,13 @@ void PartitionedTupleData::Repartition(ClientContext &context, PartitionedTupleD return; } + // Repartitioning only ever goes to more partitions - fewer would underflow the partition math and index + // out of bounds, so throw instead of corrupting memory. + if (partitions.size() > new_partitioned_data.partitions.size()) { + throw InternalException("PartitionedTupleData::Repartition into fewer partitions (%llu -> %llu)", + partitions.size(), new_partitioned_data.partitions.size()); + } + PartitionedTupleDataAppendState append_state; new_partitioned_data.InitializeAppendState(append_state); diff --git a/src/duckdb/src/execution/operator/helper/physical_execute.cpp b/src/duckdb/src/execution/operator/helper/physical_execute.cpp index 722c18cba..27a4356f4 100644 --- a/src/duckdb/src/execution/operator/helper/physical_execute.cpp +++ b/src/duckdb/src/execution/operator/helper/physical_execute.cpp @@ -4,8 +4,10 @@ namespace duckdb { -PhysicalExecute::PhysicalExecute(PhysicalPlan &physical_plan, PhysicalOperator &plan) - : PhysicalOperator(physical_plan, PhysicalOperatorType::EXECUTE, plan.types, idx_t(-1)), plan(plan) { +PhysicalExecute::PhysicalExecute(PhysicalPlan &physical_plan, PhysicalOperator &plan, + shared_ptr prepared) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXECUTE, plan.types, idx_t(-1)), plan(plan), + prepared(std::move(prepared)) { } vector> PhysicalExecute::GetChildren() const { diff --git a/src/duckdb/src/execution/operator/persistent/physical_insert.cpp b/src/duckdb/src/execution/operator/persistent/physical_insert.cpp index 1565ae697..b3b33f770 100644 --- a/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +++ b/src/duckdb/src/execution/operator/persistent/physical_insert.cpp @@ -82,11 +82,15 @@ InsertGlobalState::InsertGlobalState(ClientContext &context, const vector &types, - const vector> &bound_constraints) + const vector> &bound_constraints, + OnConflictAction action_type) : collection_index(DConstants::INVALID_INDEX), bound_constraints(bound_constraints) { auto &allocator = Allocator::Get(context); update_chunk.Initialize(allocator, types); append_chunk.Initialize(allocator, types); + if (action_type == OnConflictAction::UPDATE) { + updated_rows = make_uniq(context, vector {LogicalType::ROW_TYPE}); + } } ConstraintState &InsertLocalState::GetConstraintState(DataTable &table, TableCatalogEntry &table_ref) { @@ -122,7 +126,7 @@ unique_ptr PhysicalInsert::GetGlobalSinkState(ClientContext &co } unique_ptr PhysicalInsert::GetLocalSinkState(ExecutionContext &context) const { - return make_uniq(context.client, insert_types, bound_constraints); + return make_uniq(context.client, insert_types, bound_constraints, action_type); } bool AllConflictsMeetCondition(DataChunk &result) { @@ -291,7 +295,8 @@ static idx_t PerformOnConflictAction(InsertLocalState &lstate, InsertGlobalState static void RegisterUpdatedRows(InsertLocalState &lstate, const Vector &row_ids, idx_t count) { // Register all row-ids; a distinct count below `count` means a row-id repeated, which we reject. - auto distinct = RegisterRowIds(lstate.updated_rows, row_ids, count); + D_ASSERT(lstate.updated_rows); + auto distinct = lstate.updated_rows->Register(row_ids, count); if (distinct != count) { // This is following postgres behavior: throw InvalidInputException( @@ -486,8 +491,9 @@ static idx_t HandleInsertConflicts(DuckTableEntry &table, ExecutionContext &cont VerifyOnConflictCondition(context, combined_chunk, on_conflict_condition, constraint_state, tuples, data_table, local_storage); - if (&tuples == &lstate.update_chunk) { - // Allow updating duplicate rows for the 'update_chunk' + if (RefersToSameObject(tuples, lstate.update_chunk)) { + D_ASSERT(op.action_type == OnConflictAction::UPDATE); + // Reject updating the same target row more than once. RegisterUpdatedRows(lstate, row_ids, conflict_count); } auto affected_tuples = PerformOnConflictAction(lstate, gstate, context, combined_chunk, table, row_ids, op); diff --git a/src/duckdb/src/execution/operator/persistent/physical_merge_into.cpp b/src/duckdb/src/execution/operator/persistent/physical_merge_into.cpp index 07726a0d5..7772d762e 100644 --- a/src/duckdb/src/execution/operator/persistent/physical_merge_into.cpp +++ b/src/duckdb/src/execution/operator/persistent/physical_merge_into.cpp @@ -111,25 +111,36 @@ class MergeIntoLocalState : public LocalSinkState { class MergeIntoGlobalState : public GlobalSinkState { public: - MergeIntoGlobalState(ClientContext &context, const PhysicalMergeInto &op) : op(op) { + MergeIntoGlobalState(ClientContext &context, const PhysicalMergeInto &op) + : op(op), matched_rows(context, GetRowIdTypes(op)) { for (auto &action : op.actions) { sink_states.push_back(action->op ? action->op->GetGlobalSinkState(context) : nullptr); } merged_count = 0; } + + static vector GetRowIdTypes(const PhysicalMergeInto &op) { + auto &input_types = op.children[0].get().types; + if (op.row_id_index >= input_types.size()) { + throw InternalException("MERGE row ID index is out of range"); + } + auto row_id_offset = NumericCast::difference_type>(op.row_id_index); + return vector(input_types.begin() + row_id_offset, input_types.end()); + } + const PhysicalMergeInto &op; idx_t finalize_idx = 0; vector> sink_states; atomic merged_count; //! Target row-ids already matched by a WHEN MATCHED modifying action, to detect a second action on a row. mutex match_lock; - unordered_set matched_rows; + RowIdDeduplicator matched_rows; - //! Record the matched target rows; throw if any row was already matched (cardinality violation). Uses the - //! shared row-id registration primitive; a distinct count below the input count means a row-id repeated. + //! Record the matched target rows; throw if any row was already matched (cardinality violation). A distinct + //! count below the input count means a row ID repeated. void CheckMatchedRows(DataChunk &matched, idx_t row_id_index) { lock_guard glock(match_lock); - auto distinct = RegisterRowIds(matched_rows, matched.data[row_id_index], matched.size()); + auto distinct = matched_rows.Register(matched, row_id_index); if (distinct != matched.size()) { throw InvalidInputException( "MERGE INTO command cannot affect the same target row more than once. A target row matched more " @@ -313,6 +324,7 @@ void PhysicalMergeInto::ComputeMatches(MergeIntoLocalState &local_state, DataChu not_matched.count = 0; not_matched_by_source.count = 0; + // The first row-ID component is also the target-presence marker and must be non-NULL for target rows. auto row_id_validity = chunk.data[row_id_index].Validity(); if (source_marker.IsValid()) { // source marker - check both row id and source marker diff --git a/src/duckdb/src/execution/operator/persistent/physical_update.cpp b/src/duckdb/src/execution/operator/persistent/physical_update.cpp index 36e9a034b..15d5eabec 100644 --- a/src/duckdb/src/execution/operator/persistent/physical_update.cpp +++ b/src/duckdb/src/execution/operator/persistent/physical_update.cpp @@ -55,13 +55,17 @@ PhysicalUpdate::PhysicalUpdate(PhysicalPlan &physical_plan, vector //===--------------------------------------------------------------------===// class UpdateGlobalState : public GlobalSinkState { public: - explicit UpdateGlobalState(ClientContext &context, const vector &return_types) + explicit UpdateGlobalState(ClientContext &context, const vector &return_types, + RowIdHandling row_id_handling) : updated_count(0), return_collection(context, return_types) { + if (row_id_handling != RowIdHandling::ASSUME_UNIQUE) { + updated_rows = make_uniq(context, vector {LogicalType::ROW_TYPE}); + } } mutex lock; atomic updated_count; - unordered_set updated_rows; + unique_ptr updated_rows; ColumnDataCollection return_collection; }; @@ -173,7 +177,8 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, if (deduplicate) { sel.Initialize(update_chunk.size()); lock_guard glock(g_state.lock); - update_count = RegisterRowIds(g_state.updated_rows, row_ids, update_chunk.size(), sel); + D_ASSERT(g_state.updated_rows); + update_count = g_state.updated_rows->Register(row_ids, update_chunk.size(), sel); if (row_id_handling == RowIdHandling::ERROR && update_count != update_chunk.size()) { throw InvalidInputException("UPDATE command cannot update the same row more than once"); } @@ -218,7 +223,8 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, idx_t update_count = update_chunk.size(); const bool deduplicate = row_id_handling != RowIdHandling::ASSUME_UNIQUE; if (deduplicate) { - update_count = RegisterRowIds(g_state.updated_rows, row_ids, update_chunk.size(), sel); + D_ASSERT(g_state.updated_rows); + update_count = g_state.updated_rows->Register(row_ids, update_chunk.size(), sel); if (row_id_handling == RowIdHandling::ERROR && update_count != update_chunk.size()) { throw InvalidInputException("UPDATE command cannot update the same row more than once"); } @@ -274,7 +280,7 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, } unique_ptr PhysicalUpdate::GetGlobalSinkState(ClientContext &context) const { - return make_uniq(context, GetTypes()); + return make_uniq(context, GetTypes(), row_id_handling); } unique_ptr PhysicalUpdate::GetLocalSinkState(ExecutionContext &context) const { diff --git a/src/duckdb/src/execution/physical_plan/plan_execute.cpp b/src/duckdb/src/execution/physical_plan/plan_execute.cpp index 66a0ca033..8e43330f0 100644 --- a/src/duckdb/src/execution/physical_plan/plan_execute.cpp +++ b/src/duckdb/src/execution/physical_plan/plan_execute.cpp @@ -7,15 +7,13 @@ namespace duckdb { PhysicalOperator &PhysicalPlanGenerator::CreatePlan(LogicalExecute &op) { if (op.prepared->physical_plan) { D_ASSERT(op.children.empty()); - return Make(op.prepared->physical_plan->Root()); + auto &plan = op.prepared->physical_plan->Root(); + return Make(plan, op.prepared); } D_ASSERT(op.children.size() == 1); auto &plan = CreatePlan(*op.children[0]); - auto &execute = Make(plan); - auto &cast_execute = execute.Cast(); - cast_execute.prepared = op.prepared; - return execute; + return Make(plan, op.prepared); } } // namespace duckdb diff --git a/src/duckdb/src/execution/radix_partitioned_hashtable.cpp b/src/duckdb/src/execution/radix_partitioned_hashtable.cpp index 4736c0d79..0401d32e7 100644 --- a/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +++ b/src/duckdb/src/execution/radix_partitioned_hashtable.cpp @@ -527,6 +527,21 @@ void DecideAdaptation(RadixHTGlobalSinkState &gstate, RadixHTLocalSinkState &lst } } +// Grow abandoned_data (frozen at the radix bits of when we first went external) up to the current radix bits, so it +// never has fewer partitions than the data merged into it - Repartition/Combine only handle new >= old. +void GrowAbandonedDataToRadixBits(ClientContext &context, RadixHTGlobalSinkState &gstate, RadixHTLocalSinkState &lstate, + const idx_t radix_bits) { + if (!lstate.abandoned_data || + lstate.abandoned_data->PartitionCount() >= RadixPartitioning::NumberOfPartitions(radix_bits)) { + return; + } + auto new_abandoned_data = make_uniq( + BufferManager::GetBufferManager(context), gstate.radix_ht.GetLayoutPtr(), MemoryTag::HASH_TABLE, radix_bits, + gstate.radix_ht.GetLayout().ColumnCount() - 1, context); + lstate.abandoned_data->Repartition(context, *new_abandoned_data); + lstate.abandoned_data = std::move(new_abandoned_data); +} + void MaybeRepartition(ClientContext &context, RadixHTGlobalSinkState &gstate, RadixHTLocalSinkState &lstate, const bool combine) { auto &config = gstate.config; @@ -556,12 +571,15 @@ void MaybeRepartition(ClientContext &context, RadixHTGlobalSinkState &gstate, Ra if (total_size > gstate.GetThreadLimit()) { if (gstate.config.SetRadixBitsToExternal()) { // We're approaching the memory limit, unpin the data + const auto external_radix_bits = config.GetRadixBits(); if (!lstate.abandoned_data) { lstate.abandoned_data = make_uniq( BufferManager::GetBufferManager(context), gstate.radix_ht.GetLayoutPtr(), MemoryTag::HASH_TABLE, - config.GetRadixBits(), gstate.radix_ht.GetLayout().ColumnCount() - 1, context); + external_radix_bits, gstate.radix_ht.GetLayout().ColumnCount() - 1, context); + } else { + GrowAbandonedDataToRadixBits(context, gstate, lstate, external_radix_bits); } - ht.SetRadixBits(gstate.config.GetRadixBits()); + ht.SetRadixBits(external_radix_bits); ht.AcquirePartitionedData()->Repartition(context, *lstate.abandoned_data); } } @@ -670,6 +688,8 @@ void RadixPartitionedHashTable::Combine(ExecutionContext &context, GlobalSinkSta auto lstate_data = ht.AcquirePartitionedData(); if (lstate.abandoned_data) { D_ASSERT(gstate.external); + // The global radix bits may have grown after we last sized abandoned_data - grow it to match before combining + GrowAbandonedDataToRadixBits(context.client, gstate, lstate, gstate.config.GetRadixBits()); D_ASSERT(lstate.abandoned_data->PartitionCount() == lstate.ht->GetPartitionedData().PartitionCount()); D_ASSERT(lstate.abandoned_data->PartitionCount() == RadixPartitioning::NumberOfPartitions(gstate.config.GetRadixBits())); diff --git a/src/duckdb/src/execution/row_id_deduplicator.cpp b/src/duckdb/src/execution/row_id_deduplicator.cpp new file mode 100644 index 000000000..e6a6d28eb --- /dev/null +++ b/src/duckdb/src/execution/row_id_deduplicator.cpp @@ -0,0 +1,66 @@ +#include "duckdb/execution/row_id_deduplicator.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/execution/aggregate_hashtable.hpp" +#include "duckdb/main/client_context.hpp" + +namespace duckdb { + +RowIdDeduplicator::RowIdDeduplicator(ClientContext &context, vector row_id_types_p) + : row_id_types(std::move(row_id_types_p)), addresses(LogicalType::POINTER), new_groups(STANDARD_VECTOR_SIZE) { + if (row_id_types.empty()) { + throw InternalException("Cannot deduplicate an empty row ID"); + } + row_id_chunk.InitializeEmpty(row_id_types); + hash_table = make_uniq(context, Allocator::Get(context), row_id_types); +} + +RowIdDeduplicator::~RowIdDeduplicator() { +} + +idx_t RowIdDeduplicator::Register(DataChunk &input, idx_t row_id_start, optional_ptr sel) { + if (row_id_start >= input.ColumnCount() || input.ColumnCount() - row_id_start != row_id_types.size()) { + throw InternalException("Row ID column range does not match the deduplicator types"); + } + for (idx_t i = 0; i < row_id_types.size(); i++) { + auto &input_vector = input.data[row_id_start + i]; + if (input_vector.GetType() != row_id_types[i]) { + throw InternalException("Row ID type mismatch in deduplicator"); + } + row_id_chunk.data[i].Reference(input_vector); + } + row_id_chunk.SetCardinalityUnsafe(input.size()); + return Register(row_id_chunk, sel); +} + +idx_t RowIdDeduplicator::Register(const Vector &row_ids, idx_t count, optional_ptr sel) { + if (row_id_types.size() != 1 || row_ids.GetType() != row_id_types[0]) { + throw InternalException("Single row ID vector does not match the deduplicator types"); + } + if (count > row_ids.size()) { + throw InternalException("Row ID count exceeds vector size"); + } + row_id_chunk.data[0].Reference(row_ids); + row_id_chunk.SetCardinalityUnsafe(count); + return Register(row_id_chunk, sel); +} + +idx_t RowIdDeduplicator::Register(DataChunk &row_ids, optional_ptr sel) { + auto count = row_ids.size(); + if (count == 0) { + return 0; + } + addresses.Reserve(count); + auto &result_sel = sel ? *sel : new_groups; + if (result_sel.Capacity() < count) { + result_sel.Initialize(count); + } + auto distinct_count = hash_table->FindOrCreateGroups(row_ids, addresses, result_sel); + if (sel) { + // The hash table may discover new groups in probe order rather than input order. + result_sel.Sort(distinct_count); + } + return distinct_count; +} + +} // namespace duckdb diff --git a/src/duckdb/src/function/cast/string_cast.cpp b/src/duckdb/src/function/cast/string_cast.cpp index e38f6592c..68ff74c06 100644 --- a/src/duckdb/src/function/cast/string_cast.cpp +++ b/src/duckdb/src/function/cast/string_cast.cpp @@ -161,6 +161,9 @@ bool VectorStringToList::StringToNestedTypeCastLoop(const string_t *source_data, auto varchar_vector_validity = varchar_vector.Validity(); // Something went wrong in the conversion, we need to nullify the parent for (idx_t i = 0; i < count; i++) { + if (!result_mask.RowIsValid(i)) { + continue; + } for (idx_t j = list_data[i].offset; j < list_data[i].offset + list_data[i].length; j++) { if (!result_child_validity.IsValid(j) && varchar_vector_validity.IsValid(j)) { result_mask.SetInvalid(i); diff --git a/src/duckdb/src/function/cast/vector_cast_helpers.cpp b/src/duckdb/src/function/cast/vector_cast_helpers.cpp index 467264eff..af1a97c56 100644 --- a/src/duckdb/src/function/cast/vector_cast_helpers.cpp +++ b/src/duckdb/src/function/cast/vector_cast_helpers.cpp @@ -459,6 +459,9 @@ bool VectorStringToStruct::SplitStruct(const string_t &input, vector &va auto end_char = buf[pos] == '{' ? '}' : ')'; pos++; SkipWhitespace(input_state); + if (pos == len) { + return false; + } if (buf[pos] == end_char) { pos++; SkipWhitespace(input_state); diff --git a/src/duckdb/src/function/scalar_macro_function.cpp b/src/duckdb/src/function/scalar_macro_function.cpp index c71f94474..756fffe2f 100644 --- a/src/duckdb/src/function/scalar_macro_function.cpp +++ b/src/duckdb/src/function/scalar_macro_function.cpp @@ -33,8 +33,7 @@ void RemoveQualificationRecursive(unique_ptr &root_expr) { ParsedExpressionIterator::VisitExpressionMutable( *root_expr, [&](ColumnRefExpression &col_ref) { auto &col_names = col_ref.ColumnNamesMutable(); - if (col_names.size() == 2 && - StringUtil::StartsWith(col_names[0].GetIdentifierName(), DummyBinding::DUMMY_NAME)) { + if (col_names.size() == 2 && col_names[0].StartsWith(DummyBinding::DUMMY_NAME)) { col_names.erase(col_names.begin()); } }); diff --git a/src/duckdb/src/function/table/system/duckdb_constraints.cpp b/src/duckdb/src/function/table/system/duckdb_constraints.cpp index a5c5acb16..10e61bd92 100644 --- a/src/duckdb/src/function/table/system/duckdb_constraints.cpp +++ b/src/duckdb/src/function/table/system/duckdb_constraints.cpp @@ -316,15 +316,15 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ column_index_list.push_back(Value::UBIGINT(col_index.index)); } for (auto &name : info.column_names) { - column_name_list.push_back(Value(std::move(name))); + column_name_list.push_back(Value(name)); } for (auto &name : info.referenced_columns) { - referenced_column_name_list.push_back(Value(std::move(name))); + referenced_column_name_list.push_back(Value(name)); } constraint_column_indexes.Append(Value::LIST(LogicalType::BIGINT, std::move(column_index_list))); constraint_column_names.Append(Value::LIST(LogicalType::VARCHAR, std::move(column_name_list))); constraint_name_vec.Append(Value(std::move(constraint_name))); - referenced_table.Append(info.referenced_table.empty() ? Value() : Value(std::move(info.referenced_table))); + referenced_table.Append(info.referenced_table.empty() ? Value() : Value(info.referenced_table)); referenced_column_names.Append(Value::LIST(LogicalType::VARCHAR, std::move(referenced_column_name_list))); count++; } diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index 37094234f..34d2e77c4 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "0-alpha36050" +#define DUCKDB_PATCH_VERSION "0-alpha36121" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 0 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 2 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v2.0.0-alpha36050" +#define DUCKDB_VERSION "v2.0.0-alpha36121" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "af1b4a9bd2" +#define DUCKDB_SOURCE_ID "d54e4f6a20" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/function/window/window_boundaries_state.cpp b/src/duckdb/src/function/window/window_boundaries_state.cpp index efb6c3dba..53fabcc95 100644 --- a/src/duckdb/src/function/window/window_boundaries_state.cpp +++ b/src/duckdb/src/function/window/window_boundaries_state.cpp @@ -755,8 +755,9 @@ void WindowBoundariesState::FrameBegin(idx_t row_idx, const idx_t count, WindowI break; case WindowBoundary::EXPR_FOLLOWING_RANGE: for (idx_t chunk_idx = 0; chunk_idx < count; ++chunk_idx, ++row_idx) { + const auto peer_begin = peer_begin_data[chunk_idx]; if (boundary_begin.CellIsNull(chunk_idx)) { - window_start = peer_begin_data[chunk_idx]; + window_start = peer_begin; } else { const auto valid_end = valid_end_data[chunk_idx]; prev.end = valid_end; @@ -765,7 +766,7 @@ void WindowBoundariesState::FrameBegin(idx_t row_idx, const idx_t count, WindowI prev.start = valid_begin_data[chunk_idx]; prev_partition = cur_partition; } - window_start = FindOrderedRangeBound(*range_lo, *range_hi, range_sense, row_idx, valid_end, + window_start = FindOrderedRangeBound(*range_lo, *range_hi, range_sense, peer_begin, valid_end, start_boundary, boundary_begin, chunk_idx, prev); prev.start = window_start; } @@ -891,8 +892,9 @@ void WindowBoundariesState::FrameEnd(idx_t row_idx, const idx_t count, WindowInp break; case WindowBoundary::EXPR_PRECEDING_RANGE: for (idx_t chunk_idx = 0; chunk_idx < count; ++chunk_idx, ++row_idx) { + const auto peer_end = peer_end_data[chunk_idx]; if (boundary_end.CellIsNull(chunk_idx)) { - window_end = peer_end_data[chunk_idx]; + window_end = peer_end; } else { const auto valid_start = valid_begin_data[chunk_idx]; prev.start = valid_start; @@ -901,7 +903,7 @@ void WindowBoundariesState::FrameEnd(idx_t row_idx, const idx_t count, WindowInp prev.end = valid_end; prev_partition = cur_partition; } - window_end = FindOrderedRangeBound(*range_lo, *range_hi, range_sense, valid_start, row_idx + 1, + window_end = FindOrderedRangeBound(*range_lo, *range_hi, range_sense, valid_start, peer_end, end_boundary, boundary_end, chunk_idx, prev); prev.end = window_end; } diff --git a/src/duckdb/src/function/window/window_rank_function.cpp b/src/duckdb/src/function/window/window_rank_function.cpp index 3cbdd8dca..bebc683d8 100644 --- a/src/duckdb/src/function/window/window_rank_function.cpp +++ b/src/duckdb/src/function/window/window_rank_function.cpp @@ -498,7 +498,8 @@ unique_ptr WindowCumeDistExecutor::GetLocal(ExecutionContext &co static inline double CumeDist(const idx_t begin, const idx_t end, const idx_t peer_end) { const auto denom = static_cast(NumericCast(end - begin)); - const auto num = static_cast(peer_end - begin); + const auto num_begin = MaxValue(peer_end, begin); + const auto num = static_cast(num_begin - begin); return denom > 0 ? (num / denom) : 0; } diff --git a/src/duckdb/src/function/window/window_value_function.cpp b/src/duckdb/src/function/window/window_value_function.cpp index 4ebe4f467..f95f0f0e4 100644 --- a/src/duckdb/src/function/window/window_value_function.cpp +++ b/src/duckdb/src/function/window/window_value_function.cpp @@ -863,11 +863,11 @@ void WindowFirstValueExecutor::GetData(ExecutionContext &context, DataChunk &eva if (frame_width) { const auto first_idx = gvstate.value_tree->SelectNth(frames, 0); - D_ASSERT(first_idx.second == 0); - if (first_idx.first < cursor.Count()) { - cursor.CopyCell(0, first_idx.first, result, i); - } else { + if (first_idx.second || first_idx.first >= cursor.Count()) { + // No first value - give up. FlatVector::SetNull(result, i, true); + } else { + cursor.CopyCell(0, first_idx.first, result, i); } } else { FlatVector::SetNull(result, i, true); diff --git a/src/duckdb/src/include/duckdb/catalog/catalog.hpp b/src/duckdb/src/include/duckdb/catalog/catalog.hpp index 46340a07d..78fdaf812 100644 --- a/src/duckdb/src/include/duckdb/catalog/catalog.hpp +++ b/src/duckdb/src/include/duckdb/catalog/catalog.hpp @@ -513,13 +513,17 @@ class Catalog { OnEntryNotFound if_not_found); static CatalogEntryLookup TryLookupEntry(CatalogEntryRetriever &retriever, const vector &lookups, const EntryLookupInfo &lookup_info, OnEntryNotFound if_not_found, - bool allow_default_table_lookup); + bool allow_default_lookup); //! Looks for a Catalog with a DefaultTable that matches the lookup static CatalogEntryLookup TryLookupDefaultTable(CatalogEntryRetriever &retriever, const EntryLookupInfo &lookup_info, bool allow_ignore_at_clause = false); + //! Looks for a non-table entry in the default schema of any implicit search catalog + static CatalogEntryLookup TryLookupDefaultSchema(CatalogEntryRetriever &retriever, + const EntryLookupInfo &lookup_info); + //! Return an exception with did-you-mean suggestion. static CatalogException CreateMissingEntryException(CatalogEntryRetriever &retriever, const EntryLookupInfo &lookup_info, diff --git a/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp b/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp index 83189646d..e5ab48e2a 100644 --- a/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +++ b/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp @@ -19,9 +19,14 @@ namespace duckdb { class ClientContext; struct CatalogSearchEntry { - CatalogSearchEntry(Identifier catalog, Identifier schema); + CatalogSearchEntry(Identifier catalog, Identifier schema, bool default_schema_precedence = false); public: + //! For unqualified non-table lookups, search this implicit catalog's default schema at its position in the search + //! path rather than as a fallback + bool DefaultSchemaPrecedence() const { + return default_schema_precedence; + } const Identifier &GetCatalog() const { return catalog; } @@ -47,6 +52,7 @@ struct CatalogSearchEntry { private: Identifier catalog; Identifier schema; + bool default_schema_precedence; }; enum class CatalogSetPathType { SET_SCHEMA, SET_SCHEMAS, SET_DIRECTLY }; @@ -76,6 +82,10 @@ class CatalogSearchPath { DUCKDB_API vector GetSchemasForCatalog(const Identifier &catalog) const; DUCKDB_API vector GetCatalogsForSchema(const Identifier &schema) const; + DUCKDB_API vector GetImplicitSearchCatalogs() const; + + DUCKDB_API vector GetWithPrecedenceSchemas(ClientContext &context) const; + DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const Identifier &catalog_name, const Identifier &schema_name) const; diff --git a/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp b/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp index a1466dd63..109cb06e1 100644 --- a/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +++ b/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp @@ -137,6 +137,13 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const AdbcStatusCode ConnectionGetTableTypes(struct AdbcConnection *connection, struct ArrowArrayStream *out, struct AdbcError *error); +AdbcStatusCode ConnectionGetStatistics(struct AdbcConnection *connection, const char *catalog, const char *db_schema, + const char *table_name, char approximate, struct ArrowArrayStream *out, + struct AdbcError *error); + +AdbcStatusCode ConnectionGetStatisticNames(struct AdbcConnection *connection, struct ArrowArrayStream *out, + struct AdbcError *error); + AdbcStatusCode ConnectionReadPartition(struct AdbcConnection *connection, const uint8_t *serialized_partition, size_t serialized_length, struct ArrowArrayStream *out, struct AdbcError *error); diff --git a/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp b/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp index b65da8176..a8ac667dc 100644 --- a/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp +++ b/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp @@ -69,8 +69,15 @@ struct ArrowEnumData : public ArrowScalarBaseData { static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) { result.GetMainBuffer().reserve(capacity * sizeof(TGT)); - // construct the enum child data - auto enum_data = ArrowAppender::InitializeChild(LogicalType::VARCHAR, EnumType::GetSize(type), result.options); + // EnumAppendVector always writes the dictionary in the regular int32-offset string + // layout, so force a non-view, regular-offset child regardless of the connection's + // produce_arrow_string_view / arrow_large_buffer_size settings. Otherwise the child + // would be finalized as a view (4-buffer) or large (int64-offset) array over int32 + // data, producing a malformed dictionary that consumers read as garbage. + ClientProperties dict_options = result.options; + dict_options.produce_arrow_string_view = false; + dict_options.arrow_offset_size = ArrowOffsetSize::REGULAR; + auto enum_data = ArrowAppender::InitializeChild(LogicalType::VARCHAR, EnumType::GetSize(type), dict_options); EnumAppendVector(*enum_data, EnumType::GetValuesInsertOrder(type), EnumType::GetSize(type)); result.child_data.push_back(std::move(enum_data)); } diff --git a/src/duckdb/src/include/duckdb/common/http_util.hpp b/src/duckdb/src/include/duckdb/common/http_util.hpp index efbb8b322..0ea4f0368 100644 --- a/src/duckdb/src/include/duckdb/common/http_util.hpp +++ b/src/duckdb/src/include/duckdb/common/http_util.hpp @@ -310,6 +310,12 @@ class HTTPUtil { virtual unique_ptr SendRequest(BaseRequest &request, unique_ptr &client); virtual void LogRequest(BaseRequest &request, optional_ptr response); + //! Whether a failed request should be retried, possibly using HTTPResponse information, and allowing overrides + DUCKDB_API virtual bool ShouldRetry(const BaseRequest &request, const HTTPResponse &response); + + //! Whether replaying this request is safe. POST is the only method we cannot assume is idempotent. + DUCKDB_API static bool IsIdempotent(RequestType type); + static void ParseHTTPProxyHost(string &proxy_value, string &hostname_out, idx_t &port_out, idx_t default_port = 80); static void DecomposeURL(const string &url, string &path_out, string &proto_host_port_out); static HTTPStatusCode ToStatusCode(int32_t status_code); diff --git a/src/duckdb/src/include/duckdb/common/identifier.hpp b/src/duckdb/src/include/duckdb/common/identifier.hpp index 7978c89fc..94c99c50d 100644 --- a/src/duckdb/src/include/duckdb/common/identifier.hpp +++ b/src/duckdb/src/include/duckdb/common/identifier.hpp @@ -77,6 +77,9 @@ class Identifier { return value.c_str(); } + //! Whether the identifier starts with the given prefix (case-insensitive) + DUCKDB_API bool StartsWith(const string &prefix) const; + //! Case-insensitive hash of the identifier DUCKDB_API hash_t Hash() const; diff --git a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp index 99eb8259b..2016972c8 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp @@ -427,9 +427,44 @@ class MultiFileFunction : public TableFunction { } static void InitializeDecodeChunk(ClientContext &context, MultiFileLocalState &lstate, - vector &projection_ids) { + MultiFileGlobalState &gstate) { auto &reader = *lstate.job.reader; auto &reader_data = *lstate.job.reader_data; + auto &projection_ids = gstate.projection_ids; + + if (!reader_data.extra_columns.empty()) { + if (!gstate.multi_file_reader_state || !gstate.multi_file_reader_state->supports_local_extra_columns) { + throw InternalException( + "MultiFileReader added local extra columns without declaring support in its global state"); + } + } + if (!projection_ids.empty()) { + vector extra_columns; + if (gstate.multi_file_reader_state) { + extra_columns = reader_data.GetExtraColumns(*gstate.multi_file_reader_state); + } + const auto expected_expression_count = gstate.scanned_types.size() + extra_columns.size(); + if (reader_data.expressions.size() != expected_expression_count) { + throw InternalException( + "MultiFileReader input schema contains %llu columns, but the reader produced %llu expressions", + expected_expression_count, reader_data.expressions.size()); + } + for (idx_t i = 0; i < gstate.scanned_types.size(); i++) { + if (reader_data.expressions[i]->GetReturnType() != gstate.scanned_types[i]) { + throw InternalException( + "MultiFileReader input column %llu has type %s, but the reader expression has type %s", i, + gstate.scanned_types[i], reader_data.expressions[i]->GetReturnType()); + } + } + for (idx_t i = 0; i < extra_columns.size(); i++) { + auto expression_idx = gstate.scanned_types.size() + i; + if (reader_data.expressions[expression_idx]->GetReturnType() != extra_columns[i]) { + throw InternalException( + "MultiFileReader extra column %llu has type %s, but the reader expression has type %s", i, + extra_columns[i], reader_data.expressions[expression_idx]->GetReturnType()); + } + } + } //! Initialize the intermediate chunk to be used by the underlying reader before being finalized vector intermediate_chunk_types; auto &local_column_ids = reader.column_indexes; @@ -677,15 +712,10 @@ class MultiFileFunction : public TableFunction { } result->scanned_types.emplace_back(entry->second.type); } else { - result->scanned_types.push_back(table_types[column_id]); + result->scanned_types.push_back(col_idx.HasType() ? col_idx.GetScanType() : table_types[column_id]); } } } - if (require_extra_columns) { - for (const auto &column_type : result->multi_file_reader_state->extra_columns) { - result->scanned_types.push_back(column_type); - } - } InitializeReadAhead(context, bind_data, *result); return std::move(result); } @@ -734,7 +764,7 @@ class MultiFileFunction : public TableFunction { MultiFileBindData &bind_data, DataChunk &output) { if (data.scan_chunk_file_index != data.job.file_index) { // if the file changes we need to initialize the chunk again - InitializeDecodeChunk(context, data, gstate.projection_ids); + InitializeDecodeChunk(context, data, gstate); } auto &scan_chunk = data.scan_chunk; if (!data.resuming_blocked_scan) { diff --git a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp index ccb3ba9a5..c7dc82c1e 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp @@ -41,18 +41,22 @@ struct MultiFileReaderBindData { //! Global state for MultiFileReads struct MultiFileReaderGlobalState { - MultiFileReaderGlobalState(vector extra_columns_p, optional_ptr file_list_p) - : extra_columns(std::move(extra_columns_p)), file_list(file_list_p) {}; + MultiFileReaderGlobalState(vector extra_columns_p, optional_ptr file_list_p, + bool supports_local_extra_columns_p = false) + : extra_columns(std::move(extra_columns_p)), file_list(file_list_p), + supports_local_extra_columns(supports_local_extra_columns_p) {}; virtual ~MultiFileReaderGlobalState(); //! extra columns that will be produced during scanning const vector extra_columns; // the file list driving the current scan const optional_ptr file_list; + //! Whether individual readers can add extra columns during InitializeReader + const bool supports_local_extra_columns; //! Indicates that the MultiFileReader has added columns to be scanned that are not in the projection - bool RequiresExtraColumns() { - return !extra_columns.empty(); + bool RequiresExtraColumns() const { + return !extra_columns.empty() || supports_local_extra_columns; } template @@ -135,6 +139,15 @@ struct MultiFileReaderData { MultiFileConstantMap constant_map; //! The set of expressions that should be evaluated to obtain the final result vector> expressions; + //! Extra columns required by FinalizeChunk for this file, appended after any global extra columns + //! Is only allowed to be populated when MultiFileReaderGlobalState::supports_local_extra_columns is set + vector extra_columns; + + vector GetExtraColumns(const MultiFileReaderGlobalState &global_state) const { + auto result = global_state.extra_columns; + result.insert(result.end(), extra_columns.begin(), extra_columns.end()); + return result; + } //! (only set when file_state is UNOPENED) the file to be opened OpenFileInfo file_to_be_opened; diff --git a/src/duckdb/src/include/duckdb/common/sql_identifier.hpp b/src/duckdb/src/include/duckdb/common/sql_identifier.hpp index d7dbb6e4b..f8edf6f5f 100644 --- a/src/duckdb/src/include/duckdb/common/sql_identifier.hpp +++ b/src/duckdb/src/include/duckdb/common/sql_identifier.hpp @@ -59,6 +59,9 @@ class SQLQuotedIdentifier { public: explicit SQLQuotedIdentifier(const string &raw_string) : raw_string(raw_string) { } + explicit SQLQuotedIdentifier(const char *raw_string) : raw_string(raw_string) { + } + explicit SQLQuotedIdentifier(const Identifier &id); //! Emits an optionally quoted identifier including required escapes (i.e. ident -> ident, table -> "table") static string ToString(const string &identifier); diff --git a/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp b/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp index bf62eb7af..8cb9823d0 100644 --- a/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp +++ b/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp @@ -443,9 +443,26 @@ void MergeSortTree::BuildRun(idx_t level_idx, idx_t run_idx) { template pair MergeSortTree::SelectNth(const SubFrames &frames, idx_t n) const { - // Handle special case of a one-element tree + // Find Nth element in a top-down traversal + idx_t result = 0; + + // Handle special case of a 0/1-element tree if (tree.size() < 2) { - return {0, 0}; + ++n; + + // Zero element tree always fails + if (tree[0].first.empty()) { + return {result, n}; + } + + // Either the element is in the frame or it isn't + const auto *level_data = tree[0].first.data(); + const auto v = level_data[result]; + for (const auto &frame : frames) { + n -= (v >= frame.start) && (v < frame.end); + } + + return {result, n}; } // The first level contains a single run, @@ -456,9 +473,6 @@ pair MergeSortTree::SelectNth(const SubFrames &fr level_width *= FANOUT; } - // Find Nth element in a top-down traversal - idx_t result = 0; - // First, handle levels with cascading pointers const auto min_cascaded = LowestCascadingLevel(); if (level_no > min_cascaded) { diff --git a/src/duckdb/src/include/duckdb/execution/operator/helper/physical_execute.hpp b/src/duckdb/src/include/duckdb/execution/operator/helper/physical_execute.hpp index ab9605290..ea62de16c 100644 --- a/src/duckdb/src/include/duckdb/execution/operator/helper/physical_execute.hpp +++ b/src/duckdb/src/include/duckdb/execution/operator/helper/physical_execute.hpp @@ -18,9 +18,10 @@ class PhysicalExecute : public PhysicalOperator { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EXECUTE; public: - PhysicalExecute(PhysicalPlan &physical_plan, PhysicalOperator &plan); + PhysicalExecute(PhysicalPlan &physical_plan, PhysicalOperator &plan, shared_ptr prepared); PhysicalOperator &plan; + //! The prepared statement we are executing - kept alive here because it can own the plan we refer to shared_ptr prepared; public: diff --git a/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp b/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp index 7394b0fc5..b170ad802 100644 --- a/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +++ b/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp @@ -18,6 +18,7 @@ #include "duckdb/storage/table/delete_state.hpp" #include "duckdb/storage/optimistic_data_writer.hpp" #include "duckdb/common/types/column/column_data_collection.hpp" +#include "duckdb/execution/row_id_deduplicator.hpp" namespace duckdb { @@ -39,7 +40,7 @@ class InsertLocalState : public LocalSinkState { public: public: InsertLocalState(ClientContext &context, const vector &types, - const vector> &bound_constraints); + const vector> &bound_constraints, OnConflictAction action_type); public: ConstraintState &GetConstraintState(DataTable &table, TableCatalogEntry &table_ref); @@ -53,7 +54,7 @@ class InsertLocalState : public LocalSinkState { PhysicalIndex collection_index; unique_ptr optimistic_writer; // Rows that have been updated by a DO UPDATE conflict - unordered_set updated_rows; + unique_ptr updated_rows; idx_t update_count = 0; unique_ptr constraint_state; const vector> &bound_constraints; diff --git a/src/duckdb/src/include/duckdb/execution/row_id_deduplicator.hpp b/src/duckdb/src/include/duckdb/execution/row_id_deduplicator.hpp index 1fe22bb48..e7bcd60db 100644 --- a/src/duckdb/src/include/duckdb/execution/row_id_deduplicator.hpp +++ b/src/duckdb/src/include/duckdb/execution/row_id_deduplicator.hpp @@ -9,34 +9,33 @@ #pragma once #include "duckdb/common/optional_ptr.hpp" -#include "duckdb/common/types.hpp" -#include "duckdb/common/types/vector.hpp" -#include "duckdb/common/unordered_set.hpp" -#include "duckdb/common/vector/vector_iterator.hpp" +#include "duckdb/common/types/data_chunk.hpp" namespace duckdb { -//! Registers the first `count` row-ids of `row_ids` into `seen`, handling any vector layout. When `sel` is -//! provided, records the input position of each first-seen row-id (keep-first deduplication). Returns the number -//! of distinct (first-seen) row-ids. The caller owns its locking and duplicate policy: keep-first by slicing with -//! `sel`, or raise its own error when the returned count is less than `count`. Shared by UPDATE, MERGE, and -//! ON CONFLICT DO UPDATE, which otherwise duplicated this row-id iteration. -inline idx_t RegisterRowIds(unordered_set &seen, const Vector &row_ids, idx_t count, - optional_ptr sel = nullptr) { - idx_t distinct_count = 0; - for (const auto &entry : row_ids.Values()) { - // the caller may pass fewer meaningful entries than the vector holds (e.g. conflict row-ids) - if (entry.GetIndex() >= count) { - break; - } - if (seen.insert(entry.GetValue()).second) { - if (sel) { - sel->set_index(distinct_count, entry.GetIndex()); - } - distinct_count++; - } - } - return distinct_count; -} +class ClientContext; +class GroupedAggregateHashTable; + +//! Registers row IDs across chunks. The caller owns locking and decides whether duplicates are kept or rejected. +class RowIdDeduplicator { +public: + RowIdDeduplicator(ClientContext &context, vector row_id_types); + ~RowIdDeduplicator(); + + //! Registers the trailing row-ID columns in input, starting at row_id_start. + idx_t Register(DataChunk &input, idx_t row_id_start, optional_ptr sel = nullptr); + //! Registers the first count entries of a single-column row-ID vector. + idx_t Register(const Vector &row_ids, idx_t count, optional_ptr sel = nullptr); + +private: + idx_t Register(DataChunk &row_ids, optional_ptr sel); + +private: + vector row_id_types; + unique_ptr hash_table; + Vector addresses; + SelectionVector new_groups; + DataChunk row_id_chunk; +}; } // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp b/src/duckdb/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp index 67d1d5b98..04e82c735 100644 --- a/src/duckdb/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp +++ b/src/duckdb/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp @@ -3,6 +3,7 @@ #include "duckdb/common/common.hpp" #include "duckdb/storage/arena_allocator.hpp" #include "duckdb/common/algorithm.hpp" +#include "duckdb/common/operator/comparison_operators.hpp" #include "duckdb/common/pair.hpp" #include "duckdb/common/types/string_type.hpp" #include "duckdb/common/types/vector.hpp" @@ -386,7 +387,7 @@ struct ValueOrNull { bool operator>(const ValueOrNull &other) const { if (is_valid && other.is_valid) { - return value > other.value; + return GreaterThan::Operation(value, other.value); } if (!is_valid && !other.is_valid) { return false; diff --git a/src/duckdb/src/include/duckdb/main/client_context.hpp b/src/duckdb/src/include/duckdb/main/client_context.hpp index eb10643dd..631921f3d 100644 --- a/src/duckdb/src/include/duckdb/main/client_context.hpp +++ b/src/duckdb/src/include/duckdb/main/client_context.hpp @@ -172,6 +172,14 @@ class ClientContext : public enable_shared_from_this { PendingQuery(const string &query, identifier_map_t &values, QueryParameters query_parameters); DUCKDB_API unique_ptr PendingQuery(const string &query, PendingQueryParameters parameters); + //! Run a statement that was generated internally rather than parsed from user SQL. Statement verification + //! is skipped, and the client context lock is held for the entire duration of the query. + DUCKDB_API unique_ptr RunInternalStatement(unique_ptr statement, + const PendingQueryParameters ¶meters); + //! Same as RunInternalStatement, but returns a pending query result that the caller drives + DUCKDB_API unique_ptr PendingInternalStatement(unique_ptr statement, + const PendingQueryParameters ¶meters); + //! Destroy the client context DUCKDB_API void Destroy(); @@ -201,27 +209,12 @@ class ClientContext : public enable_shared_from_this { DUCKDB_API unique_ptr Prepare(const string &query); //! Directly prepare a SQL statement DUCKDB_API unique_ptr Prepare(unique_ptr statement); + //! Deallocate the prepared statement with the given name - does nothing if it does not exist + DUCKDB_API void RemovePreparedStatement(const string &name); //! Bind a statement and return its signature, without building a PreparedStatement, optimizing, or //! executing. Read-only: binding touches no in-flight query state, so a live result survives. Throws on error. DUCKDB_API StatementSignature BindStatement(unique_ptr statement); - //! Create a pending query result from a prepared statement with the given name and set of parameters - //! It is possible that the prepared statement will be re-bound. This will generally happen if the catalog is - //! modified in between the prepared statement being bound and the prepared statement being run. - DUCKDB_API unique_ptr PendingQuery(const string &query, - shared_ptr &prepared, - const PendingQueryParameters ¶meters); - - //! Execute a prepared statement with the given name and set of parameters - //! It is possible that the prepared statement will be re-bound. This will generally happen if the catalog is - //! modified in between the prepared statement being bound and the prepared statement being run. - DUCKDB_API unique_ptr - Execute(const string &query, shared_ptr &prepared, - identifier_map_t &values, - QueryParameters query_parameters = QueryResultOutputType::ALLOW_STREAMING); - DUCKDB_API unique_ptr Execute(const string &query, shared_ptr &prepared, - const PendingQueryParameters ¶meters); - //! Gets current percentage of the query's progress, returns 0 in case the progress bar is disabled. DUCKDB_API QueryProgress GetQueryProgress(); @@ -302,23 +295,18 @@ class ClientContext : public enable_shared_from_this { //! Internal clean up, does not lock. Caller must hold the context_lock. void CleanupInternal(ClientContextLock &lock, BaseQueryResult *result = nullptr, bool invalidate_transaction = false); - unique_ptr PendingStatementOrPreparedStatement(ClientContextLock &lock, const string &query, - unique_ptr statement, - shared_ptr &prepared, - const PendingQueryParameters ¶meters); - unique_ptr PendingPreparedStatement(ClientContextLock &lock, const string &query, - shared_ptr statement_p, - const PendingQueryParameters ¶meters); + unique_ptr PendingStatement(ClientContextLock &lock, const string &query, + unique_ptr statement, + const PendingQueryParameters ¶meters); unique_ptr PendingPreparedStatementInternal(ClientContextLock &lock, shared_ptr statement_data_p, const PendingQueryParameters ¶meters); void CheckIfPreparedStatementIsExecutable(PreparedStatementData &statement); //! Internally prepare a SQL statement. Caller must hold the context_lock. - shared_ptr - CreatePreparedStatement(ClientContextLock &lock, const string &query, unique_ptr statement, - PendingQueryParameters parameters, - PreparedStatementMode mode = PreparedStatementMode::PREPARE_ONLY); + shared_ptr CreatePreparedStatement(ClientContextLock &lock, const string &query, + unique_ptr statement, + PendingQueryParameters parameters); unique_ptr PendingStatementInternal(ClientContextLock &lock, const string &query, unique_ptr statement, const PendingQueryParameters ¶meters); @@ -340,20 +328,9 @@ class ClientContext : public enable_shared_from_this { void WaitForTask(ClientContextLock &lock, BaseQueryResult &result); PendingExecutionResult ExecuteTaskInternal(ClientContextLock &lock, BaseQueryResult &result, bool dry_run = false); - unique_ptr PendingStatementOrPreparedStatementInternal( - ClientContextLock &lock, const string &query, unique_ptr statement, - shared_ptr &prepared, const PendingQueryParameters ¶meters); - - unique_ptr PendingQueryPreparedInternal(ClientContextLock &lock, const string &query, - shared_ptr &prepared, - const PendingQueryParameters ¶meters); - unique_ptr PendingQueryInternal(ClientContextLock &, const shared_ptr &relation, QueryParameters query_parameters); - void RebindPreparedStatement(ClientContextLock &lock, const string &query, - shared_ptr &prepared, const PendingQueryParameters ¶meters); - template unique_ptr ErrorResult(ErrorData error, const string &query = string()); diff --git a/src/duckdb/src/include/duckdb/main/client_context_state.hpp b/src/duckdb/src/include/duckdb/main/client_context_state.hpp index 754493486..be459bf95 100644 --- a/src/duckdb/src/include/duckdb/main/client_context_state.hpp +++ b/src/duckdb/src/include/duckdb/main/client_context_state.hpp @@ -26,15 +26,6 @@ class RegisteredStateManager; enum class RebindQueryInfo { DO_NOT_REBIND, ATTEMPT_TO_REBIND }; -struct PreparedStatementCallbackInfo { - PreparedStatementCallbackInfo(PreparedStatementData &prepared_statement, const PendingQueryParameters ¶meters) - : prepared_statement(prepared_statement), parameters(parameters) { - } - - PreparedStatementData &prepared_statement; - const PendingQueryParameters ¶meters; -}; - struct BindPreparedStatementCallbackInfo { PreparedStatementData &prepared_statement; optional_ptr> parameters; @@ -75,10 +66,6 @@ class ClientContextState { PreparedStatementMode mode) { return RebindQueryInfo::DO_NOT_REBIND; } - virtual RebindQueryInfo OnExecutePrepared(ClientContext &context, PreparedStatementCallbackInfo &info, - RebindQueryInfo current_rebind) { - return RebindQueryInfo::DO_NOT_REBIND; - } virtual RebindQueryInfo OnRebindPreparedStatement(ClientContext &context, BindPreparedStatementCallbackInfo &info, RebindQueryInfo current_rebind) { return RebindQueryInfo::DO_NOT_REBIND; diff --git a/src/duckdb/src/include/duckdb/main/extension_entries.hpp b/src/duckdb/src/include/duckdb/main/extension_entries.hpp index db0470dba..58787140b 100644 --- a/src/duckdb/src/include/duckdb/main/extension_entries.hpp +++ b/src/duckdb/src/include/duckdb/main/extension_entries.hpp @@ -1151,9 +1151,13 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"iceberg_use_metadata_log", "iceberg"}, {"iceberg_via_aws_sdk_for_catalog_interactions", "iceberg"}, {"merge_http_secret_into_s3_request", "httpfs"}, + {"mysql_adaptive_replan_enabled", "mysql_scanner"}, + {"mysql_aggregate_pushdown_enabled", "mysql_scanner"}, + {"mysql_allow_results_streaming", "mysql_scanner"}, {"mysql_bit1_as_boolean", "mysql_scanner"}, {"mysql_debug_show_queries", "mysql_scanner"}, {"mysql_enable_filter_pushdown", "mysql_scanner"}, + {"mysql_enable_predicate_analyzer", "mysql_scanner"}, {"mysql_enable_transactions", "mysql_scanner"}, {"mysql_incomplete_dates_as_nulls", "mysql_scanner"}, {"mysql_pool_acquire_mode", "mysql_scanner"}, diff --git a/src/duckdb/src/include/duckdb/main/prepared_statement.hpp b/src/duckdb/src/include/duckdb/main/prepared_statement.hpp index b4d04d346..7f2bee90c 100644 --- a/src/duckdb/src/include/duckdb/main/prepared_statement.hpp +++ b/src/duckdb/src/include/duckdb/main/prepared_statement.hpp @@ -20,32 +20,51 @@ namespace duckdb { class ClientContext; -class PreparedStatementData; +class SQLStatement; + +//! The metadata of a prepared statement, as it was prepared +struct PreparedStatementInfo { + //! The result names of the prepared statement + vector names; + //! The result types of the prepared statement + vector types; + //! The type of the statement that was prepared + StatementType statement_type = StatementType::INVALID_STATEMENT; + //! The properties of the statement that was prepared + StatementProperties properties; + //! The mapping of parameter identifier to parameter index + identifier_map_t named_param_map; + //! The expected type of each parameter whose type could be resolved when preparing + identifier_map_t parameter_types; +}; -//! A prepared statement +//! A handle to a prepared statement that lives in the client context. The statement itself is prepared through +//! `PREPARE AS ...` and executed through `EXECUTE (...)` - this class only holds the name. class PreparedStatement { public: - //! Create a successfully prepared prepared statement object with the given name - DUCKDB_API PreparedStatement(shared_ptr context, shared_ptr data, - string query, identifier_map_t named_param_map); + //! Create a handle to the prepared statement with the given name in the client context + DUCKDB_API PreparedStatement(const shared_ptr &context, string name, string query, + PreparedStatementInfo info); //! Create a prepared statement that was not successfully prepared DUCKDB_API explicit PreparedStatement(ErrorData error); DUCKDB_API ~PreparedStatement(); + //! Destroying this object deallocates the prepared statement - so it cannot be copied + PreparedStatement(const PreparedStatement &) = delete; + PreparedStatement &operator=(const PreparedStatement &) = delete; + public: //! The client context this prepared statement belongs to - shared_ptr context; - //! The prepared statement data - shared_ptr data; + weak_ptr context; + //! The name of the prepared statement within the client context + string name; //! The query that is being prepared string query; //! Whether or not the statement was successfully prepared bool success; //! The error message (if success = false) ErrorData error; - //! The parameter mapping - identifier_map_t named_param_map; public: //! Returns the stored error message @@ -54,16 +73,24 @@ class PreparedStatement { DUCKDB_API ErrorData &GetErrorObject(); //! Returns whether or not an error occurred DUCKDB_API bool HasError() const; + //! Returns the client context this statement was prepared in - or nullptr if it has been destroyed + DUCKDB_API shared_ptr TryGetContext() const; //! Returns the number of columns in the result - DUCKDB_API idx_t ColumnCount(); + DUCKDB_API idx_t ColumnCount() const; //! Returns the statement type of the underlying prepared statement object - DUCKDB_API StatementType GetStatementType(); + DUCKDB_API StatementType GetStatementType() const; //! Returns the underlying statement properties - DUCKDB_API StatementProperties GetStatementProperties(); + DUCKDB_API const StatementProperties &GetStatementProperties() const; //! Returns the result SQL types of the prepared statement - DUCKDB_API const vector &GetTypes(); + DUCKDB_API const vector &GetTypes() const; //! Returns the result names of the prepared statement - DUCKDB_API const vector &GetNames(); + DUCKDB_API const vector &GetNames() const; + //! Returns the mapping of parameter identifier to parameter index + DUCKDB_API const identifier_map_t &GetNamedParameterMap() const; + //! Returns the number of parameters of the prepared statement + DUCKDB_API idx_t GetParameterCount() const; + //! Try to get the expected type of the parameter with the given identifier + DUCKDB_API bool TryGetParameterType(const Identifier &identifier, LogicalType &result) const; //! Returns the map of parameter index to the expected type of parameter DUCKDB_API case_insensitive_map_t GetExpectedParameterTypes() const; @@ -168,10 +195,14 @@ class PreparedStatement { } } - //! Returns whether or not we can / want to cache a logical plan - static bool CanCachePlan(const LogicalOperator &op); +private: + //! The metadata of the statement, as it was prepared + PreparedStatementInfo info; private: + //! Create the `EXECUTE (...)` statement that runs this prepared statement with the given values + unique_ptr CreateExecuteStatement(const identifier_map_t &named_values) const; + unique_ptr PendingQueryRecursive(vector &values) { return PendingQuery(values); } diff --git a/src/duckdb/src/include/duckdb/optimizer/topn_window_elimination.hpp b/src/duckdb/src/include/duckdb/optimizer/topn_window_elimination.hpp index 07adbabf6..f18bafe90 100644 --- a/src/duckdb/src/include/duckdb/optimizer/topn_window_elimination.hpp +++ b/src/duckdb/src/include/duckdb/optimizer/topn_window_elimination.hpp @@ -29,6 +29,8 @@ struct TopNWindowEliminationParameters { bool include_row_number; //! Whether the val or arg column contains null values bool can_be_null = false; + //! The number of semantic window partitions + idx_t partition_count = 0; }; class TopNWindowElimination : public BaseColumnPruner { @@ -62,7 +64,8 @@ class TopNWindowElimination : public BaseColumnPruner { unique_ptr UpdateTopmostBindings(TableIndex window_idx, unique_ptr op, const vector &types, const map &group_idxs, const vector &topmost_bindings, - vector &new_bindings, ColumnBindingReplacer &replacer); + vector &new_bindings, ColumnBindingReplacer &replacer, + const TopNWindowEliminationParameters ¶ms); TopNWindowEliminationParameters ExtractOptimizerParameters(const LogicalWindow &window, const LogicalFilter &filter, const vector &bindings, vector> &aggregate_payload); diff --git a/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp b/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp index 9971aa290..d37922210 100644 --- a/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp +++ b/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp @@ -10,6 +10,7 @@ #include "duckdb/parser/parsed_expression.hpp" #include "duckdb/parser/sql_statement.hpp" +#include "duckdb/planner/expression/bound_parameter_data.hpp" namespace duckdb { @@ -22,6 +23,9 @@ class ExecuteStatement : public SQLStatement { Identifier name; identifier_map_t> named_values; + //! Parameter values that are already typed - set when executing a prepared statement through the C/C++ API + //! instead of through SQL, where the values would have to be bound as literals first + identifier_map_t bound_values; protected: ExecuteStatement(const ExecuteStatement &other); diff --git a/src/duckdb/src/include/duckdb/planner/binder.hpp b/src/duckdb/src/include/duckdb/planner/binder.hpp index 63be977c0..bd3252bfd 100644 --- a/src/duckdb/src/include/duckdb/planner/binder.hpp +++ b/src/duckdb/src/include/duckdb/planner/binder.hpp @@ -626,7 +626,8 @@ class Binder : public enable_shared_from_this { Identifier BindCatalog(const Identifier &catalog_name); SchemaCatalogEntry &BindCreateSchema(CreateInfo &info); - vector GetSearchPath(Catalog &catalog, const Identifier &schema_name); + vector GetSearchPath(Catalog &catalog, const Identifier &schema_name, + bool default_schema_precedence = false); LogicalType BindLogicalTypeInternal(const unique_ptr &type_expr); diff --git a/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp b/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp index 049aca822..0eec0b559 100644 --- a/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp +++ b/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp @@ -36,8 +36,6 @@ struct BoundCreateTableInfo { ColumnDependencyManager column_dependency_manager; //! List of constraints on the table vector> constraints; - //! Dependents of the table (in e.g. default values) - LogicalDependencyList dependencies; //! The existing table data on disk (if any) unique_ptr data; //! CREATE TABLE from QUERY diff --git a/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp b/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp index 693c7848b..a1f9ee015 100644 --- a/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp +++ b/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp @@ -16,8 +16,6 @@ #include "duckdb/storage/compression/patas/patas.hpp" #include "duckdb/storage/table/column_data.hpp" -#include - namespace duckdb { template @@ -66,8 +64,19 @@ struct AlpAnalyzeState : public AnalyzeState { template unique_ptr AlpInitAnalyze(ColumnData &col_data, PhysicalType type) { - auto state = make_uniq>(col_data.GetBlockManager()); - state->storage_version = col_data.GetStorageManager().GetStorageVersion(); + auto &storage_manager = col_data.GetStorageManager(); + auto &block_manager = col_data.GetBlockManager(); + const auto storage_version = storage_manager.GetStorageVersion(); + + if (block_manager.GetBlockSize() + block_manager.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) { + if (StorageManager::IsPriorToVersion(StorageVersion::V1_5_0, storage_version)) { + // Before v1.5.0, blocks cannot use uncompressed-vector fallback + return nullptr; + } + } + + auto state = make_uniq>(block_manager); + state->storage_version = storage_version; return unique_ptr(std::move(state)); } @@ -76,10 +85,6 @@ unique_ptr AlpInitAnalyze(ColumnData &col_data, PhysicalType type) */ template bool AlpAnalyze(AnalyzeState &state, const Vector &input) { - if (state.info.GetBlockSize() + state.info.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) { - return false; - } - auto &analyze_state = state.Cast>(); const auto count = input.size(); @@ -146,7 +151,7 @@ bool AlpAnalyze(AnalyzeState &state, const Vector &input) { */ template idx_t AlpFinalAnalyze(AnalyzeState &state) { - auto &analyze_state = (AlpAnalyzeState &)state; + auto &analyze_state = state.Cast>(); // Finding the Top K combinations of Exponent and Factor alp::AlpCompression::FindTopKCombinations(analyze_state.rowgroup_sample, analyze_state.compression_data); diff --git a/src/duckdb/src/include/duckdb/storage/compression/alp/alp_scan.hpp b/src/duckdb/src/include/duckdb/storage/compression/alp/alp_scan.hpp index 6da67fdae..b92880f5a 100644 --- a/src/duckdb/src/include/duckdb/storage/compression/alp/alp_scan.hpp +++ b/src/duckdb/src/include/duckdb/storage/compression/alp/alp_scan.hpp @@ -218,6 +218,14 @@ struct AlpScanState : public SegmentScanState { memcpy(vector_state.exceptions_positions, (void *)vector_ptr, exceptions_positions_copy_size); vector_ptr += exceptions_positions_copy_size; read_bytes += exceptions_positions_copy_size; + + //! The exception positions index into the decoded vector, so they must stay within its bounds + for (idx_t i = 0; i < vector_state.exceptions_count; i++) { + if (vector_state.exceptions_positions[i] >= vector_size) { + throw IOException("Corrupted ALP segment: exception position (%d) exceeds vector_size (%d)", + vector_state.exceptions_positions[i], vector_size); + } + } } // Decode all the vector values to the specified 'value_buffer' diff --git a/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp b/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp index 730cbf203..5db691658 100644 --- a/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp +++ b/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp @@ -10,6 +10,7 @@ #include "duckdb/common/numeric_utils.hpp" #include "duckdb/function/compression_function.hpp" +#include "duckdb/storage/storage_manager.hpp" #include "duckdb/storage/compression/alp/alp_constants.hpp" #include "duckdb/storage/compression/alp/alp_utils.hpp" #include "duckdb/storage/compression/alprd/algorithm/alprd.hpp" @@ -38,7 +39,17 @@ struct AlpRDAnalyzeState : public AnalyzeState { template unique_ptr AlpRDInitAnalyze(ColumnData &col_data, PhysicalType type) { - return make_uniq>(col_data.GetBlockManager()); + auto &storage_manager = col_data.GetStorageManager(); + auto &block_manager = col_data.GetBlockManager(); + + if (block_manager.GetBlockSize() + block_manager.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) { + if (StorageManager::IsPriorToVersion(StorageVersion::V1_5_0, storage_manager.GetStorageVersion())) { + // Before v1.5.0, blocks cannot use uncompressed-vector fallback + return nullptr; + } + } + + return make_uniq>(block_manager); } /* @@ -46,10 +57,6 @@ unique_ptr AlpRDInitAnalyze(ColumnData &col_data, PhysicalType typ */ template bool AlpRDAnalyze(AnalyzeState &state, const Vector &input) { - if (state.info.GetBlockSize() + state.info.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) { - return false; - } - using EXACT_TYPE = typename FloatingToExact::TYPE; auto &analyze_state = state.Cast>(); diff --git a/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp b/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp index 90af37523..9d143f4f8 100644 --- a/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp +++ b/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp @@ -246,6 +246,14 @@ struct AlpRDScanState : public SegmentScanState { memcpy(vector_state.exceptions_positions, (void *)vector_ptr, exceptions_positions_copy_size); vector_ptr += exceptions_positions_copy_size; read_bytes += exceptions_positions_copy_size; + + //! The exception positions index into the decoded vector, so they must stay within its bounds + for (idx_t i = 0; i < vector_state.exceptions_count; i++) { + if (vector_state.exceptions_positions[i] >= vector_size) { + throw IOException("Corrupted ALPRD segment: exception position (%d) exceeds vector_size (%d)", + vector_state.exceptions_positions[i], vector_size); + } + } } // Decode all the vector values to the specified 'value_buffer' diff --git a/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/analyze.hpp b/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/analyze.hpp index 69d2c1ea6..0a1825c9b 100644 --- a/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/analyze.hpp +++ b/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/analyze.hpp @@ -21,6 +21,12 @@ struct DictFSSTAnalyzeState : public AnalyzeState { public: idx_t max_string_length = 0; bool contains_nulls = false; + //! Effective exclusive size limit for plain dictionary encoding. + idx_t string_size_limit = 0; + //! Effective exclusive input size limit for worst-case FSST encoding. + idx_t fsst_string_size_limit = 0; + //! Flag which disables the usage of FSST if worst-case encoding blowup does not fit the block size. + bool disable_fsst = false; idx_t total_string_length = 0; idx_t total_count = 0; }; diff --git a/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/common.hpp b/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/common.hpp index 46af3ced9..0531010ec 100644 --- a/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/common.hpp +++ b/src/duckdb/src/include/duckdb/storage/compression/dict_fsst/common.hpp @@ -1,9 +1,9 @@ #pragma once #include "duckdb/common/typedefs.hpp" -#include "duckdb/function/compression_function.hpp" #include "duckdb/common/bitpacking.hpp" #include "duckdb/storage/string_uncompressed.hpp" +#include "fsst.h" namespace duckdb { @@ -35,6 +35,7 @@ enum class DictionaryAppendState : uint8_t { struct DictFSSTCompression { public: + static constexpr uint16_t FSST_SYMBOL_TABLE_SIZE = sizeof(duckdb_fsst_decoder_t); //! Dictionary header size at the beginning of the string segment (offset + length) static constexpr uint16_t DICTIONARY_HEADER_SIZE = sizeof(dict_fsst_compression_header_t); static constexpr idx_t STRING_SIZE_LIMIT = 16384; diff --git a/src/duckdb/src/include/duckdb/storage/storage_info.hpp b/src/duckdb/src/include/duckdb/storage/storage_info.hpp index 02d6abc04..b6f0d9fd2 100644 --- a/src/duckdb/src/include/duckdb/storage/storage_info.hpp +++ b/src/duckdb/src/include/duckdb/storage/storage_info.hpp @@ -136,10 +136,13 @@ enum class StorageVersion : uint64_t { V1_4_2 = 67, V1_4_3 = 67, V1_4_4 = 67, + V1_4_5 = 67, V1_5_0 = 68, V1_5_1 = 68, V1_5_2 = 68, V1_5_3 = 68, + V1_5_4 = 68, + V1_5_5 = 68, V2_0_0 = 69, LATEST = 69, DEPRECATED = 999, @@ -169,10 +172,13 @@ enum class SerializationVersionDeprecated : uint64_t { V1_4_2 = 6, V1_4_3 = 6, V1_4_4 = 6, + V1_4_5 = 6, V1_5_0 = 7, V1_5_1 = 7, V1_5_2 = 7, V1_5_3 = 7, + V1_5_4 = 7, + V1_5_5 = 7, V2_0_0 = 8, LATEST = 8, INVALID = UINT64_MAX diff --git a/src/duckdb/src/include/duckdb/storage/table/per_column_metadata_blocks.hpp b/src/duckdb/src/include/duckdb/storage/table/per_column_metadata_blocks.hpp index b7fb99f16..3e2c8cc6b 100644 --- a/src/duckdb/src/include/duckdb/storage/table/per_column_metadata_blocks.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/per_column_metadata_blocks.hpp @@ -32,7 +32,11 @@ class PerColumnMetadataBlocks { //! Add a column entry with its block IDs void AddColumn(idx_t col_idx, const vector &blocks); - //! Remove a column entry and all its block IDs (linear scan) + //! Clear a column's entry and all its block IDs in place, leaving the indices of the other + //! columns untouched (linear scan), for use when the column is rewritten (e.g. ALTER TYPE) + void ClearColumn(idx_t col_idx); + //! Remove a column's entry and shift down the indices of all subsequent columns, + //! for use when the column is positionally removed (e.g. DROP COLUMN) void RemoveColumn(idx_t col_idx); //! Merge two PerColumnMetadataBlocks sorted by column index with disjoint column sets static PerColumnMetadataBlocks Merge(const PerColumnMetadataBlocks &a, const PerColumnMetadataBlocks &b); diff --git a/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp b/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp index 13b22588d..ee4bbca88 100644 --- a/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp @@ -252,6 +252,8 @@ class RowGroupCollection { vector metadata_pointers; //! Controls whether the next append creates a new row group or reuses the existing one RowGroupAppendMode row_group_append_mode; + //! Whether or not we can append to a checkpointed row group + bool can_append_to_checkpointed_row_group = true; }; class RowGroupIterationHelper { diff --git a/src/duckdb/src/main/capi/arrow-c.cpp b/src/duckdb/src/main/capi/arrow-c.cpp index 93565561d..fdd1bf737 100644 --- a/src/duckdb/src/main/capi/arrow-c.cpp +++ b/src/duckdb/src/main/capi/arrow-c.cpp @@ -2,7 +2,6 @@ #include "duckdb/common/arrow/arrow_converter.hpp" #include "duckdb/function/table/arrow.hpp" #include "duckdb/main/capi/capi_internal.hpp" -#include "duckdb/main/prepared_statement_data.hpp" #include "fmt/format.h" using duckdb::ArrowConverter; @@ -191,14 +190,18 @@ duckdb_state duckdb_prepared_arrow_schema(duckdb_prepared_statement prepared, du return DuckDBSuccess; } auto wrapper = reinterpret_cast(prepared); - if (!wrapper || !wrapper->statement || !wrapper->statement->data) { + if (!wrapper || !wrapper->statement) { return DuckDBError; } - auto properties = wrapper->statement->context->GetClientProperties(); + auto context = wrapper->statement->TryGetContext(); + if (!context) { + return DuckDBError; + } + auto properties = context->GetClientProperties(); duckdb::vector prepared_types; duckdb::vector prepared_names; - auto count = wrapper->statement->data->properties.parameter_count; + auto count = wrapper->statement->GetParameterCount(); for (idx_t i = 0; i < count; i++) { // Every prepared parameter type is UNKNOWN, which we need to map to NULL according to the spec of // 'AdbcStatementGetParameterSchema' diff --git a/src/duckdb/src/main/capi/helper-c.cpp b/src/duckdb/src/main/capi/helper-c.cpp index 5ea57636e..7fb8ff627 100644 --- a/src/duckdb/src/main/capi/helper-c.cpp +++ b/src/duckdb/src/main/capi/helper-c.cpp @@ -310,6 +310,12 @@ duckdb_statement_type StatementTypeToC(const StatementType type) { return DUCKDB_STATEMENT_TYPE_DETACH; case StatementType::MULTI_STATEMENT: return DUCKDB_STATEMENT_TYPE_MULTI; + case StatementType::COPY_DATABASE_STATEMENT: + return DUCKDB_STATEMENT_TYPE_COPY_DATABASE; + case StatementType::UPDATE_EXTENSIONS_STATEMENT: + return DUCKDB_STATEMENT_TYPE_UPDATE_EXTENSIONS; + case StatementType::MERGE_INTO_STATEMENT: + return DUCKDB_STATEMENT_TYPE_MERGE_INTO; default: return DUCKDB_STATEMENT_TYPE_INVALID; } diff --git a/src/duckdb/src/main/capi/prepared-c.cpp b/src/duckdb/src/main/capi/prepared-c.cpp index 2f194401e..852d66a0f 100644 --- a/src/duckdb/src/main/capi/prepared-c.cpp +++ b/src/duckdb/src/main/capi/prepared-c.cpp @@ -1,6 +1,5 @@ #include "duckdb/main/capi/capi_internal.hpp" #include "duckdb/main/query_result.hpp" -#include "duckdb/main/prepared_statement_data.hpp" #include "duckdb/common/types/decimal.hpp" #include "duckdb/common/uhugeint.hpp" #include "duckdb/common/optional_ptr.hpp" @@ -42,9 +41,8 @@ idx_t duckdb_extract_statements(duckdb_connection connection, const char *query, } static void duckdb_prepare_param_index_to_name_map_internal(PreparedStatementWrapper *wrapper) { - auto &named_param_map = wrapper->statement->named_param_map; auto &cache = wrapper->param_index_to_name; - for (auto &kv : named_param_map) { + for (auto &kv : wrapper->statement->GetNamedParameterMap()) { cache[kv.second] = kv.first.GetIdentifierName(); } } @@ -121,7 +119,7 @@ idx_t duckdb_nparams(duckdb_prepared_statement prepared_statement) { if (!wrapper || !wrapper->statement || wrapper->statement->HasError()) { return 0; } - return wrapper->statement->named_param_map.size(); + return wrapper->param_index_to_name.size(); } static duckdb::string duckdb_parameter_name_internal(duckdb_prepared_statement prepared_statement, idx_t index) { @@ -170,7 +168,7 @@ duckdb_logical_type duckdb_param_logical_type(duckdb_prepared_statement prepared LogicalType param_type; - if (wrapper->statement->data->TryGetType(duckdb::Identifier(identifier), param_type)) { + if (wrapper->statement->TryGetParameterType(duckdb::Identifier(identifier), param_type)) { return reinterpret_cast(new LogicalType(param_type)); } // The value_map is gone after executing the prepared statement @@ -205,7 +203,6 @@ const char *duckdb_prepared_statement_column_name(duckdb_prepared_statement prep return nullptr; } auto &names = wrapper->statement->GetNames(); - if (col_idx >= names.size()) { return nullptr; } @@ -218,7 +215,7 @@ duckdb_logical_type duckdb_prepared_statement_column_logical_type(duckdb_prepare if (!wrapper || !wrapper->statement || wrapper->statement->HasError()) { return nullptr; } - auto types = wrapper->statement->GetTypes(); + auto &types = wrapper->statement->GetTypes(); if (col_idx >= types.size()) { return nullptr; } @@ -243,10 +240,10 @@ duckdb_state duckdb_bind_value(duckdb_prepared_statement prepared_statement, idx if (!wrapper || !wrapper->statement || wrapper->statement->HasError()) { return DuckDBError; } - if (param_idx <= 0 || param_idx > wrapper->statement->named_param_map.size()) { + if (param_idx <= 0 || param_idx > wrapper->param_index_to_name.size()) { wrapper->error_data = duckdb::InvalidInputException("Can not bind to parameter number %d, statement only has %d parameter(s)", - param_idx, wrapper->statement->named_param_map.size()); + param_idx, wrapper->param_index_to_name.size()); wrapper->success = false; return DuckDBError; } @@ -265,9 +262,9 @@ duckdb_state duckdb_bind_parameter_index(duckdb_prepared_statement prepared_stat return DuckDBError; } auto name = std::string(name_p); - for (auto &pair : wrapper->statement->named_param_map) { - if (pair.first == name) { - *param_idx_out = pair.second; + for (auto &pair : wrapper->param_index_to_name) { + if (StringUtil::CIEquals(pair.second, name)) { + *param_idx_out = pair.first; return DuckDBSuccess; } } @@ -462,7 +459,9 @@ duckdb_statement_type duckdb_prepared_statement_type(duckdb_prepared_statement s return DUCKDB_STATEMENT_TYPE_INVALID; } auto stmt = reinterpret_cast(statement); - + if (!stmt->statement) { + return DUCKDB_STATEMENT_TYPE_INVALID; + } return StatementTypeToC(stmt->statement->GetStatementType()); } diff --git a/src/duckdb/src/main/capi/scalar_function-c.cpp b/src/duckdb/src/main/capi/scalar_function-c.cpp index 07011aaee..2003b0c73 100644 --- a/src/duckdb/src/main/capi/scalar_function-c.cpp +++ b/src/duckdb/src/main/capi/scalar_function-c.cpp @@ -352,9 +352,14 @@ duckdb_expression duckdb_scalar_function_bind_get_argument(duckdb_bind_info info return nullptr; } auto &bind_info = GetCScalarFunctionBindInfo(info); - auto wrapper = new ExpressionWrapper(); - wrapper->expr = bind_info.arguments[index]->Copy(); - return reinterpret_cast(wrapper); + try { + auto wrapper = duckdb::make_uniq(); + wrapper->expr = bind_info.arguments[index]->Copy(); + return reinterpret_cast(wrapper.release()); + } catch (std::exception &e) { + duckdb_scalar_function_bind_set_error(info, e.what()); + return nullptr; + } } void duckdb_scalar_function_set_extra_info(duckdb_scalar_function function, void *extra_info, diff --git a/src/duckdb/src/main/client_context.cpp b/src/duckdb/src/main/client_context.cpp index 35332b2fa..1ca2ee84c 100644 --- a/src/duckdb/src/main/client_context.cpp +++ b/src/duckdb/src/main/client_context.cpp @@ -9,6 +9,7 @@ #include "duckdb/common/progress_bar/progress_bar.hpp" #include "duckdb/common/serializer/buffered_file_writer.hpp" #include "duckdb/common/types/column/column_data_collection.hpp" +#include "duckdb/common/types/uuid.hpp" #include "duckdb/execution/column_binding_resolver.hpp" #include "duckdb/execution/operator/helper/physical_result_collector.hpp" #include "duckdb/execution/physical_plan_generator.hpp" @@ -185,8 +186,8 @@ struct DebugClientContextState : public ClientContextState { } return RebindQueryInfo::DO_NOT_REBIND; } - RebindQueryInfo OnExecutePrepared(ClientContext &context, PreparedStatementCallbackInfo &info, - RebindQueryInfo current_rebind) override { + RebindQueryInfo OnRebindPreparedStatement(ClientContext &context, BindPreparedStatementCallbackInfo &info, + RebindQueryInfo current_rebind) override { return RebindQueryInfo::ATTEMPT_TO_REBIND; } #endif @@ -502,17 +503,6 @@ shared_ptr ClientContext::CreatePreparedStatementInternal #ifdef DEBUG logical_plan->Verify(*this); #endif - if (!result->value_map.empty() && !parameters.parameters) { - // if this is a prepared statement we can choose not to fully plan - // if we have parameters, we might want to re-bind when they are available as we can then do more optimizations - // in this situation we check if we want to cache the plan at all - if (!PreparedStatement::CanCachePlan(*logical_plan)) { - // we don't - early-out - result->properties.always_require_rebind = true; - return result; - } - } - bool optimize = Settings::Get(*this); if (Settings::Get(*this)) { // verify disable optimizer - disable EXCEPT for explain, otherwise every single EXPLAIN query breaks @@ -544,8 +534,7 @@ shared_ptr ClientContext::CreatePreparedStatementInternal shared_ptr ClientContext::CreatePreparedStatement(ClientContextLock &lock, const string &query, unique_ptr statement, - PendingQueryParameters parameters, - PreparedStatementMode mode) { + PendingQueryParameters parameters) { // check if any client context state could request a rebind bool can_request_rebind = false; for (auto &state : registered_state->States()) { @@ -575,7 +564,7 @@ shared_ptr ClientContext::CreatePreparedStatement(ClientC if (result) { D_ASSERT(!rebind); for (auto &state : registered_state->States()) { - auto info = state->OnFinalizePrepare(*this, *result, mode); + auto info = state->OnFinalizePrepare(*this, *result, PreparedStatementMode::PREPARE_AND_EXECUTE); if (info == RebindQueryInfo::ATTEMPT_TO_REBIND) { rebind = true; } @@ -606,21 +595,6 @@ void BindPreparedStatementParameters(ClientContext &context, PreparedStatementDa statement.Bind(context, owned_values); } -void ClientContext::RebindPreparedStatement(ClientContextLock &lock, const string &query, - shared_ptr &prepared, - const PendingQueryParameters ¶meters) { - if (!prepared->unbound_statement) { - throw InternalException("ClientContext::RebindPreparedStatement called but PreparedStatementData did not have " - "an unbound statement so rebinding cannot be done"); - } - // catalog was modified: rebind the statement before execution - auto new_prepared = CreatePreparedStatement(lock, query, prepared->unbound_statement->Copy(), parameters); - D_ASSERT(new_prepared->properties.bound_all_parameters); - new_prepared->properties.parameter_count = prepared->properties.parameter_count; - prepared = std::move(new_prepared); - prepared->properties.bound_all_parameters = false; -} - void ClientContext::CheckIfPreparedStatementIsExecutable(PreparedStatementData &statement) { if (ValidChecker::IsInvalidated(ActiveTransaction()) && statement.properties.requires_valid_transaction) { throw ErrorManager::InvalidatedTransaction(*this); @@ -698,30 +672,6 @@ ClientContext::PendingPreparedStatementInternal(ClientContextLock &lock, return pending_result; } -unique_ptr ClientContext::PendingPreparedStatement(ClientContextLock &lock, const string &query, - shared_ptr prepared, - const PendingQueryParameters ¶meters) { - CheckIfPreparedStatementIsExecutable(*prepared); - - RebindQueryInfo rebind = RebindQueryInfo::DO_NOT_REBIND; - if (prepared->RequireRebind(*this, parameters.parameters)) { - rebind = RebindQueryInfo::ATTEMPT_TO_REBIND; - } - - for (auto &state : registered_state->States()) { - PreparedStatementCallbackInfo info(*prepared, parameters); - auto new_rebind = state->OnExecutePrepared(*this, info, rebind); - if (new_rebind == RebindQueryInfo::ATTEMPT_TO_REBIND) { - rebind = RebindQueryInfo::ATTEMPT_TO_REBIND; - } - } - if (rebind == RebindQueryInfo::ATTEMPT_TO_REBIND) { - RebindPreparedStatement(lock, query, prepared, parameters); - CheckIfPreparedStatementIsExecutable(*prepared); // rerun this too as modified_databases might have changed - } - return PendingPreparedStatementInternal(lock, prepared, parameters); -} - void ClientContext::WaitForTask(ClientContextLock &lock, BaseQueryResult &result) { active_query->executor->WaitForTask(); } @@ -863,23 +813,53 @@ unique_ptr ClientContext::ExtractPlan(const string &query) { return plan; } +//! Snapshot the metadata that the PreparedStatement handle exposes to the user +static PreparedStatementInfo GetPreparedStatementInfo(PreparedStatementData &data) { + PreparedStatementInfo info; + info.names = data.names; + info.types = data.types; + info.statement_type = data.statement_type; + info.properties = data.properties; + if (data.unbound_statement) { + info.named_param_map = data.unbound_statement->named_param_map; + } + for (auto &entry : data.value_map) { + LogicalType parameter_type; + if (data.TryGetType(entry.first, parameter_type)) { + info.parameter_types.emplace(entry.first, std::move(parameter_type)); + } + } + return info; +} + unique_ptr ClientContext::PrepareInternal(ClientContextLock &lock, unique_ptr statement) { - auto named_param_map = statement->named_param_map; auto statement_query = statement->query; - shared_ptr prepared_data; - auto unbound_statement = statement->Copy(); + // prepare the statement under a generated name - the returned PreparedStatement only refers to that name + auto name = "duckdb_prepare_internal_" + UUID::ToString(UUID::GenerateRandomUUID()); + auto prepare = make_uniq(); + prepare->name = Identifier(name); + prepare->query = statement_query; + prepare->stmt_location = statement->stmt_location; + prepare->statement = std::move(statement); + PendingQueryParameters parameters; - RunFunctionInTransactionInternal( - lock, - [&]() { - ActiveQueryGuard guard(active_query, statement_query); - prepared_data = CreatePreparedStatement(lock, statement_query, std::move(statement), parameters); - }, - false); - prepared_data->unbound_statement = std::move(unbound_statement); - return make_uniq(shared_from_this(), std::move(prepared_data), std::move(statement_query), - std::move(named_param_map)); + parameters.query_parameters.output_type = QueryResultOutputType::FORCE_MATERIALIZED; + auto result = RunStatementInternal(lock, statement_query, std::move(prepare), parameters, false); + if (result->HasError()) { + result->ThrowError(); + } + auto entry = client_data->prepared_statements.find(Identifier(name)); + if (entry == client_data->prepared_statements.end()) { + throw InternalException("PREPARE succeeded but the prepared statement was not registered"); + } + return make_uniq(shared_from_this(), std::move(name), std::move(statement_query), + GetPreparedStatementInfo(*entry->second)); +} + +void ClientContext::RemovePreparedStatement(const string &name) { + auto lock = LockContext(); + client_data->prepared_statements.erase(Identifier(name)); } unique_ptr ClientContext::Prepare(unique_ptr statement) { @@ -960,43 +940,6 @@ unique_ptr ClientContext::Prepare(const string &query) { } } -unique_ptr ClientContext::PendingQueryPreparedInternal(ClientContextLock &lock, const string &query, - shared_ptr &prepared, - const PendingQueryParameters ¶meters) { - try { - InitialCleanup(lock); - } catch (std::exception &ex) { - return ErrorResult(ErrorData(ex), query); - } - return PendingStatementOrPreparedStatementInternal(lock, query, nullptr, prepared, parameters); -} - -unique_ptr ClientContext::PendingQuery(const string &query, - shared_ptr &prepared, - const PendingQueryParameters ¶meters) { - auto lock = LockContext(); - return PendingQueryPreparedInternal(*lock, query, prepared, parameters); -} - -unique_ptr ClientContext::Execute(const string &query, shared_ptr &prepared, - const PendingQueryParameters ¶meters) { - auto lock = LockContext(); - auto pending = PendingQueryPreparedInternal(*lock, query, prepared, parameters); - if (pending->HasError()) { - return ErrorResult(pending->GetErrorObject()); - } - return pending->ExecuteInternal(*lock); -} - -unique_ptr ClientContext::Execute(const string &query, shared_ptr &prepared, - identifier_map_t &values, - QueryParameters query_parameters) { - PendingQueryParameters parameters; - parameters.parameters = &values; - parameters.query_parameters = query_parameters; - return Execute(query, prepared, parameters); -} - unique_ptr ClientContext::PendingStatementInternal(ClientContextLock &lock, const string &query, unique_ptr statement, const PendingQueryParameters ¶meters) { @@ -1008,8 +951,7 @@ unique_ptr ClientContext::PendingStatementInternal(ClientCon PreparedStatement::VerifyParameters(empty_parameters, statement->named_param_map, this); } - auto prepared = CreatePreparedStatement(lock, query, std::move(statement), parameters, - PreparedStatementMode::PREPARE_AND_EXECUTE); + auto prepared = CreatePreparedStatement(lock, query, std::move(statement), parameters); if (!prepared->properties.bound_all_parameters) { return ErrorResult(InvalidInputException("Not all parameters were bound"), query); @@ -1036,46 +978,29 @@ bool ClientContext::IsActiveResult(ClientContextLock &lock, BaseQueryResult &res return active_query->IsOpenResult(result); } -unique_ptr ClientContext::PendingStatementOrPreparedStatementInternal( - ClientContextLock &lock, const string &query, unique_ptr statement, - shared_ptr &prepared, const PendingQueryParameters ¶meters) { - if (statement) { - try { - StatementVerification(lock, query, statement, parameters); - } catch (std::exception &ex) { - // preserve extra error data (like query location) - return ErrorResult(ErrorData(ex), query); - } +//! Whether this statement executes a prepared statement with parameter values supplied through the C/C++ API +static bool HasBoundParameterValues(const SQLStatement &statement) { + if (statement.type != StatementType::EXECUTE_STATEMENT) { + return false; } - return PendingStatementOrPreparedStatement(lock, query, std::move(statement), prepared, parameters); + return !statement.Cast().bound_values.empty(); } -unique_ptr ClientContext::PendingStatementOrPreparedStatement( - ClientContextLock &lock, const string &query, unique_ptr statement, - shared_ptr &prepared, const PendingQueryParameters ¶meters) { +unique_ptr ClientContext::PendingStatement(ClientContextLock &lock, const string &query, + unique_ptr statement, + const PendingQueryParameters ¶meters) { // CONNECT chokepoint: when connected, non-control SQL is rewritten in place and falls through to // the normal pipeline. No recursion — the rewrite goes through PendingStatementInternal, not back here. if (is_connected) { - bool is_control = false; - if (statement && IsConnectControlStatement(statement->type)) { - is_control = true; - } - if (!is_control) { - if (!statement) { - // Prepared-statement execution path (statement is null, prepared is set). Parameterized - // prepared statements would need parameter substitution we don't do in v0 — reject. - // No-param prepared statements have a fully-resolved `query` already, route them. - if (!prepared || prepared->properties.parameter_count > 0) { - return ErrorResult( - ErrorData(InvalidInputException( - "Parameterized prepared statements cannot be executed while CONNECT-ed; " - "DISCONNECT first, or run the SQL as a fresh statement to route through " - "the CONNECT binding")), - query); - } - // Clear prepared so the rest of the function takes the fresh-statement path on the - // rewritten SELECT below. - prepared.reset(); + if (!IsConnectControlStatement(statement->type)) { + // Parameterized prepared statements would need parameter substitution we don't do in v0 — reject. + // No-param prepared statements have a fully-resolved `query` already, route them. + if (HasBoundParameterValues(*statement)) { + return ErrorResult( + ErrorData(InvalidInputException("Parameterized prepared statements cannot be executed while " + "CONNECT-ed; DISCONNECT first, or run the SQL as a fresh " + "statement to route through the CONNECT binding")), + query); } auto live = TryGetConnectedCatalog(); if (!live) { @@ -1109,11 +1034,7 @@ unique_ptr ClientContext::PendingStatementOrPreparedStatemen bool invalidate_query = true; try { - if (statement) { - pending = PendingStatementInternal(lock, query, std::move(statement), parameters); - } else { - pending = PendingPreparedStatement(lock, query, prepared, parameters); - } + pending = PendingStatementInternal(lock, query, std::move(statement), parameters); } catch (std::exception &ex) { ErrorData error(ex); if (!ErrorInvalidatesTransaction(error.Type())) { @@ -1332,17 +1253,48 @@ unique_ptr ClientContext::PendingQuery(unique_ptr ClientContext::RunInternalStatement(unique_ptr statement, + const PendingQueryParameters ¶meters) { + auto lock = LockContext(); + auto query = statement->query; + try { + InitialCleanup(*lock); + } catch (std::exception &ex) { + return ErrorResult(ErrorData(ex), query); + } + auto pending = PendingQueryInternal(*lock, std::move(statement), parameters, false); + if (pending->HasError()) { + return ErrorResult(pending->GetErrorObject()); + } + return pending->ExecuteInternal(*lock); +} + +unique_ptr ClientContext::PendingInternalStatement(unique_ptr statement, + const PendingQueryParameters ¶meters) { + auto lock = LockContext(); + auto query = statement->query; + try { + InitialCleanup(*lock); + } catch (std::exception &ex) { + return ErrorResult(ErrorData(ex), query); + } + return PendingQueryInternal(*lock, std::move(statement), parameters, false); +} + unique_ptr ClientContext::PendingQueryInternal(ClientContextLock &lock, unique_ptr statement, const PendingQueryParameters ¶meters, bool verify) { auto query = statement->query; - shared_ptr prepared; if (verify) { - return PendingStatementOrPreparedStatementInternal(lock, query, std::move(statement), prepared, parameters); - } else { - return PendingStatementOrPreparedStatement(lock, query, std::move(statement), prepared, parameters); + try { + StatementVerification(lock, query, statement, parameters); + } catch (std::exception &ex) { + // preserve extra error data (like query location) + return ErrorResult(ErrorData(ex), query); + } } + return PendingStatement(lock, query, std::move(statement), parameters); } unique_ptr ClientContext::ExecutePendingQueryInternal(ClientContextLock &lock, PendingQueryResult &query) { diff --git a/src/duckdb/src/main/http/http_util.cpp b/src/duckdb/src/main/http/http_util.cpp index a098308f8..e05f47482 100644 --- a/src/duckdb/src/main/http/http_util.cpp +++ b/src/duckdb/src/main/http/http_util.cpp @@ -133,6 +133,14 @@ bool HTTPResponse::ShouldRetry() const { } } +bool HTTPUtil::IsIdempotent(RequestType type) { + return type != RequestType::POST_REQUEST; +} + +bool HTTPUtil::ShouldRetry(const BaseRequest &request, const HTTPResponse &response) { + return response.ShouldRetry(); +} + unique_ptr HTTPUtil::Request(BaseRequest &request) { unique_ptr client; return SendRequest(request, client); @@ -448,7 +456,7 @@ HTTPUtil::RunRequestWithRetry(const std::function(void) } // Note: request errors will always be retried - bool should_retry = !response || response->ShouldRetry(); + bool should_retry = !response || params.http_util.ShouldRetry(request, *response); if (!should_retry) { auto response_code = static_cast(response->status); if (response_code >= 200 && response_code < 300) { diff --git a/src/duckdb/src/main/prepared_statement.cpp b/src/duckdb/src/main/prepared_statement.cpp index 266ac3ce0..7415f4dec 100644 --- a/src/duckdb/src/main/prepared_statement.cpp +++ b/src/duckdb/src/main/prepared_statement.cpp @@ -1,23 +1,26 @@ #include "duckdb/main/prepared_statement.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/main/client_context.hpp" -#include "duckdb/main/prepared_statement_data.hpp" -#include "duckdb/common/enums/logical_operator_type.hpp" -#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/parser/statement/execute_statement.hpp" namespace duckdb { -PreparedStatement::PreparedStatement(shared_ptr context, shared_ptr data_p, - string query, identifier_map_t named_param_map_p) - : context(std::move(context)), data(std::move(data_p)), query(std::move(query)), success(true), - named_param_map(std::move(named_param_map_p)) { - D_ASSERT(data || !success); +PreparedStatement::PreparedStatement(const shared_ptr &context_p, string name_p, string query_p, + PreparedStatementInfo info_p) + : context(context_p), name(std::move(name_p)), query(std::move(query_p)), success(true), info(std::move(info_p)) { + D_ASSERT(!context.expired()); } -PreparedStatement::PreparedStatement(ErrorData error) : context(nullptr), success(false), error(std::move(error)) { +PreparedStatement::PreparedStatement(ErrorData error) : success(false), error(std::move(error)) { } PreparedStatement::~PreparedStatement() { + auto client_context = context.lock(); + if (!client_context) { + // the client context is gone - the prepared statement has been deallocated together with it + return; + } + client_context->RemovePreparedStatement(name); } const string &PreparedStatement::GetError() { @@ -33,67 +36,82 @@ bool PreparedStatement::HasError() const { return !success; } -idx_t PreparedStatement::ColumnCount() { - D_ASSERT(data); - return data->types.size(); +shared_ptr PreparedStatement::TryGetContext() const { + return context.lock(); +} + +idx_t PreparedStatement::ColumnCount() const { + return info.types.size(); } -StatementType PreparedStatement::GetStatementType() { - D_ASSERT(data); - return data->statement_type; +StatementType PreparedStatement::GetStatementType() const { + return info.statement_type; } -StatementProperties PreparedStatement::GetStatementProperties() { - D_ASSERT(data); - return data->properties; +const StatementProperties &PreparedStatement::GetStatementProperties() const { + return info.properties; } -const vector &PreparedStatement::GetTypes() { - D_ASSERT(data); - return data->types; +const vector &PreparedStatement::GetTypes() const { + return info.types; } -const vector &PreparedStatement::GetNames() { - D_ASSERT(data); - return data->names; +const vector &PreparedStatement::GetNames() const { + return info.names; +} + +const identifier_map_t &PreparedStatement::GetNamedParameterMap() const { + return info.named_param_map; +} + +idx_t PreparedStatement::GetParameterCount() const { + return info.named_param_map.size(); +} + +bool PreparedStatement::TryGetParameterType(const Identifier &identifier, LogicalType &result) const { + auto entry = info.parameter_types.find(identifier); + if (entry == info.parameter_types.end()) { + return false; + } + result = entry->second; + return true; } case_insensitive_map_t PreparedStatement::GetExpectedParameterTypes() const { - D_ASSERT(data); - case_insensitive_map_t expected_types(data->value_map.size()); - for (auto &it : data->value_map) { - auto &identifier = it.first; - D_ASSERT(data->value_map.count(identifier)); - D_ASSERT(it.second); - expected_types[identifier.GetIdentifierName()] = it.second->GetValue().type(); + case_insensitive_map_t expected_types(info.parameter_types.size()); + for (auto &entry : info.parameter_types) { + expected_types[entry.first.GetIdentifierName()] = entry.second; } return expected_types; } +unique_ptr +PreparedStatement::CreateExecuteStatement(const identifier_map_t &named_values) const { + auto execute = make_uniq(); + execute->name = Identifier(name); + // report the query that was prepared - not the generated EXECUTE - in errors and profiling output + execute->query = query; + execute->stmt_location = QueryLocation(0, query.size()); + // the values are already typed - pass them in pre-bound instead of as SQL literals + execute->bound_values = named_values; + return std::move(execute); +} + unique_ptr PreparedStatement::Execute(identifier_map_t &named_values, bool allow_stream_result) { if (!success) { return make_uniq( ErrorData(InvalidInputException("Attempting to execute an unsuccessfully prepared statement!"))); } - - try { - if (!named_param_map.empty()) { - VerifyParameters(named_values, named_param_map, context.get()); - } - } catch (const std::exception &ex) { - return make_uniq(ErrorData(ex)); + auto client_context = context.lock(); + if (!client_context) { + return make_uniq(ErrorData( + InvalidInputException("Attempting to execute a prepared statement after its connection was closed!"))); } - PendingQueryParameters parameters; - parameters.parameters = &named_values; - D_ASSERT(data); parameters.query_parameters.output_type = - allow_stream_result && data->properties.output_type == QueryResultOutputType::ALLOW_STREAMING - ? QueryResultOutputType::ALLOW_STREAMING - : QueryResultOutputType::FORCE_MATERIALIZED; - - return context->Execute(query, data, parameters); + allow_stream_result ? QueryResultOutputType::ALLOW_STREAMING : QueryResultOutputType::FORCE_MATERIALIZED; + return client_context->RunInternalStatement(CreateExecuteStatement(named_values), parameters); } unique_ptr PreparedStatement::Execute(vector &values, bool allow_stream_result) { @@ -119,46 +137,16 @@ unique_ptr PreparedStatement::PendingQuery(identifier_map_t< auto exception = InvalidInputException("Attempting to execute an unsuccessfully prepared statement!"); return make_uniq(ErrorData(exception)); } - PendingQueryParameters parameters; - parameters.parameters = &named_values; - - try { - if (!named_param_map.empty()) { - VerifyParameters(named_values, named_param_map, context.get()); - } - } catch (const std::exception &ex) { - return make_uniq(ErrorData(ex)); + auto client_context = context.lock(); + if (!client_context) { + auto exception = + InvalidInputException("Attempting to execute a prepared statement after its connection was closed!"); + return make_uniq(ErrorData(exception)); } - - D_ASSERT(data); + PendingQueryParameters parameters; parameters.query_parameters.output_type = - allow_stream_result && data->properties.output_type == QueryResultOutputType::ALLOW_STREAMING - ? QueryResultOutputType::ALLOW_STREAMING - : QueryResultOutputType::FORCE_MATERIALIZED; - auto result = context->PendingQuery(query, data, parameters); - // The result should not contain any reference to the 'vector parameters.parameters' - return result; -} - -bool PreparedStatement::CanCachePlan(const LogicalOperator &root) { - vector> operators; - operators.push_back(root); - - for (idx_t i = 0; i < operators.size(); i++) { - auto &op = operators[i].get(); - switch (op.type) { - case LogicalOperatorType::LOGICAL_GET: - // this operator prevents caching - return false; - default: - break; - } - // investigate the children of this operator - for (auto &child : op.children) { - operators.push_back(*child); - } - } - return true; + allow_stream_result ? QueryResultOutputType::ALLOW_STREAMING : QueryResultOutputType::FORCE_MATERIALIZED; + return client_context->PendingInternalStatement(CreateExecuteStatement(named_values), parameters); } } // namespace duckdb diff --git a/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp b/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp index 037ae908d..7a6e590d5 100644 --- a/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp +++ b/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp @@ -18,7 +18,6 @@ void CommonAggregateOptimizer::VisitOperator(LogicalOperator &op) { case LogicalOperatorType::LOGICAL_UNION: case LogicalOperatorType::LOGICAL_EXCEPT: case LogicalOperatorType::LOGICAL_INTERSECT: - case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: case LogicalOperatorType::LOGICAL_PROJECTION: { CommonAggregateOptimizer common_aggregate; common_aggregate.StandardVisitOperator(op); diff --git a/src/duckdb/src/optimizer/filter_combiner.cpp b/src/duckdb/src/optimizer/filter_combiner.cpp index 99e5b087c..fbc357d93 100644 --- a/src/duckdb/src/optimizer/filter_combiner.cpp +++ b/src/duckdb/src/optimizer/filter_combiner.cpp @@ -1456,10 +1456,19 @@ ValueComparisonResult CompareValueInformation(ExpressionValueInformation &left, // (1) prune nothing or // (2) return UNSATISFIABLE // the SMALLER THAN constant has to be greater than the BIGGER THAN constant - if (left.constant >= right.constant) { + if (left.constant > right.constant) { return ValueComparisonResult::PRUNE_NOTHING; - } else { + } else if (left.constant < right.constant) { return ValueComparisonResult::UNSATISFIABLE_CONDITION; + } else { + // the constants are equal + // This is only satisfiable if both bounds are inclusive + if (left.comparison_type == ExpressionType::COMPARE_LESSTHANOREQUALTO && + right.comparison_type == ExpressionType::COMPARE_GREATERTHANOREQUALTO) { + return ValueComparisonResult::PRUNE_NOTHING; + } else { + return ValueComparisonResult::UNSATISFIABLE_CONDITION; + } } } else { // left is [>] and right is [<] or [!=] diff --git a/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp b/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp index 0bdb67c19..6c2c8eb1e 100644 --- a/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +++ b/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp @@ -26,6 +26,10 @@ static void NormalizeColumnRefAliases(unique_ptr &expr, const Logica return; } const ColumnIndex &col_idx = column_ids[binding.column_index]; + if (!col_idx.HasPrimaryIndex()) { + ref.SetAlias(Identifier(col_idx.GetFieldName())); + return; + } const idx_t primary = col_idx.GetPrimaryIndex(); if (col_idx.IsVirtualColumn()) { if (const auto it = get.virtual_columns.find(primary); it != get.virtual_columns.end()) { diff --git a/src/duckdb/src/optimizer/remove_unused_columns.cpp b/src/duckdb/src/optimizer/remove_unused_columns.cpp index 9b1a985b9..31064c3a4 100644 --- a/src/duckdb/src/optimizer/remove_unused_columns.cpp +++ b/src/duckdb/src/optimizer/remove_unused_columns.cpp @@ -579,8 +579,6 @@ void RemoveUnusedColumns::WritePushdownExtractColumns( static unique_ptr ConstructStructExtractFromPath(ClientContext &context, unique_ptr target, const ColumnIndex &path) { - auto extract_function = GetKeyExtractFunction(); - auto &struct_type = target->GetReturnType(); D_ASSERT(struct_type.id() == LogicalTypeId::STRUCT); reference type_iter(struct_type); @@ -589,13 +587,19 @@ static unique_ptr ConstructStructExtractFromPath(ClientContext &cont auto child_index = path_iter.get().GetPrimaryIndex(); auto &child_types = StructType::GetChildTypes(type_iter.get()); D_ASSERT(child_index < child_types.size()); - auto &key = child_types[child_index].first; + auto is_unnamed = StructType::IsUnnamed(type_iter.get()); + auto function = is_unnamed ? GetIndexExtractFunction() : GetKeyExtractFunction(); + type_iter = child_types[child_index].second; vector> arguments(2); arguments[0] = (std::move(target)); - arguments[1] = (make_uniq(Value(key))); - target = extract_function.Bind(context, std::move(arguments)); + if (is_unnamed) { + arguments[1] = make_uniq(Value::BIGINT(NumericCast(child_index + 1))); + } else { + arguments[1] = make_uniq(Value(child_types[child_index].first)); + } + target = function.Bind(context, std::move(arguments)); if (!path_iter.get().HasChildren()) { break; } diff --git a/src/duckdb/src/optimizer/statistics/operator/propagate_aggregate.cpp b/src/duckdb/src/optimizer/statistics/operator/propagate_aggregate.cpp index 6723a2b77..d623b98a5 100644 --- a/src/duckdb/src/optimizer/statistics/operator/propagate_aggregate.cpp +++ b/src/duckdb/src/optimizer/statistics/operator/propagate_aggregate.cpp @@ -342,6 +342,11 @@ void StatisticsPropagator::TryExecuteAggregates(LogicalAggregate &aggr, unique_p partition_stats = std::move(precomputed_partition_stats); } + if (partition_stats.empty()) { + // no partitions can be pre-computed + return; + } + if (!min_max_columns.empty()) { // Execute min/max aggregates on partition statistics for (idx_t agg_idx = 0; agg_idx < min_max_storage_indexes.size(); agg_idx++) { diff --git a/src/duckdb/src/optimizer/topn_window_elimination.cpp b/src/duckdb/src/optimizer/topn_window_elimination.cpp index 598efa850..9d057832e 100644 --- a/src/duckdb/src/optimizer/topn_window_elimination.cpp +++ b/src/duckdb/src/optimizer/topn_window_elimination.cpp @@ -142,31 +142,39 @@ ColumnBinding GetRowNumberColumnBinding(const unique_ptr &op) { } } -idx_t TraverseAndFindAggregateOffset(const unique_ptr &op) { - reference current_op = *op; - while (current_op.get().type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { - D_ASSERT(!current_op.get().children.empty()); - current_op = *current_op.get().children[0]; - } - const auto &aggregate = current_op.get().Cast(); - return aggregate.groups.size(); -} +struct LHSColumnInfo { + string name; + LogicalType type; + ColumnBinding binding; +}; + +LHSColumnInfo GetLHSColumnInfo(const unique_ptr &op, idx_t column_id) { + D_ASSERT(column_id < op->types.size()); + const auto bindings = op->GetColumnBindings(); + D_ASSERT(column_id < bindings.size()); + + LHSColumnInfo result; + result.type = op->types[column_id]; + result.binding = bindings[column_id]; -string GetLHSRowIdColumnName(const unique_ptr &op, idx_t column_id) { reference current_op = *op; + auto get_binding = result.binding; if (op.get()->type != LogicalOperatorType::LOGICAL_GET) { D_ASSERT(op.get()->type == LogicalOperatorType::LOGICAL_PROJECTION); D_ASSERT(op.get()->expressions.size() > column_id && op.get()->expressions[column_id]->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF); const auto &colref = op.get()->expressions[column_id]->Cast(); - column_id = colref.Binding().column_index; + get_binding = colref.Binding(); current_op = *op.get()->children[0]; } const auto &logical_get = current_op.get().Cast(); - const auto column_index = logical_get.GetColumnIds()[column_id]; - return logical_get.GetColumnName(column_index).GetIdentifierName(); + D_ASSERT(get_binding.table_index == logical_get.table_index); + D_ASSERT(get_binding.column_index < logical_get.GetColumnIds().size()); + const auto column_index = logical_get.GetColumnIds()[get_binding.column_index]; + result.name = logical_get.GetColumnName(column_index).GetIdentifierName(); + return result; } //! Late materialization recreates a payload by re-scanning the base table column and re-applying a type cast at the @@ -273,7 +281,7 @@ unique_ptr TopNWindowElimination::OptimizeInternal(unique_ptrgroupings_index = optimizer.binder.GenerateTableIndex(); aggregate->groups = std::move(window_expr.PartitionsMutable()); + if (aggregate->groups.empty() && params.limit == 1) { + // A constant group preserves the window's empty-input behavior. + aggregate->groups.push_back(make_uniq(Value::BOOLEAN(true))); + } aggregate->children.push_back(std::move(window.children[0])); aggregate->ResolveOperatorTypes(); @@ -760,7 +772,8 @@ unique_ptr TopNWindowElimination::UpdateTopmostBindings(TableIndex window_idx, unique_ptr op, const vector &types, const map &group_idxs, const vector &topmost_bindings, - vector &new_bindings, ColumnBindingReplacer &replacer) { + vector &new_bindings, ColumnBindingReplacer &replacer, + const TopNWindowEliminationParameters ¶ms) { // The top-most operator's column order is: // [projected groups][aggregate args/value][row number] // Now set the new bindings according to this order and remember replacements in replacer @@ -800,8 +813,8 @@ TopNWindowElimination::UpdateTopmostBindings(TableIndex window_idx, unique_ptrtype == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { - // We do not have an aggregate index, so we need to set an offset to hit the correct columns - current_column_idx = TraverseAndFindAggregateOffset(op->children[1]); + // The late-materialized LHS contains the semantic partitions, not synthetic aggregate groups. + current_column_idx = params.partition_count; } // Project the args/value @@ -867,6 +880,7 @@ TopNWindowElimination::ExtractOptimizerParameters(const LogicalWindow &window, c params.include_row_number = BindingsReferenceRowNumber(bindings, window); params.payload_type = aggregate_payload.size() > 1 ? TopNPayloadType::STRUCT_PACK : TopNPayloadType::SINGLE_COLUMN; auto &window_expr = window.expressions[0]->Cast(); + params.partition_count = window_expr.Partitions().size(); params.order_type = window_expr.OrderBy()[0].type; VisitExpression(&window_expr.OrderByMutable()[0].expression); @@ -1225,13 +1239,12 @@ unique_ptr TopNWindowElimination::ConstructJoin(unique_ptrtypes.size() - (rowid_column_count - i); const idx_t rhs_rowid_idx = rhs_binding_offset + i; - const auto &alias = GetLHSRowIdColumnName(lhs, lhs_rowid_idx); + const auto lhs_column = GetLHSColumnInfo(lhs, lhs_rowid_idx); - auto lhs_expr = make_uniq( - Identifier(alias), lhs->types[lhs_rowid_idx], - ColumnBinding {lhs->GetTableIndex()[0], ProjectionIndex(lhs_rowid_idx)}); + auto lhs_expr = + make_uniq(Identifier(lhs_column.name), lhs_column.type, lhs_column.binding); auto rhs_expr = - make_uniq(Identifier(alias), rhs->types[aggregate_offset + i], + make_uniq(Identifier(lhs_column.name), rhs->types[aggregate_offset + i], ColumnBinding {GetAggregateIdx(rhs), ProjectionIndex(rhs_rowid_idx)}); join->conditions.push_back( JoinCondition(std::move(lhs_expr), std::move(rhs_expr), ExpressionType::COMPARE_EQUAL)); diff --git a/src/duckdb/src/parser/keyword_helper.cpp b/src/duckdb/src/parser/keyword_helper.cpp index 9334f4c9e..feb9fceaa 100644 --- a/src/duckdb/src/parser/keyword_helper.cpp +++ b/src/duckdb/src/parser/keyword_helper.cpp @@ -93,6 +93,9 @@ string SQLIdentifier::ToString(const string &identifier) { return SQLQuotedIdentifier::ToString(identifier); } +SQLQuotedIdentifier::SQLQuotedIdentifier(const Identifier &id) : raw_string(id.GetIdentifierName()) { +} + string SQLQuotedIdentifier::ToString(const string &identifier) { return KeywordHelper::WriteQuotedAndEscaped(identifier, '"'); } diff --git a/src/duckdb/src/parser/peg/transformer/transform_alter.cpp b/src/duckdb/src/parser/peg/transformer/transform_alter.cpp index 224a95118..18b910e95 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_alter.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_alter.cpp @@ -28,6 +28,10 @@ unique_ptr PEGTransformerFactory::TransformAlterStatement(PEGTrans add_column.new_column.DefaultValue().GetExpressionClass() == ExpressionClass::CONSTANT) { return std::move(result); } + if (add_column.if_column_not_exists) { + // IF NOT EXISTS is not supported by the multi-statement rewrite - keep the plain ALTER + return std::move(result); + } auto &column_entry = add_column.new_column; auto null_column = column_entry.Copy(); null_column.SetDefaultValue(make_uniq(ConstantExpression(Value(nullptr)))); diff --git a/src/duckdb/src/parser/peg/transformer/transform_expression.cpp b/src/duckdb/src/parser/peg/transformer/transform_expression.cpp index 3e0c922cc..c3860370c 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_expression.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_expression.cpp @@ -159,7 +159,7 @@ unique_ptr PEGTransformerFactory::TransformFunctionExpression( MethodArguments function_expression_arguments, optional> within_group_clause, optional> filter_clause, const bool &has_result, optional> over_clause) { - auto qualified_function = function_identifier; + const auto &qualified_function = function_identifier; bool export_clause = has_result; auto distinct = function_expression_arguments.distinct; auto function_children = std::move(function_expression_arguments.arguments); diff --git a/src/duckdb/src/parser/statement/execute_statement.cpp b/src/duckdb/src/parser/statement/execute_statement.cpp index 6a4439708..8ae41d6cc 100644 --- a/src/duckdb/src/parser/statement/execute_statement.cpp +++ b/src/duckdb/src/parser/statement/execute_statement.cpp @@ -5,7 +5,8 @@ namespace duckdb { ExecuteStatement::ExecuteStatement() : SQLStatement(StatementType::EXECUTE_STATEMENT) { } -ExecuteStatement::ExecuteStatement(const ExecuteStatement &other) : SQLStatement(other), name(other.name) { +ExecuteStatement::ExecuteStatement(const ExecuteStatement &other) + : SQLStatement(other), name(other.name), bound_values(other.bound_values) { for (const auto &item : other.named_values) { named_values.emplace(std::make_pair(item.first, item.second->Copy())); } @@ -19,11 +20,14 @@ string ExecuteStatement::ToString() const { string result = ""; result += "EXECUTE"; result += " " + name; - if (!named_values.empty()) { - vector stringified; - for (auto &val : named_values) { - stringified.push_back(StringUtil::Format("%s := %s", val.first, val.second->ToString())); - } + vector stringified; + for (auto &val : named_values) { + stringified.push_back(StringUtil::Format("%s := %s", val.first, val.second->ToString())); + } + for (auto &val : bound_values) { + stringified.push_back(StringUtil::Format("%s := %s", val.first, val.second.GetValue().ToSQLString())); + } + if (!stringified.empty()) { result += "(" + StringUtil::Join(stringified, ", ") + ")"; } result += ";"; diff --git a/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp b/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp index 4dc5f17e3..5e0b244fe 100644 --- a/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +++ b/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp @@ -65,7 +65,7 @@ void ExpressionBinder::ReplaceMacroParameters(unique_ptr &expr if (col_ref.IsQualified()) { // the table qualifier is the component directly before the column name auto &names = col_ref.ColumnNames(); - if (StringUtil::StartsWith(names[names.size() - 2].GetIdentifierName(), DummyBinding::DUMMY_NAME)) { + if (names[names.size() - 2].StartsWith(DummyBinding::DUMMY_NAME)) { bind_macro_parameter = true; } } else { diff --git a/src/duckdb/src/planner/binder/statement/bind_create.cpp b/src/duckdb/src/planner/binder/statement/bind_create.cpp index faee041c6..3b5cf804f 100644 --- a/src/duckdb/src/planner/binder/statement/bind_create.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_create.cpp @@ -330,7 +330,7 @@ void Binder::BindView(ClientContext &context, const SelectStatement &stmt, const } view_binder->SetCanContainNulls(true); - auto view_search_path = view_binder->GetSearchPath(catalog, schema_name); + auto view_search_path = view_binder->GetSearchPath(catalog, schema_name, true); view_binder->entry_retriever.SetSearchPath(std::move(view_search_path)); auto copy = stmt.Copy(); diff --git a/src/duckdb/src/planner/binder/statement/bind_create_table.cpp b/src/duckdb/src/planner/binder/statement/bind_create_table.cpp index 77f1a1018..99b012f8c 100644 --- a/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_create_table.cpp @@ -608,9 +608,10 @@ static void BindCreateTableConstraints(CreateTableInfo &create_info, CatalogEntr unique_ptr Binder::BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema, vector> &bound_defaults, AlterBindMode bind_mode) { - auto &base = info->Cast(); auto result = make_uniq(schema, std::move(info)); - auto &dependencies = result->dependencies; + auto &base = result->Base(); + base.dependencies = LogicalDependencyList(); + auto &dependencies = base.dependencies; auto &catalog = schema.ParentCatalog(); optional_ptr storage_manager; if (catalog.IsDuckCatalog() && !catalog.InMemory()) { @@ -710,7 +711,7 @@ unique_ptr Binder::BindCreateTableInfo(unique_ptrdependencies.VerifyDependencies(schema.catalog, result->Base().GetTableName()); + base.dependencies.VerifyDependencies(schema.catalog, base.GetTableName()); #ifdef DEBUG // Ensure all types are bound diff --git a/src/duckdb/src/planner/binder/statement/bind_execute.cpp b/src/duckdb/src/planner/binder/statement/bind_execute.cpp index d36a46ddb..00d99f16f 100644 --- a/src/duckdb/src/planner/binder/statement/bind_execute.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_execute.cpp @@ -29,11 +29,16 @@ BoundStatement Binder::Bind(ExecuteStatement &stmt) { auto prepared = entry->second; auto &named_param_map = prepared->unbound_statement->named_param_map; - PreparedStatement::VerifyParameters(stmt.named_values, named_param_map, &context); + // values that were supplied pre-typed do not go through the literal binding below + identifier_map_t bind_values = stmt.bound_values; + if (stmt.bound_values.empty()) { + PreparedStatement::VerifyParameters(stmt.named_values, named_param_map, &context); + } else { + PreparedStatement::VerifyParameters(stmt.bound_values, named_param_map, &context); + } auto &mapped_named_values = stmt.named_values; // bind any supplied parameters - identifier_map_t bind_values; auto constant_binder = Binder::CreateBinder(context); constant_binder->SetCanContainNulls(true); for (auto &pair : mapped_named_values) { diff --git a/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp b/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp index 22ecdc63e..b034a266c 100644 --- a/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp @@ -438,7 +438,7 @@ BoundStatement Binder::BindNode(MergeQueryNode &node) { } merge_into->row_id_start = projection_expressions.size(); - // finally bind the row id column and add them to the projection list + // Row ID columns must remain last: PhysicalMergeInto treats the trailing columns as one composite key. BindRowIdColumns(table, get, projection_expressions); auto proj = make_uniq(proj_index, std::move(projection_expressions)); diff --git a/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp b/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp index c57f79047..9f7bd5244 100644 --- a/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +++ b/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp @@ -100,7 +100,8 @@ unique_ptr Binder::BindAtClause(optional_ptr at_clause) return make_uniq(at_clause->Unit(), std::move(val)); } -vector Binder::GetSearchPath(Catalog &catalog, const Identifier &schema_name) { +vector Binder::GetSearchPath(Catalog &catalog, const Identifier &schema_name, + bool default_schema_precedence) { vector view_search_path; auto &catalog_name = catalog.GetName(); if (!schema_name.empty()) { @@ -111,7 +112,7 @@ vector Binder::GetSearchPath(Catalog &catalog, const Identif view_search_path.emplace_back(catalog_name, Identifier(default_schema)); } //! Signal that this catalog should be checked, regardless of the schema in the reference - view_search_path.emplace_back(catalog_name, INVALID_SCHEMA); + view_search_path.emplace_back(catalog_name, INVALID_SCHEMA, default_schema_precedence); return view_search_path; } @@ -309,7 +310,7 @@ BoundStatement Binder::Bind(BaseTableRef &ref) { // when binding a view, we always look into the catalog/schema where the view is stored first auto view_search_path = - GetSearchPath(view_catalog_entry.ParentCatalog(), view_catalog_entry.ParentSchema().name); + GetSearchPath(view_catalog_entry.ParentCatalog(), view_catalog_entry.ParentSchema().name, true); view_binder->entry_retriever.SetSearchPath(std::move(view_search_path)); // propagate the AT clause through the view view_binder->entry_retriever.SetAtClause(entry_at_clause); diff --git a/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp b/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp index d88b5c3cd..30c7528b8 100644 --- a/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +++ b/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp @@ -65,9 +65,8 @@ static void ExtractPivotExpressions(ParsedExpression &root_expr, identifier_set_ ParsedExpressionIterator::VisitExpression( root_expr, [&](const ColumnRefExpression &child_colref) { if (child_colref.IsQualified()) { - if (StringUtil::StartsWith(child_colref.ColumnNames()[0].GetIdentifierName(), - DummyBinding::DUMMY_NAME) && - macro_binding && macro_binding->HasMatchingBinding(Identifier(child_colref.GetName()))) { + if (child_colref.ColumnNames()[0].StartsWith(DummyBinding::DUMMY_NAME) && macro_binding && + macro_binding->HasMatchingBinding(Identifier(child_colref.GetName()))) { throw ParameterNotResolvedException(); } throw BinderException(child_colref, "PIVOT expression cannot contain qualified columns"); diff --git a/src/duckdb/src/planner/expression_binder/table_function_binder.cpp b/src/duckdb/src/planner/expression_binder/table_function_binder.cpp index b484add62..a6b44271f 100644 --- a/src/duckdb/src/planner/expression_binder/table_function_binder.cpp +++ b/src/duckdb/src/planner/expression_binder/table_function_binder.cpp @@ -35,8 +35,8 @@ BindResult TableFunctionBinder::BindColumnReference(unique_ptr if (binder.macro_binding && binder.macro_binding->HasMatchingBinding(Identifier(col_ref.GetName()))) { throw ParameterNotResolvedException(); } - } else if (StringUtil::StartsWith(col_ref.ColumnNames()[0].GetIdentifierName(), DummyBinding::DUMMY_NAME) && - binder.macro_binding && binder.macro_binding->HasMatchingBinding(Identifier(col_ref.GetName()))) { + } else if (col_ref.ColumnNames()[0].StartsWith(DummyBinding::DUMMY_NAME) && binder.macro_binding && + binder.macro_binding->HasMatchingBinding(Identifier(col_ref.GetName()))) { throw ParameterNotResolvedException(); } diff --git a/src/duckdb/src/storage/checkpoint_manager.cpp b/src/duckdb/src/storage/checkpoint_manager.cpp index da07aeddc..9e07c9c4b 100644 --- a/src/duckdb/src/storage/checkpoint_manager.cpp +++ b/src/duckdb/src/storage/checkpoint_manager.cpp @@ -715,10 +715,6 @@ void CheckpointReader::ReadTable(CatalogTransaction transaction, Deserializer &d auto &schema = *catalog.GetSchema(transaction, schema_path, OnEntryNotFound::THROW_EXCEPTION); auto bound_info = Binder::BindCreateTableCheckpoint(std::move(info), schema); - for (auto &dep : bound_info->Base().dependencies.Set()) { - bound_info->dependencies.AddDependency(dep); - } - // now read the actual table data and place it into the CreateTableInfo ReadTableData(transaction, deserializer, *bound_info); diff --git a/src/duckdb/src/storage/compression/dict_fsst.cpp b/src/duckdb/src/storage/compression/dict_fsst.cpp index a17ae4427..0d9cd73f3 100644 --- a/src/duckdb/src/storage/compression/dict_fsst.cpp +++ b/src/duckdb/src/storage/compression/dict_fsst.cpp @@ -237,6 +237,34 @@ static void DictFSSTFilter(ColumnSegment &segment, ColumnScanState &state, idx_t ColumnSegment::FilterSelection(sel, result, filter_state, vector_count, sel_count); } +static string DictFSSTModeToString(const DictFSSTMode mode) { + switch (mode) { + case DictFSSTMode::DICTIONARY: + return "DICTIONARY"; + case DictFSSTMode::DICT_FSST: + return "DICT_FSST"; + case DictFSSTMode::FSST_ONLY: + return "FSST_ONLY"; + default: + return "UNKNOWN"; + } +} + +//===--------------------------------------------------------------------===// +// GetSegmentInfo +//===--------------------------------------------------------------------===// +static InsertionOrderPreservingMap DictFSSTGetSegmentInfo(QueryContext, ColumnSegment &segment) { + auto &buffer_manager = BufferManager::GetBufferManager(segment.GetDatabase()); + auto state = make_uniq(segment, buffer_manager.Pin(segment.GetBlockHandle())); + state->Initialize(false); + + const auto tuple_count = segment.count.load(); + + InsertionOrderPreservingMap result; + result[DictFSSTModeToString(state->mode)] = StringUtil::Format("%d", tuple_count); + return result; +} + } // namespace dict_fsst //===--------------------------------------------------------------------===// @@ -255,6 +283,7 @@ CompressionFunction DictFSSTCompressionFun::GetFunction(PhysicalType data_type) res.validity = CompressionValidity::NO_VALIDITY_REQUIRED; res.select = dict_fsst::DictFSSTSelect; res.filter = dict_fsst::DictFSSTFilter; + res.get_segment_info = dict_fsst::DictFSSTGetSegmentInfo; return res; } diff --git a/src/duckdb/src/storage/compression/dict_fsst/analyze.cpp b/src/duckdb/src/storage/compression/dict_fsst/analyze.cpp index 07215b6ca..40abe466e 100644 --- a/src/duckdb/src/storage/compression/dict_fsst/analyze.cpp +++ b/src/duckdb/src/storage/compression/dict_fsst/analyze.cpp @@ -3,7 +3,52 @@ namespace duckdb { namespace dict_fsst { +//! Determine the size requirements for the worst case, which is when a single string fills an +//! entire segment on its own. +static idx_t GetStringSizeLimit(const idx_t available_space, const bool fsst_encoded) { + idx_t max_str_len = DictFSSTCompression::STRING_SIZE_LIMIT - 1; + if (fsst_encoded) { + // In the worst case FSST may double the string length by prepending every byte with an exception + max_str_len *= 2; + } + + // Dictionary contains NULL and current string + const bitpacking_width_t string_lengths_width = BitpackingPrimitives::MinimumBitWidth(max_str_len); + const idx_t string_lengths_space = BitpackingPrimitives::GetRequiredSize(2, string_lengths_width); + + // Dictionary stores only one valid string + const bitpacking_width_t dict_indices_width = BitpackingPrimitives::MinimumBitWidth(1); + const idx_t dict_indices_space = BitpackingPrimitives::GetRequiredSize(1, dict_indices_width); + + idx_t metadata_size = 0; + metadata_size += AlignValue(sizeof(dict_fsst_compression_header_t)); + if (fsst_encoded) { + // As denoted in fsst.h + metadata_size += 7; + } + // Reserve maximum alignment padding for variable length string + metadata_size += sizeof(idx_t) - 1; + if (fsst_encoded) { + metadata_size += AlignValue(DictFSSTCompression::FSST_SYMBOL_TABLE_SIZE); + } + metadata_size += AlignValue(string_lengths_space); + metadata_size += dict_indices_space; + + D_ASSERT(metadata_size < available_space); + idx_t max_string_size = available_space - metadata_size; + + if (fsst_encoded) { + max_string_size = max_string_size / 2; + } + + return MinValue(DictFSSTCompression::STRING_SIZE_LIMIT, max_string_size + 1); +} + DictFSSTAnalyzeState::DictFSSTAnalyzeState(BlockManager &block_manager) : AnalyzeState(block_manager) { + const auto block_size = info.GetBlockSize(); + + string_size_limit = GetStringSizeLimit(block_size, false); + fsst_string_size_limit = GetStringSizeLimit(block_size, true); } bool DictFSSTAnalyzeState::Analyze(const Vector &input) { @@ -18,10 +63,15 @@ bool DictFSSTAnalyzeState::Analyze(const Vector &input) { if (str_len > max_string_length) { max_string_length = str_len; } - if (str_len >= DictFSSTCompression::STRING_SIZE_LIMIT) { - //! This string is too long, we don't want to use DICT_FSST for this rowgroup + if (str_len >= string_size_limit) { + // A segment cannot be spread out over multiple blocks, so if a string cannot fit in an empty segment + // the encoding will fail return false; } + if (str_len >= fsst_string_size_limit) { + // FSST strings may be up to two times larger than their plain equivalent + disable_fsst = true; + } } total_count += input.size(); return true; diff --git a/src/duckdb/src/storage/compression/dict_fsst/compression.cpp b/src/duckdb/src/storage/compression/dict_fsst/compression.cpp index 1e2302ba1..a8a623425 100644 --- a/src/duckdb/src/storage/compression/dict_fsst/compression.cpp +++ b/src/duckdb/src/storage/compression/dict_fsst/compression.cpp @@ -31,7 +31,6 @@ DictFSSTCompressionState::~DictFSSTCompressionState() { } } -static constexpr uint16_t FSST_SYMBOL_TABLE_SIZE = sizeof(duckdb_fsst_decoder_t); static constexpr idx_t DICTIONARY_ENCODE_THRESHOLD = 4096; static inline bool IsEncoded(DictionaryAppendState state) { @@ -54,6 +53,9 @@ static DictFSSTMode ConvertToMode(DictionaryAppendState &state) { idx_t DictFSSTCompressionState::Finalize() { const bool is_fsst_encoded = IsEncoded(append_state); + if (is_fsst_encoded) { + D_ASSERT(analyze->disable_fsst == false); + } // calculate sizes #ifdef DEBUG @@ -240,7 +242,8 @@ void DictFSSTCompressionState::FlushEncodingBuffer() { void DictFSSTCompressionState::CreateEmptySegment() { CreateAndPinNewSegment(); - append_state = DictionaryAppendState::REGULAR; + // If analysis determined that FSST cannot be used, skip the decision phase. + append_state = analyze->disable_fsst ? DictionaryAppendState::NOT_ENCODED : DictionaryAppendState::REGULAR; string_lengths_width = 0; real_string_lengths_width = 0; dictionary_indices_width = 0; @@ -329,7 +332,7 @@ static inline bool AddLookup(DictFSSTCompressionState &state, idx_t lookup, cons idx_t available_space = state.info.GetBlockSize(); if (APPEND_STATE == DictionaryAppendState::REGULAR) { - available_space -= FSST_SYMBOL_TABLE_SIZE; + available_space -= DictFSSTCompression::FSST_SYMBOL_TABLE_SIZE; } if (required_space > available_space) { if (fail_on_no_space) { @@ -412,7 +415,7 @@ static inline bool AddToDictionary(DictFSSTCompressionState &state, const string idx_t available_space = state.info.GetBlockSize(); if (APPEND_STATE == DictionaryAppendState::REGULAR) { - available_space -= FSST_SYMBOL_TABLE_SIZE; + available_space -= DictFSSTCompression::FSST_SYMBOL_TABLE_SIZE; } if (required_space > available_space) { if (fail_on_no_space) { diff --git a/src/duckdb/src/storage/compression/rle.cpp b/src/duckdb/src/storage/compression/rle.cpp index 7c212a1a3..60dfa4dfb 100644 --- a/src/duckdb/src/storage/compression/rle.cpp +++ b/src/duckdb/src/storage/compression/rle.cpp @@ -265,7 +265,7 @@ struct RLEScanState : public SegmentScanState { //! This would make the index_pointer start outside of the segment throw DataCorruptionException("Corrupted RLE segment: rle_count_offset is corrupted"); } - if ((rle_count_offset - RLEConstants::RLE_HEADER_SIZE) / sizeof(T) > max_entry_pos) { + if (rle_count_offset > AlignValue(RLEConstants::RLE_HEADER_SIZE + max_entry_pos * sizeof(T))) { //! This would make the indexing of the index_pointer[entry_pos] reach outside of the segment throw DataCorruptionException("Corrupted RLE segment: rle_count_offset is corrupted"); } diff --git a/src/duckdb/src/storage/data_table.cpp b/src/duckdb/src/storage/data_table.cpp index 4ee67bb51..b6aa6efc9 100644 --- a/src/duckdb/src/storage/data_table.cpp +++ b/src/duckdb/src/storage/data_table.cpp @@ -1367,6 +1367,11 @@ void DataTable::RevertAppend(DuckTransaction &transaction, idx_t start_row, idx_ } #endif + if (!IsMainTable()) { + //! The table was altered by another transaction since we staged our commit, revert it + //! Causing the commit of the other transaction to fail + version = DataTableVersion::MAIN_TABLE; + } // revert the data table append RevertAppendInternal(start_row); } diff --git a/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp b/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp index 2b143c688..966ce131b 100644 --- a/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp +++ b/src/duckdb/src/storage/external_file_cache/caching_file_system.cpp @@ -352,7 +352,16 @@ FileBufferHandleGroup CachingFileHandle::Read(const idx_t nr_bytes, const idx_t } FileBufferHandleGroup CachingFileHandle::Read(idx_t &nr_bytes) { - if (!external_file_cache.IsEnabled() || !CanSeek()) { + // Only cache when file metadata is available. + bool no_validation_metadata = false; + if (Validate()) { + annotated_lock_guard guard(file_handle_mutex); + no_validation_metadata = version_tag.empty() && (!last_modified.IsFinite() || last_modified == timestamp_t(0)); + } + + // If we can't seek, we can't use the cache for these calls, + // because we won't be able to seek over any parts we skipped by reading from the cache + if (!external_file_cache.IsEnabled() || !CanSeek() || no_validation_metadata) { auto buf = AllocateUncachedReadBuffer(external_file_cache.GetBufferManager(), nr_bytes); auto file_handle = GetFileHandle(); nr_bytes = NumericCast(file_handle->Read(context, buf.GetDataMutable(), nr_bytes)); diff --git a/src/duckdb/src/storage/storage_info.cpp b/src/duckdb/src/storage/storage_info.cpp index 8059b1f04..e5535cdfd 100644 --- a/src/duckdb/src/storage/storage_info.cpp +++ b/src/duckdb/src/storage/storage_info.cpp @@ -82,10 +82,13 @@ static const StorageVersionInfo storage_version_info[] = { {"v1.4.2", StorageVersion::V1_4_2}, {"v1.4.3", StorageVersion::V1_4_3}, {"v1.4.4", StorageVersion::V1_4_4}, + {"v1.4.5", StorageVersion::V1_4_5}, {"v1.5.0", StorageVersion::V1_5_0}, {"v1.5.1", StorageVersion::V1_5_1}, {"v1.5.2", StorageVersion::V1_5_2}, {"v1.5.3", StorageVersion::V1_5_3}, + {"v1.5.4", StorageVersion::V1_5_4}, + {"v1.5.5", StorageVersion::V1_5_5}, {"v2.0.0", StorageVersion::V2_0_0}, {"latest", StorageVersion::V2_0_0}, {nullptr, StorageVersion::INVALID} @@ -118,10 +121,13 @@ static const SerializationVersionInfo serialization_version_info[] = { {"v1.4.2", SerializationVersionDeprecated::V1_4_2}, {"v1.4.3", SerializationVersionDeprecated::V1_4_3}, {"v1.4.4", SerializationVersionDeprecated::V1_4_4}, + {"v1.4.5", SerializationVersionDeprecated::V1_4_5}, {"v1.5.0", SerializationVersionDeprecated::V1_5_0}, {"v1.5.1", SerializationVersionDeprecated::V1_5_1}, {"v1.5.2", SerializationVersionDeprecated::V1_5_2}, {"v1.5.3", SerializationVersionDeprecated::V1_5_3}, + {"v1.5.4", SerializationVersionDeprecated::V1_5_4}, + {"v1.5.5", SerializationVersionDeprecated::V1_5_5}, {"v2.0.0", SerializationVersionDeprecated::V2_0_0}, {"latest", SerializationVersionDeprecated::V2_0_0}, {nullptr, SerializationVersionDeprecated::INVALID} diff --git a/src/duckdb/src/storage/table/per_column_metadata_blocks.cpp b/src/duckdb/src/storage/table/per_column_metadata_blocks.cpp index afc7d6f34..da026e54b 100644 --- a/src/duckdb/src/storage/table/per_column_metadata_blocks.cpp +++ b/src/duckdb/src/storage/table/per_column_metadata_blocks.cpp @@ -100,7 +100,7 @@ PerColumnMetadataBlocks PerColumnMetadataBlocks::Merge(const PerColumnMetadataBl return result; } -void PerColumnMetadataBlocks::RemoveColumn(idx_t col_idx) { +void PerColumnMetadataBlocks::ClearColumn(idx_t col_idx) { idx_t start = data.size(); idx_t end = data.size(); for (idx_t i = 0; i < data.size(); i++) { @@ -122,4 +122,13 @@ void PerColumnMetadataBlocks::RemoveColumn(idx_t col_idx) { } } +void PerColumnMetadataBlocks::RemoveColumn(idx_t col_idx) { + ClearColumn(col_idx); + for (auto &entry : data) { + if (entry.is_column_index && entry.index > col_idx) { + entry.index--; + } + } +} + } // namespace duckdb diff --git a/src/duckdb/src/storage/table/row_group.cpp b/src/duckdb/src/storage/table/row_group.cpp index a1863db6f..cae215dcc 100644 --- a/src/duckdb/src/storage/table/row_group.cpp +++ b/src/duckdb/src/storage/table/row_group.cpp @@ -519,7 +519,7 @@ unique_ptr RowGroup::AlterType(RowGroupCollection &new_collection, con } if (has_per_column_metadata_blocks) { row_group->per_column_metadata_blocks = per_column_metadata_blocks; - row_group->per_column_metadata_blocks.RemoveColumn(changed_idx); + row_group->per_column_metadata_blocks.ClearColumn(changed_idx); } lock.unlock(); row_group->Verify(); @@ -613,6 +613,8 @@ unique_ptr RowGroup::RemoveColumn(RowGroupCollection &new_collection, } if (has_per_column_metadata_blocks) { row_group->per_column_metadata_blocks = per_column_metadata_blocks; + // the columns after the removed one shift down by one position, so their + // metadata block entries (keyed by column index) must shift down as well row_group->per_column_metadata_blocks.RemoveColumn(removed_column); } lock.unlock(); diff --git a/src/duckdb/src/storage/table/row_group_collection.cpp b/src/duckdb/src/storage/table/row_group_collection.cpp index 70f7e8f73..59b80417d 100644 --- a/src/duckdb/src/storage/table/row_group_collection.cpp +++ b/src/duckdb/src/storage/table/row_group_collection.cpp @@ -149,6 +149,7 @@ RowGroupCollection::RowGroupCollection(shared_ptr info_p, BlockMa if (TypeVisitor::Contains(type, LogicalTypeId::VARIANT) || TypeVisitor::Contains(type, LogicalTypeId::GEOMETRY)) { row_group_append_mode = RowGroupAppendMode::REQUIRE_NEW; + can_append_to_checkpointed_row_group = false; break; } } @@ -233,6 +234,10 @@ void RowGroupCollection::Initialize(PersistentCollectionData &data) { } void RowGroupCollection::SetRowGroupAppendMode(RowGroupAppendMode mode) { + if (mode == RowGroupAppendMode::SUGGEST_NEW && !can_append_to_checkpointed_row_group) { + // if we cannot append to existing (checkpointed) row groups we need to promote SUGGEST_NEW to REQUIRE_NEW + mode = RowGroupAppendMode::REQUIRE_NEW; + } if (mode > row_group_append_mode) { // We never downgrade the mode, i.e. if REQUIRE_NEW was already set then we do not set it back to SUGGEST_NEW row_group_append_mode = mode; diff --git a/src/duckdb/src/transaction/commit_state.cpp b/src/duckdb/src/transaction/commit_state.cpp index 81d7f238e..fd83741aa 100644 --- a/src/duckdb/src/transaction/commit_state.cpp +++ b/src/duckdb/src/transaction/commit_state.cpp @@ -277,6 +277,15 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data, CommitInfo &info) D_ASSERT(catalog.IsDuckCatalog()); auto &new_entry = old_entry.Parent(); + if (old_entry.type == CatalogType::TABLE_ENTRY && new_entry.type == CatalogType::TABLE_ENTRY) { + auto &old_storage = old_entry.Cast().GetStorage(); + auto &new_storage = new_entry.Cast().GetStorage(); + if (!RefersToSameObject(old_storage, new_storage) && old_storage.IsMainTable()) { + throw TransactionException("Failed to alter table \"%s\" because the underlying table state was " + "reverted by a concurrent transaction", + old_entry.name); + } + } if (new_entry.type == CatalogType::DEPENDENCY_ENTRY) { auto &dep = new_entry.Cast(); if (dep.Side() == DependencyEntryType::SUBJECT) { diff --git a/src/duckdb/src/transaction/meta_transaction.cpp b/src/duckdb/src/transaction/meta_transaction.cpp index 213fc271d..2dd3a367c 100644 --- a/src/duckdb/src/transaction/meta_transaction.cpp +++ b/src/duckdb/src/transaction/meta_transaction.cpp @@ -203,6 +203,7 @@ idx_t MetaTransaction::GetActiveQuery() { } void MetaTransaction::SetActiveQuery(transaction_t query_number) { + lock_guard guard(lock); active_query = query_number; for (auto &entry : transactions) { entry.second.transaction.active_query = query_number; diff --git a/src/duckdb/third_party/fmt/include/fmt/core.h b/src/duckdb/third_party/fmt/include/fmt/core.h index 652a03c71..66b0c9d10 100644 --- a/src/duckdb/third_party/fmt/include/fmt/core.h +++ b/src/duckdb/third_party/fmt/include/fmt/core.h @@ -297,11 +297,26 @@ template class basic_string_view { : data_(s.data()), size_(s.size()) {} - template < - typename S, - FMT_ENABLE_IF(std::is_same>::value)> - FMT_CONSTEXPR basic_string_view(S s) FMT_NOEXCEPT : data_(s.data()), - size_(s.size()) {} + /** Constructs a string reference from a ``std::basic_string_view`` object. */ + // Traits is deduced rather than defaulted, so that declaring this constructor + // never names std::char_traits. libc++ deprecates char_traits for T + // other than the standard character types, and that warning is not suppressed + // by SFINAE -- it fires whenever basic_string_view is instantiated with + // a non-character Char during overload resolution. This mirrors the + // std::basic_string constructor above. + // See duckdb-internal/9978 +#if defined(FMT_USE_STRING_VIEW) + template + FMT_CONSTEXPR basic_string_view( + std::basic_string_view s) FMT_NOEXCEPT : data_(s.data()), + size_(s.size()) {} +#elif defined(FMT_USE_EXPERIMENTAL_STRING_VIEW) + template + FMT_CONSTEXPR basic_string_view( + std::experimental::basic_string_view s) FMT_NOEXCEPT + : data_(s.data()), + size_(s.size()) {} +#endif /** Returns a pointer to the string data. */ FMT_CONSTEXPR const Char* data() const { return data_; } @@ -396,12 +411,25 @@ inline basic_string_view to_string_view(basic_string_view s) { return s; } -template >::value)> +// Traits is deduced rather than defaulted so that forming the parameter type +// never names std::char_traits. Deducing it lets substitution fail on the +// shape of the argument instead, before char_traits is instantiated -- +// libc++ deprecates char_traits for T other than the standard character +// types, and that deprecation warning is not suppressed by SFINAE. +// See duckdb-internal/9978 +#if defined(FMT_USE_STRING_VIEW) +template +inline basic_string_view to_string_view( + std::basic_string_view s) { + return s; +} +#elif defined(FMT_USE_EXPERIMENTAL_STRING_VIEW) +template inline basic_string_view to_string_view( - internal::std_string_view s) { + std::experimental::basic_string_view s) { return s; } +#endif // A base class for compile-time strings. It is defined in the fmt namespace to // make formatting functions visible via ADL, e.g. format(fmt("{}"), 42). diff --git a/src/duckdb/ub_src_execution.cpp b/src/duckdb/ub_src_execution.cpp index 2253c8430..e8eaa0781 100644 --- a/src/duckdb/ub_src_execution.cpp +++ b/src/duckdb/ub_src_execution.cpp @@ -22,3 +22,5 @@ #include "src/execution/radix_partitioned_hashtable.cpp" +#include "src/execution/row_id_deduplicator.cpp" +