Merge branch 'main' into codex/update-lance-8-0-0

Author: Xuanwo

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 
 ## [Unreleased]
 
+### Changed
+
+- `COPY`, `CREATE TABLE`, and CTAS now default new or replaced datasets to Lance data storage version `2.2`.
+
 ## [0.5.1] - 2026-02-10
 
 ### Added

docs/sql.md

@@ -201,6 +201,7 @@ COPY (
 
 Notes:
 - `mode` supports at least `overwrite` and `append`.
+- `data_storage_version` defaults to `2.2` when `COPY` creates or overwrites a dataset. Appends preserve the existing storage version.
 - `write_empty_file` controls whether an empty dataset is materialized when the input produces zero rows.
 
 ### `CREATE TABLE` / `CTAS` in an attached namespace
@@ -223,6 +224,9 @@ SELECT count(*) FROM ns.main.my_dataset;
 DETACH ns;
 ```
 
+`CREATE TABLE` and CTAS default to data storage version `2.2`. Override it with
+`WITH (data_storage_version = '<version>')` when another version is required.
+
 ## DML on attached tables
 
 These statements apply to tables inside an attached namespace (e.g. `ns.main.my_table`).

src/include/lance_common.hpp

@@ -32,6 +32,7 @@ static constexpr uint64_t LANCE_DEFAULT_MAX_ROWS_PER_FILE = 1024ULL * 1024ULL;
 static constexpr uint64_t LANCE_DEFAULT_MAX_ROWS_PER_GROUP = 1024ULL;
 static constexpr uint64_t LANCE_DEFAULT_MAX_BYTES_PER_FILE =
     90ULL * 1024ULL * 1024ULL * 1024ULL;
+static constexpr const char *LANCE_DEFAULT_DATA_STORAGE_VERSION = "2.2";
 
 void ResolveLanceNamespaceAuth(ClientContext &context, const string &endpoint,
                                const unordered_map<string, Value> &options,

src/lance_storage.cpp

@@ -628,7 +628,7 @@ static string
 GetCreateTableDataStorageVersionOption(const CreateTableInfo &create_info) {
   auto it = create_info.options.find("data_storage_version");
   if (it == create_info.options.end()) {
-    return "";
+    return LANCE_DEFAULT_DATA_STORAGE_VERSION;
   }
   if (!it->second) {
     throw BinderException("data_storage_version option cannot be NULL");

src/lance_write.cpp

@@ -16,7 +16,7 @@ namespace duckdb {
 
 struct LanceWriteBindData : public FunctionData {
   string mode = "create";
-  string data_storage_version;
+  string data_storage_version = LANCE_DEFAULT_DATA_STORAGE_VERSION;
   uint64_t max_rows_per_file = LANCE_DEFAULT_MAX_ROWS_PER_FILE;
   uint64_t max_rows_per_group = LANCE_DEFAULT_MAX_ROWS_PER_GROUP;
   uint64_t max_bytes_per_file = LANCE_DEFAULT_MAX_BYTES_PER_FILE;

test/sql/dml_copy_to.test

@@ -24,6 +24,15 @@ SELECT sum(id) FROM 'test/.tmp/lance_write_basic.lance';
 ----
 3
 
+# 322E32 is the hexadecimal representation of "2.2" in the manifest.
+query I
+SELECT contains(hex(content), '322E32')::INTEGER
+FROM read_blob('test/.tmp/lance_write_basic.lance/_versions/*.manifest')
+ORDER BY last_modified DESC, filename
+LIMIT 1;
+----
+1
+
 statement ok
 COPY (
   SELECT 3::BIGINT AS id, 'c'::VARCHAR AS s
@@ -81,6 +90,19 @@ SELECT count(*) FROM 'test/.tmp/lance_write_with_version_21.lance';
 ----
 1
 
+statement ok
+COPY (
+  SELECT 2::BIGINT AS id, 'w'::VARCHAR AS s
+) TO 'test/.tmp/lance_write_with_version_21.lance' (FORMAT lance, mode 'append');
+
+query I
+SELECT contains(hex(content), '322E31')::INTEGER
+FROM read_blob('test/.tmp/lance_write_with_version_21.lance/_versions/*.manifest')
+ORDER BY last_modified DESC, filename
+LIMIT 1;
+----
+1
+
 statement ok
 COPY (
   SELECT 1::BIGINT AS id, 'v'::VARCHAR AS s

test/sql/namespace_create_table.test

@@ -18,6 +18,15 @@ SELECT count(*) FROM ns.main.attach_create_table_empty
 ----
 0
 
+# 322E32 is the hexadecimal representation of "2.2" in the manifest.
+query I
+SELECT contains(hex(content), '322E32')::INTEGER
+FROM read_blob('test/.tmp/nsroot_create_table_125/attach_create_table_empty.lance/_versions/*.manifest')
+ORDER BY last_modified DESC, filename
+LIMIT 1;
+----
+1
+
 statement ok
 CREATE OR REPLACE TABLE ns.main.attach_create_table_with_version
   (id BIGINT, s VARCHAR)
@@ -44,6 +53,14 @@ SELECT count(*) FROM 'test/.tmp/nsroot_create_table_125/attach_create_table_ctas
 ----
 2
 
+query I
+SELECT contains(hex(content), '322E32')::INTEGER
+FROM read_blob('test/.tmp/nsroot_create_table_125/attach_create_table_ctas.lance/_versions/*.manifest')
+ORDER BY last_modified DESC, filename
+LIMIT 1;
+----
+1
+
 statement ok
 CREATE OR REPLACE TABLE ns.main.attach_create_table_ctas AS
   SELECT 3::BIGINT AS id, 'c'::VARCHAR AS s;