From b97f9a6f6e0625bdb0ea45f87b022f89ed75f93a Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Sun, 5 Jul 2026 18:34:41 +0300 Subject: [PATCH] chore(deps): drop the vendored lance-table sources (pin retired with the 9.0.0-beta.15 bump) --- vendor/lance-table/.cargo_vcs_info.json | 6 - vendor/lance-table/Cargo.lock | 5741 ----------------- vendor/lance-table/Cargo.toml | 263 - vendor/lance-table/Cargo.toml.orig | 80 - vendor/lance-table/README.md | 6 - vendor/lance-table/README.omnigraph.md | 42 - vendor/lance-table/benches/manifest_intern.rs | 261 - vendor/lance-table/benches/row_id_index.rs | 323 - vendor/lance-table/build.rs | 29 - vendor/lance-table/protos/AGENTS.md | 18 - vendor/lance-table/protos/CLAUDE.md | 18 - vendor/lance-table/protos/ann.proto | 55 - .../lance-table/protos/encodings_v2_0.proto | 347 - .../lance-table/protos/encodings_v2_1.proto | 511 -- vendor/lance-table/protos/file.proto | 207 - vendor/lance-table/protos/file2.proto | 210 - vendor/lance-table/protos/filtered_read.proto | 99 - vendor/lance-table/protos/index.proto | 249 - vendor/lance-table/protos/index_old.proto | 42 - vendor/lance-table/protos/license_header.txt | 2 - vendor/lance-table/protos/rowids.proto | 113 - vendor/lance-table/protos/table.proto | 717 -- .../lance-table/protos/table_identifier.proto | 19 - vendor/lance-table/protos/transaction.proto | 354 - vendor/lance-table/src/feature_flags.rs | 184 - vendor/lance-table/src/format.rs | 70 - vendor/lance-table/src/format/fragment.rs | 841 --- vendor/lance-table/src/format/index.rs | 368 -- vendor/lance-table/src/format/manifest.rs | 1490 ----- vendor/lance-table/src/format/transaction.rs | 42 - vendor/lance-table/src/io.rs | 6 - vendor/lance-table/src/io/commit.rs | 1898 ------ vendor/lance-table/src/io/commit/dynamodb.rs | 495 -- .../src/io/commit/external_manifest.rs | 515 -- vendor/lance-table/src/io/deletion.rs | 370 -- vendor/lance-table/src/io/manifest.rs | 344 - vendor/lance-table/src/lib.rs | 8 - vendor/lance-table/src/rowids.rs | 1364 ---- vendor/lance-table/src/rowids/bitmap.rs | 314 - .../lance-table/src/rowids/encoded_array.rs | 400 -- vendor/lance-table/src/rowids/index.rs | 822 --- vendor/lance-table/src/rowids/segment.rs | 1141 ---- vendor/lance-table/src/rowids/serde.rs | 239 - vendor/lance-table/src/rowids/version.rs | 713 -- vendor/lance-table/src/utils.rs | 47 - vendor/lance-table/src/utils/stream.rs | 806 --- 46 files changed, 22189 deletions(-) delete mode 100644 vendor/lance-table/.cargo_vcs_info.json delete mode 100644 vendor/lance-table/Cargo.lock delete mode 100644 vendor/lance-table/Cargo.toml delete mode 100644 vendor/lance-table/Cargo.toml.orig delete mode 100644 vendor/lance-table/README.md delete mode 100644 vendor/lance-table/README.omnigraph.md delete mode 100644 vendor/lance-table/benches/manifest_intern.rs delete mode 100644 vendor/lance-table/benches/row_id_index.rs delete mode 100644 vendor/lance-table/build.rs delete mode 100644 vendor/lance-table/protos/AGENTS.md delete mode 100644 vendor/lance-table/protos/CLAUDE.md delete mode 100644 vendor/lance-table/protos/ann.proto delete mode 100644 vendor/lance-table/protos/encodings_v2_0.proto delete mode 100644 vendor/lance-table/protos/encodings_v2_1.proto delete mode 100644 vendor/lance-table/protos/file.proto delete mode 100644 vendor/lance-table/protos/file2.proto delete mode 100644 vendor/lance-table/protos/filtered_read.proto delete mode 100644 vendor/lance-table/protos/index.proto delete mode 100644 vendor/lance-table/protos/index_old.proto delete mode 100644 vendor/lance-table/protos/license_header.txt delete mode 100644 vendor/lance-table/protos/rowids.proto delete mode 100644 vendor/lance-table/protos/table.proto delete mode 100644 vendor/lance-table/protos/table_identifier.proto delete mode 100644 vendor/lance-table/protos/transaction.proto delete mode 100644 vendor/lance-table/src/feature_flags.rs delete mode 100644 vendor/lance-table/src/format.rs delete mode 100644 vendor/lance-table/src/format/fragment.rs delete mode 100644 vendor/lance-table/src/format/index.rs delete mode 100644 vendor/lance-table/src/format/manifest.rs delete mode 100755 vendor/lance-table/src/format/transaction.rs delete mode 100644 vendor/lance-table/src/io.rs delete mode 100644 vendor/lance-table/src/io/commit.rs delete mode 100644 vendor/lance-table/src/io/commit/dynamodb.rs delete mode 100644 vendor/lance-table/src/io/commit/external_manifest.rs delete mode 100644 vendor/lance-table/src/io/deletion.rs delete mode 100644 vendor/lance-table/src/io/manifest.rs delete mode 100644 vendor/lance-table/src/lib.rs delete mode 100644 vendor/lance-table/src/rowids.rs delete mode 100644 vendor/lance-table/src/rowids/bitmap.rs delete mode 100644 vendor/lance-table/src/rowids/encoded_array.rs delete mode 100644 vendor/lance-table/src/rowids/index.rs delete mode 100644 vendor/lance-table/src/rowids/segment.rs delete mode 100644 vendor/lance-table/src/rowids/serde.rs delete mode 100644 vendor/lance-table/src/rowids/version.rs delete mode 100644 vendor/lance-table/src/utils.rs delete mode 100644 vendor/lance-table/src/utils/stream.rs diff --git a/vendor/lance-table/.cargo_vcs_info.json b/vendor/lance-table/.cargo_vcs_info.json deleted file mode 100644 index 88182098..00000000 --- a/vendor/lance-table/.cargo_vcs_info.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "git": { - "sha1": "a15ae30939b9242d74b00aed1fb83abf7d15bf7f" - }, - "path_in_vcs": "rust/lance-table" -} \ No newline at end of file diff --git a/vendor/lance-table/Cargo.lock b/vendor/lance-table/Cargo.lock deleted file mode 100644 index b0bac7de..00000000 --- a/vendor/lance-table/Cargo.lock +++ /dev/null @@ -1,5741 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "const-random", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "aligned-vec" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" -dependencies = [ - "equator", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "arrow" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", -] - -[[package]] -name = "arrow-arith" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "num-traits", -] - -[[package]] -name = "arrow-array" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" -dependencies = [ - "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "chrono-tz", - "half", - "hashbrown 0.17.1", - "num-complex", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-buffer" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-cast" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num-traits", - "ryu", -] - -[[package]] -name = "arrow-csv" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" -dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "csv", - "csv-core", - "regex", -] - -[[package]] -name = "arrow-data" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-ipc" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "flatbuffers", - "lz4_flex", - "zstd", -] - -[[package]] -name = "arrow-json" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "chrono", - "half", - "indexmap 2.14.0", - "itoa", - "lexical-core", - "memchr", - "num-traits", - "ryu", - "serde_core", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-ord" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", -] - -[[package]] -name = "arrow-row" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "half", -] - -[[package]] -name = "arrow-schema" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" -dependencies = [ - "bitflags 2.11.1", -] - -[[package]] -name = "arrow-select" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num-traits", -] - -[[package]] -name = "arrow-string" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - -[[package]] -name = "async-compression" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" -dependencies = [ - "compression-codecs", - "compression-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-lock" -version = "3.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-recursion" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "aws-config" -version = "1.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 1.4.1", - "ring", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - -[[package]] -name = "aws-lc-rs" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-runtime" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "bytes-utils", - "fastrand", - "http 1.4.1", - "http-body 1.0.1", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-dynamodb" -version = "1.107.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561bf86e858a2759c6876b517b13f3f4051a6484abbb0d8a1f4dfc5d902cc85a" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.1", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.95.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.1", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.97.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.1", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.99.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand", - "http 0.2.12", - "http 1.4.1", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9" -dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "form_urlencoded", - "hex", - "hmac", - "http 0.2.12", - "http 1.4.1", - "percent-encoding", - "sha2", - "time", - "tracing", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc50d0f63e714784b84223abd7abbc8577de8c35d699e0edd19f0a88a08ae13" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-http" -version = "0.63.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d619373d490ad70966994801bc126846afaa0d1ee920697a031f0cf63f2568e7" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "futures-util", - "http 1.4.1", - "http-body 1.0.1", - "http-body-util", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-client" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ccbb08c10f6bcf912f398188e42ee2eab5f1767ce215a02a73bc5df1bbdd95" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2", - "http 1.4.1", - "hyper", - "hyper-rustls", - "hyper-util", - "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.62.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-observability" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" -dependencies = [ - "aws-smithy-runtime-api", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f76a580e3d8f8961e5d48763214025a2af65c2fa4cd1fb7f270a0e107a71b0" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ccf7f6eba8b2dcf8ce9b74806c6c185659c311665c4bf8d6e71ebd454db6bf" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-client", - "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.1", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af6e5def28be846479bbeac55aa4603d6f7986fc5da4601ba324dd5d377516" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.4.1", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-types" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca2734c16913a45343b37313605d84e7d8b34a4611598ce1d25b35860a2bed3" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.4.1", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53543b4b86ed43f051644f704a98c7291b3618b67adf057ee77a366fa52fcaa" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "rustc_version", - "tracing", -] - -[[package]] -name = "backon" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" -dependencies = [ - "fastrand", - "gloo-timers", - "tokio", -] - -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "brotli" -version = "8.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - -[[package]] -name = "cc" -version = "1.2.62" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chacha20" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - -[[package]] -name = "chrono" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "chrono-tz" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" -dependencies = [ - "chrono", - "phf", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - -[[package]] -name = "combine" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" -dependencies = [ - "bytes", - "memchr", -] - -[[package]] -name = "comfy-table" -version = "7.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" -dependencies = [ - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "compression-codecs" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" -dependencies = [ - "compression-core", - "flate2", - "memchr", -] - -[[package]] -name = "compression-core" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpp_demangle" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "futures", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "csv" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde_core", -] - -[[package]] -name = "csv-core" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" -dependencies = [ - "memchr", -] - -[[package]] -name = "ctor" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" -dependencies = [ - "ctor-proc-macro", - "dtor", -] - -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - -[[package]] -name = "darling" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" -dependencies = [ - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", -] - -[[package]] -name = "darling_macro" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "datafusion-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" -dependencies = [ - "ahash", - "arrow", - "arrow-ipc", - "chrono", - "half", - "hashbrown 0.16.1", - "indexmap 2.14.0", - "itertools 0.14.0", - "libc", - "log", - "paste", - "tokio", - "web-time", -] - -[[package]] -name = "debugid" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" -dependencies = [ - "uuid", -] - -[[package]] -name = "deepsize" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" -dependencies = [ - "deepsize_derive", -] - -[[package]] -name = "deepsize_derive" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", - "serde_core", -] - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "displaydoc" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equator" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" -dependencies = [ - "equator-macro", -] - -[[package]] -name = "equator-macro" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "ethnum" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" - -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "fast-float2" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" - -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "findshlibs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" -dependencies = [ - "cc", - "lazy_static", - "libc", - "winapi", -] - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "flatbuffers" -version = "25.12.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" -dependencies = [ - "bitflags 2.11.1", - "rustc_version", -] - -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - -[[package]] -name = "fsst" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd0ce0249ac12fd44fcde62d435c36d881952c2f0df4d1de24b45e1dbba5ddb" -dependencies = [ - "arrow-array", - "rand 0.9.4", -] - -[[package]] -name = "futures" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-executor" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" - -[[package]] -name = "futures-macro" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-timer" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" - -[[package]] -name = "futures-util" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 5.3.0", - "wasip2", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasip2", - "wasip3", -] - -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "h2" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.4.1", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "num-traits", - "zerocopy", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http 1.4.1", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http 1.4.1", - "http-body 1.0.1", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "humantime" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" - -[[package]] -name = "hyper" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http 1.4.1", - "http-body 1.0.1", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" -dependencies = [ - "http 1.4.1", - "hyper", - "hyper-util", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http 1.4.1", - "http-body 1.0.1", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "hyperloglogplus" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" -dependencies = [ - "serde", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", -] - -[[package]] -name = "inferno" -version = "0.11.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" -dependencies = [ - "ahash", - "indexmap 2.14.0", - "is-terminal", - "itoa", - "log", - "num-format", - "once_cell", - "quick-xml 0.26.0", - "rgb", - "str_stack", -] - -[[package]] -name = "io-uring" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" -dependencies = [ - "bitflags 2.11.1", - "cfg-if", - "libc", -] - -[[package]] -name = "ipnet" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "jiff" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392c70591e8749fe235ddaf513e6f58b26bce3dcc16524cecc8936f75afa161e" -dependencies = [ - "jiff-static", - "jiff-tzdb-platform", - "js-sys", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "jiff-static" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b605b0c050d845fc355bb11eb3f9a8deddc218ea60c76e61aa1f2adfb2c96a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "jiff-tzdb" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - -[[package]] -name = "jni" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" -dependencies = [ - "cfg-if", - "combine", - "jni-macros", - "jni-sys", - "log", - "simd_cesu8", - "thiserror 2.0.18", - "walkdir", - "windows-link", -] - -[[package]] -name = "jni-macros" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "simd_cesu8", - "syn 2.0.117", -] - -[[package]] -name = "jni-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" -dependencies = [ - "jni-sys-macros", -] - -[[package]] -name = "jni-sys-macros" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" -dependencies = [ - "quote", - "syn 2.0.117", -] - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" -dependencies = [ - "cfg-if", - "futures-util", - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "jsonb" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" -dependencies = [ - "byteorder", - "ethnum", - "fast-float2", - "itoa", - "jiff", - "nom", - "num-traits", - "ordered-float", - "rand 0.9.4", - "serde", - "serde_json", - "zmij", -] - -[[package]] -name = "lance-arrow" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "arrow-select", - "bytes", - "futures", - "getrandom 0.2.17", - "half", - "jsonb", - "num-traits", - "rand 0.9.4", -] - -[[package]] -name = "lance-bitpacking" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c4d12521b1945041dd515a56aa0854973138e7ac12111c92572e33e4ecb593" -dependencies = [ - "arrayref", - "paste", - "seq-macro", -] - -[[package]] -name = "lance-core" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "async-trait", - "byteorder", - "bytes", - "deepsize", - "futures", - "itertools 0.13.0", - "lance-arrow", - "libc", - "log", - "moka", - "num_cpus", - "object_store", - "pin-project", - "prost", - "rand 0.9.4", - "roaring", - "serde_json", - "snafu", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "url", -] - -[[package]] -name = "lance-datagen" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046f5506ed2271cd941a050de7bf535dd3aedc291aadec836a63fa56c5926e3b" -dependencies = [ - "arrow", - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "futures", - "half", - "hex", - "rand 0.9.4", - "rand_distr", - "rand_xoshiro", - "random_word", -] - -[[package]] -name = "lance-encoding" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af54edf43dcf9d6a56cc636eb35d457e68373c6448dca3f0891b3325b4a24e6" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", - "bytemuck", - "byteorder", - "bytes", - "fsst", - "futures", - "hex", - "hyperloglogplus", - "itertools 0.13.0", - "lance-arrow", - "lance-bitpacking", - "lance-core", - "log", - "lz4", - "num-traits", - "prost", - "prost-build", - "rand 0.9.4", - "strum", - "tokio", - "tracing", - "xxhash-rust", - "zstd", -] - -[[package]] -name = "lance-file" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0772ae2d6207995dc1eb28aff9507f78e90b3362b58f311da001e9dc25f3d736" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "async-recursion", - "async-trait", - "byteorder", - "bytes", - "datafusion-common", - "deepsize", - "futures", - "lance-arrow", - "lance-core", - "lance-encoding", - "lance-io", - "log", - "num-traits", - "object_store", - "prost", - "prost-build", - "prost-types", - "tokio", - "tracing", -] - -[[package]] -name = "lance-io" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab8c98ef1b870b20541d27f3ca4efdf7c9f5c25214233be07d231ba88900219" -dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", - "async-recursion", - "async-trait", - "aws-config", - "aws-credential-types", - "byteorder", - "bytes", - "chrono", - "deepsize", - "futures", - "http 1.4.1", - "io-uring", - "lance-arrow", - "lance-core", - "lance-namespace", - "log", - "moka", - "object_store", - "object_store_opendal", - "opendal", - "path_abs", - "pin-project", - "prost", - "rand 0.9.4", - "serde", - "tempfile", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "lance-namespace" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "lance-core", - "lance-namespace-reqwest-client", - "snafu", -] - -[[package]] -name = "lance-namespace-reqwest-client" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99" -dependencies = [ - "reqwest 0.12.28", - "serde", - "serde_json", - "serde_repr", - "serde_with", - "url", -] - -[[package]] -name = "lance-table" -version = "7.0.0" -dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ipc", - "arrow-schema", - "async-trait", - "aws-credential-types", - "aws-sdk-dynamodb", - "byteorder", - "bytes", - "chrono", - "criterion", - "deepsize", - "futures", - "lance-arrow", - "lance-core", - "lance-datagen", - "lance-file", - "lance-io", - "log", - "object_store", - "pprof", - "pretty_assertions", - "proptest", - "prost", - "prost-build", - "prost-types", - "protobuf-src", - "rand 0.9.4", - "rangemap", - "roaring", - "rstest", - "semver", - "serde", - "serde_json", - "snafu", - "tokio", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "lz4" -version = "1.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" -dependencies = [ - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "lz4_flex" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" -dependencies = [ - "twox-hash", -] - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "mea" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" -dependencies = [ - "slab", -] - -[[package]] -name = "memchr" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" - -[[package]] -name = "memmap2" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" -dependencies = [ - "libc", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "mio" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "moka" -version = "0.12.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "event-listener", - "futures-util", - "parking_lot", - "portable-atomic", - "smallvec", - "tagptr", - "uuid", -] - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", -] - -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - -[[package]] -name = "object_store" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" -dependencies = [ - "async-trait", - "base64", - "bytes", - "chrono", - "form_urlencoded", - "futures-channel", - "futures-core", - "futures-util", - "http 1.4.1", - "http-body-util", - "humantime", - "hyper", - "itertools 0.14.0", - "md-5", - "parking_lot", - "percent-encoding", - "quick-xml 0.39.4", - "rand 0.10.1", - "reqwest 0.12.28", - "ring", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror 2.0.18", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - -[[package]] -name = "object_store_opendal" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08298874eee5935c95bcaa393148834f9c53d904461ca15584a041d8a1c907c2" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "mea", - "object_store", - "opendal", - "pin-project", - "tokio", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - -[[package]] -name = "opendal" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236" -dependencies = [ - "ctor", - "opendal-core", - "opendal-layer-concurrent-limit", - "opendal-layer-logging", - "opendal-layer-retry", - "opendal-layer-timeout", - "opendal-service-s3", -] - -[[package]] -name = "opendal-core" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1849dd2687e173e776d3af5fce1ba3ae47b9dd37a09d1c4deba850ef45fe00ca" -dependencies = [ - "anyhow", - "base64", - "bytes", - "futures", - "http 1.4.1", - "http-body 1.0.1", - "jiff", - "log", - "md-5", - "mea", - "percent-encoding", - "quick-xml 0.38.4", - "reqsign-core", - "reqwest 0.13.4", - "serde", - "serde_json", - "tokio", - "url", - "uuid", - "web-time", -] - -[[package]] -name = "opendal-layer-concurrent-limit" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "048b1b29c503263bdd80a9afe46a68cd02ea9bd361185b1feab4b151078998e9" -dependencies = [ - "futures", - "http 1.4.1", - "mea", - "opendal-core", -] - -[[package]] -name = "opendal-layer-logging" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2645adc988b12eda106e2679ae529facfbbaa868ceb706f6f8125c6af15c47b" -dependencies = [ - "log", - "opendal-core", -] - -[[package]] -name = "opendal-layer-retry" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eac134ffa4ddda6131a640a84a5315996424b9416c85052f8c64c1a33b70ad4" -dependencies = [ - "backon", - "log", - "opendal-core", -] - -[[package]] -name = "opendal-layer-timeout" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619586ab7480c2e3009f6d18eabab18957bc094778fd130bcc38924970a90f4c" -dependencies = [ - "opendal-core", - "tokio", -] - -[[package]] -name = "opendal-service-s3" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dadddeb9bb50b0d30927dd914c298c4ddca47e4c1cfa7674d311f0cf9b051c8" -dependencies = [ - "base64", - "bytes", - "crc32c", - "http 1.4.1", - "log", - "md-5", - "opendal-core", - "quick-xml 0.38.4", - "reqsign-aws-v4", - "reqsign-core", - "reqsign-file-read-tokio", - "serde", - "url", -] - -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "ordered-float" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - -[[package]] -name = "outref" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "path_abs" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3" -dependencies = [ - "serde", - "serde_derive", - "std_prelude", - "stfu8", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.14.0", -] - -[[package]] -name = "phf" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "portable-atomic-util" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "pprof" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afad4d4df7b31280028245f152d5a575083e2abb822d05736f5e47653e77689f" -dependencies = [ - "aligned-vec", - "backtrace", - "cfg-if", - "criterion", - "findshlibs", - "inferno", - "libc", - "log", - "nix", - "once_cell", - "smallvec", - "spin", - "symbolic-demangle", - "tempfile", - "thiserror 1.0.69", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - -[[package]] -name = "proc-macro-crate" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" -dependencies = [ - "toml_edit", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "proptest" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" -dependencies = [ - "bit-set", - "bit-vec", - "bitflags 2.11.1", - "num-traits", - "rand 0.9.4", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "rusty-fork", - "tempfile", - "unarray", -] - -[[package]] -name = "prost" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" -dependencies = [ - "heck", - "itertools 0.14.0", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.117", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" -dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" -dependencies = [ - "prost", -] - -[[package]] -name = "protobuf-src" -version = "2.1.1+27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6217c3504da19b85a3a4b2e9a5183d635822d83507ba0986624b5c05b83bfc40" -dependencies = [ - "cmake", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quick-xml" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" -dependencies = [ - "memchr", -] - -[[package]] -name = "quick-xml" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick-xml" -version = "0.39.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quinn" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.18", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" -dependencies = [ - "aws-lc-rs", - "bytes", - "getrandom 0.3.4", - "lru-slab", - "rand 0.9.4", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.60.2", -] - -[[package]] -name = "quote" -version = "1.0.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha", - "rand_core 0.9.5", -] - -[[package]] -name = "rand" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" -dependencies = [ - "chacha20", - "getrandom 0.4.2", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_distr" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" -dependencies = [ - "num-traits", - "rand 0.9.4", -] - -[[package]] -name = "rand_xorshift" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" -dependencies = [ - "rand_core 0.9.5", -] - -[[package]] -name = "rand_xoshiro" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" -dependencies = [ - "rand_core 0.9.5", -] - -[[package]] -name = "random_word" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81" -dependencies = [ - "ahash", - "brotli", - "paste", - "rand 0.9.4", - "unicase", -] - -[[package]] -name = "rangemap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags 2.11.1", -] - -[[package]] -name = "ref-cast" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-lite" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - -[[package]] -name = "relative-path" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" - -[[package]] -name = "reqsign-aws-v4" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" -dependencies = [ - "anyhow", - "bytes", - "form_urlencoded", - "http 1.4.1", - "log", - "percent-encoding", - "quick-xml 0.39.4", - "reqsign-core", - "rust-ini", - "serde", - "serde_json", - "serde_urlencoded", - "sha1", -] - -[[package]] -name = "reqsign-core" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" -dependencies = [ - "anyhow", - "base64", - "bytes", - "form_urlencoded", - "futures", - "hex", - "hmac", - "http 1.4.1", - "jiff", - "log", - "percent-encoding", - "sha1", - "sha2", - "windows-sys 0.61.2", -] - -[[package]] -name = "reqsign-file-read-tokio" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" -dependencies = [ - "anyhow", - "reqsign-core", - "tokio", -] - -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http 1.4.1", - "http-body 1.0.1", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams 0.4.2", - "web-sys", -] - -[[package]] -name = "reqwest" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" -dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "http 1.4.1", - "http-body 1.0.1", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams 0.5.0", - "web-sys", -] - -[[package]] -name = "rgb" -version = "0.8.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "roaring" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" -dependencies = [ - "bytemuck", - "byteorder", -] - -[[package]] -name = "rstest" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" -dependencies = [ - "futures", - "futures-timer", - "rstest_macros", - "rustc_version", -] - -[[package]] -name = "rstest_macros" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "825ea780781b15345a146be27eaefb05085e337e869bff01b4306a4fd4a9ad5a" -dependencies = [ - "cfg-if", - "glob", - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn 2.0.117", - "unicode-ident", -] - -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - -[[package]] -name = "rustc-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags 2.11.1", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls" -version = "0.23.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" -dependencies = [ - "aws-lc-rs", - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pki-types" -version = "1.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-platform-verifier" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" -dependencies = [ - "core-foundation", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" - -[[package]] -name = "rustls-webpki" -version = "0.103.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" -dependencies = [ - "aws-lc-rs", - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "rusty-fork" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "schemars" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "schemars" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags 2.11.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_json" -version = "1.0.150" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_repr" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" -dependencies = [ - "base64", - "bs58", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.14.0", - "schemars 0.9.0", - "schemars 1.2.1", - "serde_core", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - -[[package]] -name = "simd-adler32" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" - -[[package]] -name = "simd_cesu8" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" -dependencies = [ - "rustc_version", - "simdutf8", -] - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "snafu" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" -dependencies = [ - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "socket2" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "spin" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" -dependencies = [ - "lock_api", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "std_prelude" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe" - -[[package]] -name = "stfu8" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" - -[[package]] -name = "str_stack" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "symbolic-common" -version = "12.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" -dependencies = [ - "debugid", - "memmap2", - "stable_deref_trait", - "uuid", -] - -[[package]] -name = "symbolic-demangle" -version = "12.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" -dependencies = [ - "cpp_demangle", - "rustc-demangle", - "symbolic-common", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.2", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "time" -version = "0.3.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "time-macros" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.52.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" -dependencies = [ - "bytes", - "libc", - "mio", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] - -[[package]] -name = "toml_edit" -version = "0.25.12+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" -dependencies = [ - "indexmap 2.14.0", - "toml_datetime", - "toml_parser", - "winnow", -] - -[[package]] -name = "toml_parser" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" -dependencies = [ - "winnow", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "async-compression", - "bitflags 2.11.1", - "bytes", - "futures-core", - "futures-util", - "http 1.4.1", - "http-body 1.0.1", - "http-body-util", - "pin-project-lite", - "tokio", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "url", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "twox-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" - -[[package]] -name = "typenum" -version = "1.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" - -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-segmentation" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" -dependencies = [ - "getrandom 0.4.2", - "js-sys", - "serde_core", - "wasm-bindgen", -] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - -[[package]] -name = "wait-timeout" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" -dependencies = [ - "libc", -] - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.117", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "wasm-streams" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.11.1", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - -[[package]] -name = "web-sys" -version = "0.3.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-root-certs" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "winnow" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.11.1", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - -[[package]] -name = "xxhash-rust" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" - -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - -[[package]] -name = "yoke" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/vendor/lance-table/Cargo.toml b/vendor/lance-table/Cargo.toml deleted file mode 100644 index 01f85ee7..00000000 --- a/vendor/lance-table/Cargo.toml +++ /dev/null @@ -1,263 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2024" -rust-version = "1.91.0" -name = "lance-table" -version = "7.0.0" -authors = ["Lance Devs "] -build = "build.rs" -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Utilities for the Lance table format" -readme = "README.md" -keywords = [ - "data-format", - "data-science", - "machine-learning", - "apache-arrow", - "data-analytics", -] -categories = [ - "database-implementations", - "data-structures", - "development-tools", - "science", -] -license = "Apache-2.0" -repository = "https://github.com/lance-format/lance" - -[package.metadata.docs.rs] -features = ["protoc"] - -[features] -dynamodb = [ - "dep:aws-sdk-dynamodb", - "dep:aws-credential-types", - "lance-io/aws", -] -protoc = ["dep:protobuf-src"] - -[lib] -name = "lance_table" -path = "src/lib.rs" - -[[bench]] -name = "manifest_intern" -path = "benches/manifest_intern.rs" -harness = false - -[[bench]] -name = "row_id_index" -path = "benches/row_id_index.rs" -harness = false - -[dependencies.arrow] -version = "58.0.0" -features = ["prettyprint"] - -[dependencies.arrow-array] -version = "58.0.0" - -[dependencies.arrow-buffer] -version = "58.0.0" - -[dependencies.arrow-ipc] -version = "58.0.0" -features = ["zstd"] - -[dependencies.arrow-schema] -version = "58.0.0" - -[dependencies.async-trait] -version = "0.1" - -[dependencies.aws-credential-types] -version = "1.2.0" -optional = true - -[dependencies.aws-sdk-dynamodb] -version = "1.38.0" -features = [ - "default-https-client", - "rt-tokio", -] -optional = true -default-features = false - -[dependencies.byteorder] -version = "1.5" - -[dependencies.bytes] -version = "1.11.1" - -[dependencies.chrono] -version = "0.4.41" -features = [ - "std", - "now", - "serde", -] -default-features = false - -[dependencies.deepsize] -version = "0.2.0" - -[dependencies.futures] -version = "0.3" - -[dependencies.lance-arrow] -version = "=7.0.0" - -[dependencies.lance-core] -version = "=7.0.0" - -[dependencies.lance-file] -version = "=7.0.0" - -[dependencies.lance-io] -version = "=7.0.0" -default-features = false - -[dependencies.log] -version = "0.4" - -[dependencies.object_store] -version = "0.13.2" - -[dependencies.prost] -version = "0.14.1" - -[dependencies.prost-types] -version = "0.14.1" - -[dependencies.rand] -version = "0.9.1" -features = ["small_rng"] - -[dependencies.rangemap] -version = "1.0" - -[dependencies.roaring] -version = "0.11" - -[dependencies.semver] -version = "1.0" - -[dependencies.serde] -version = "^1" - -[dependencies.serde_json] -version = "1" - -[dependencies.snafu] -version = "0.9" - -[dependencies.tokio] -version = "1.23" -features = [ - "rt-multi-thread", - "macros", - "fs", - "sync", -] - -[dependencies.tracing] -version = "0.1" - -[dependencies.url] -version = "2.5.7" - -[dependencies.uuid] -version = "1.2" -features = [ - "v4", - "serde", -] - -[dev-dependencies.arrow-schema] -version = "58.0.0" - -[dev-dependencies.criterion] -version = "0.5" -features = [ - "async", - "async_tokio", - "html_reports", -] - -[dev-dependencies.lance-datagen] -version = "=7.0.0" - -[dev-dependencies.pretty_assertions] -version = "1.4.0" - -[dev-dependencies.proptest] -version = "1.3.1" - -[dev-dependencies.rstest] -version = "0.23.0" - -[build-dependencies.prost-build] -version = "0.14.1" - -[build-dependencies.protobuf-src] -version = "2.1" -optional = true - -[target.'cfg(target_os = "linux")'.dev-dependencies.pprof] -version = "0.14.0" -features = [ - "flamegraph", - "criterion", -] - -[lints.clippy] -dbg_macro = "deny" -disallowed_macros = "deny" -fallible_impl_from = "deny" -large_futures = "deny" -manual_let_else = "deny" -multiple-crate-versions = "allow" -print_stderr = "deny" -print_stdout = "deny" -redundant_clone = "deny" -redundant_pub_crate = "deny" -single_range_in_vec_init = "allow" -string_add = "deny" -string_add_assign = "deny" -string_lit_as_bytes = "deny" -trait_duplication_in_bounds = "deny" -use_self = "deny" - -[lints.clippy.all] -level = "deny" -priority = -1 - -[lints.clippy.cargo] -level = "deny" -priority = -1 - -[lints.clippy.style] -level = "deny" -priority = -1 - -[lints.rust] -unsafe_op_in_unsafe_fn = "allow" - -[lints.rust.unexpected_cfgs] -level = "warn" -priority = 0 -check-cfg = ["cfg(coverage,coverage_nightly)"] diff --git a/vendor/lance-table/Cargo.toml.orig b/vendor/lance-table/Cargo.toml.orig deleted file mode 100644 index c9cfc828..00000000 --- a/vendor/lance-table/Cargo.toml.orig +++ /dev/null @@ -1,80 +0,0 @@ -[package] -name = "lance-table" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -readme = "README.md" -description = "Utilities for the Lance table format" -keywords.workspace = true -categories.workspace = true -rust-version.workspace = true - -[dependencies] -lance-arrow.workspace = true -lance-core.workspace = true -lance-file.workspace = true -lance-io.workspace = true -arrow.workspace = true -arrow-array.workspace = true -arrow-buffer.workspace = true -arrow-ipc.workspace = true -arrow-schema.workspace = true -async-trait.workspace = true -aws-credential-types = { workspace = true, optional = true } -aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] } -byteorder.workspace = true -bytes.workspace = true -chrono.workspace = true -deepsize.workspace = true -futures.workspace = true -log.workspace = true -object_store.workspace = true -prost.workspace = true -prost-types.workspace = true -rand.workspace = true -rangemap.workspace = true -roaring.workspace = true -serde.workspace = true -serde_json.workspace = true -semver.workspace = true -snafu.workspace = true -tokio.workspace = true -tracing.workspace = true -url.workspace = true -uuid.workspace = true - -[dev-dependencies] -lance-datagen.workspace = true -arrow-schema.workspace = true -criterion.workspace = true -pretty_assertions.workspace = true -proptest.workspace = true -rstest.workspace = true - -[target.'cfg(target_os = "linux")'.dev-dependencies] -pprof = { workspace = true } - -[build-dependencies] -prost-build.workspace = true -protobuf-src = { version = "2.1", optional = true } - -[features] -dynamodb = ["dep:aws-sdk-dynamodb", "dep:aws-credential-types", "lance-io/aws"] -protoc = ["dep:protobuf-src"] - -[package.metadata.docs.rs] -# docs.rs uses an older version of Ubuntu that does not have the necessary protoc version -features = ["protoc"] - -[[bench]] -name = "row_id_index" -harness = false - -[[bench]] -name = "manifest_intern" -harness = false - -[lints] -workspace = true diff --git a/vendor/lance-table/README.md b/vendor/lance-table/README.md deleted file mode 100644 index d646747e..00000000 --- a/vendor/lance-table/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# lance-table - -`lance-table` is an internal sub-crate for the -[Lance table format](https://lance.org/format/table/). - -**Important Note**: This crate is **not intended for external usage**. diff --git a/vendor/lance-table/README.omnigraph.md b/vendor/lance-table/README.omnigraph.md deleted file mode 100644 index d4404a9a..00000000 --- a/vendor/lance-table/README.omnigraph.md +++ /dev/null @@ -1,42 +0,0 @@ -# Vendored `lance-table` 7.0.0 + lance#7480 (omnigraph patch pin) - -This directory is the **pristine `lance-table` 7.0.0 crates.io source** (unpacked -from the published `.crate`) carrying exactly one upstream fix, cherry-picked -from [lance-format/lance#7480](https://github.com/lance-format/lance/pull/7480) -(merged to Lance main 2026-07-01, first present in no release ≤ 8.0.0): - -- `src/rowids/index.rs` — `RowIdIndex::new` no longer asserts that overlapping - row-id chunks densely tile their range (an update-style `merge_insert` - legally reuses the updated rows' stable ids in new fragments while the - superseded fragment keeps its full sequence + a deletion vector; a later - delete leaves the union short of the span). The real invariant — the same - live id claimed by two fragments — is now a hard error in - `merge_overlapping_chunks` instead. Upstream's regression unit test is - included. - -Without the fix, any filtered read that builds the row-id index on such a -table fails: `rowids/index.rs:50` "Wrong range" debug assert; "all columns in -a record batch must have the same length" (or a silently-wrong batch) in -release. Bug: [lance#7444](https://github.com/lance-format/lance/issues/7444), -tracked as `iss-merge-rowid-overlap-corrupts-filtered-reads` / -`blk-lance-7444` on the dev graph. - -Wired up via `[patch.crates-io] lance-table = { path = "vendor/lance-table" }` -in the workspace root `Cargo.toml`. - -## Removal condition - -Delete this directory and the `[patch.crates-io]` entry at the **first Lance -bump whose `lance-table` ships lance#7480** — 9.0.0, or a backported 8.0.1 if -upstream cuts one. The runtime guard -`crates/omnigraph/tests/lance_surface_guards.rs::filtered_scan_tolerates_merge_update_row_id_overlap` -pins the fixed behavior: it goes red if the patch is dropped too early or a -future bump regresses the fix. - -## Verifying the delta - -```bash -# The full diff vs the published crate should be ONLY the #7480 hunk + this README: -tar -xzf ~/.cargo/registry/cache/index.crates.io-*/lance-table-7.0.0.crate -C /tmp -diff -ru /tmp/lance-table-7.0.0 vendor/lance-table -``` diff --git a/vendor/lance-table/benches/manifest_intern.rs b/vendor/lance-table/benches/manifest_intern.rs deleted file mode 100644 index 09ba9264..00000000 --- a/vendor/lance-table/benches/manifest_intern.rs +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -// Benchmarks use eprintln! to report memory stats alongside criterion output. -#![allow(clippy::print_stderr)] - -//! Benchmark for manifest fragment interning. -//! -//! Measures memory savings and deserialization throughput when interning -//! `DataFile.fields`, `DataFile.column_indices`, and -//! `RowDatasetVersionMeta::Inline` bytes across many fragments. - -use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use deepsize::DeepSizeOf; -use prost::Message; - -use lance_table::format::pb; -use lance_table::format::{DataFileFieldInterner, Fragment}; - -fn num_fragments() -> u64 { - std::env::var("BENCH_NUM_FRAGMENTS") - .map(|s| s.parse().unwrap()) - .unwrap_or(100_000) -} - -/// Build a vector of protobuf DataFragment messages that simulate a -/// homogeneous, post-compaction table: every fragment has the same field -/// list, column indices, and version metadata bytes. -fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec { - let fields: Vec = (0..num_fields as i32).collect(); - let column_indices: Vec = (0..num_fields as i32).collect(); - - // Simulate version metadata: a small protobuf-encoded payload - // (identical across all fragments post-compaction) - let version_bytes: Vec = { - let seq = pb::RowDatasetVersionSequence { - runs: vec![pb::RowDatasetVersionRun { - span: Some(pb::U64Segment { - segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range { - start: 0, - end: 1000, - })), - }), - version: 42, - }], - }; - seq.encode_to_vec() - }; - - (0..n) - .map(|i| pb::DataFragment { - id: i, - files: vec![pb::DataFile { - path: format!("data/{i}.lance"), - fields: fields.clone(), - column_indices: column_indices.clone(), - file_major_version: 2, - file_minor_version: 0, - file_size_bytes: 0, - base_id: None, - }], - deletion_file: None, - row_id_sequence: None, - physical_rows: 1000, - last_updated_at_version_sequence: Some( - pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( - version_bytes.clone(), - ), - ), - created_at_version_sequence: Some( - pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions( - version_bytes.clone(), - ), - ), - }) - .collect() -} - -/// Deserialize protobuf fragments WITHOUT interning (baseline). -fn deserialize_without_interning(protos: &[pb::DataFragment]) -> Vec { - protos - .iter() - .map(|p| Fragment::try_from(p.clone()).unwrap()) - .collect() -} - -/// Deserialize protobuf fragments WITH interning. -fn deserialize_with_interning(protos: &[pb::DataFragment]) -> Vec { - let mut interner = DataFileFieldInterner::default(); - protos - .iter() - .map(|p| interner.intern_fragment(p.clone()).unwrap()) - .collect() -} - -/// Build fragments where each group shares the same version metadata, -/// simulating many small appends without compaction. -fn make_diverse_pb_fragments( - n: u64, - num_fields: usize, - unique_versions: u64, -) -> Vec { - let fields: Vec = (0..num_fields as i32).collect(); - let column_indices: Vec = (0..num_fields as i32).collect(); - let group_size = n / unique_versions; - - let version_payloads: Vec> = (0..unique_versions) - .map(|v| { - let seq = pb::RowDatasetVersionSequence { - runs: vec![pb::RowDatasetVersionRun { - span: Some(pb::U64Segment { - segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range { - start: 0, - end: 1000, - })), - }), - version: v, - }], - }; - seq.encode_to_vec() - }) - .collect(); - - (0..n) - .map(|i| { - let version_idx = (i / group_size).min(unique_versions - 1) as usize; - pb::DataFragment { - id: i, - files: vec![pb::DataFile { - path: format!("data/{i}.lance"), - fields: fields.clone(), - column_indices: column_indices.clone(), - file_major_version: 2, - file_minor_version: 0, - file_size_bytes: 0, - base_id: None, - }], - deletion_file: None, - row_id_sequence: None, - physical_rows: 1000, - last_updated_at_version_sequence: Some( - pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( - version_payloads[version_idx].clone(), - ), - ), - created_at_version_sequence: Some( - pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions( - version_payloads[version_idx].clone(), - ), - ), - } - }) - .collect() -} - -fn bench_deserialization(c: &mut Criterion) { - let mut group = c.benchmark_group("manifest_intern"); - let n = num_fragments(); - - for num_fields in [10, 50] { - let protos = make_uniform_pb_fragments(n, num_fields); - - group.bench_with_input( - BenchmarkId::new("deserialize_no_intern", num_fields), - &num_fields, - |b, _| { - b.iter(|| deserialize_without_interning(&protos)); - }, - ); - - group.bench_with_input( - BenchmarkId::new("deserialize_with_intern", num_fields), - &num_fields, - |b, _| { - b.iter(|| deserialize_with_interning(&protos)); - }, - ); - } - - // Benchmark with many unique version payloads - for unique_versions in [10, 100, 500] { - let protos = make_diverse_pb_fragments(n, 10, unique_versions); - - group.bench_with_input( - BenchmarkId::new("deserialize_no_intern_diverse", unique_versions), - &unique_versions, - |b, _| { - b.iter(|| deserialize_without_interning(&protos)); - }, - ); - - group.bench_with_input( - BenchmarkId::new("deserialize_with_intern_diverse", unique_versions), - &unique_versions, - |b, _| { - b.iter(|| deserialize_with_interning(&protos)); - }, - ); - } - - group.finish(); -} - -fn bench_memory(c: &mut Criterion) { - let mut group = c.benchmark_group("manifest_memory"); - let n = num_fragments(); - - for num_fields in [10, 50] { - let protos = make_uniform_pb_fragments(n, num_fields); - - let no_intern = deserialize_without_interning(&protos); - let with_intern = deserialize_with_interning(&protos); - - let size_no_intern = no_intern.deep_size_of(); - let size_with_intern = with_intern.deep_size_of(); - - eprintln!( - "\n[{} fragments, {} fields] Memory without interning: {:.2} MB", - n, - num_fields, - size_no_intern as f64 / 1_048_576.0 - ); - eprintln!( - "[{} fragments, {} fields] Memory with interning: {:.2} MB", - n, - num_fields, - size_with_intern as f64 / 1_048_576.0 - ); - eprintln!( - "[{} fragments, {} fields] Savings: {:.2} MB ({:.1}%)", - n, - num_fields, - (size_no_intern - size_with_intern) as f64 / 1_048_576.0, - (1.0 - size_with_intern as f64 / size_no_intern as f64) * 100.0 - ); - - // Benchmark deep_size_of measurement itself (sanity check) - group.bench_with_input( - BenchmarkId::new("deep_size_of_interned", num_fields), - &num_fields, - |b, _| { - b.iter(|| with_intern.deep_size_of()); - }, - ); - - drop(no_intern); - drop(with_intern); - } - - group.finish(); -} - -#[cfg(target_os = "linux")] -criterion_group!( - name = benches; - config = Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new(100, pprof::criterion::Output::Flamegraph(None))); - targets = bench_deserialization, bench_memory -); -#[cfg(not(target_os = "linux"))] -criterion_group!(benches, bench_deserialization, bench_memory); -criterion_main!(benches); diff --git a/vendor/lance-table/benches/row_id_index.rs b/vendor/lance-table/benches/row_id_index.rs deleted file mode 100644 index f73251a0..00000000 --- a/vendor/lance-table/benches/row_id_index.rs +++ /dev/null @@ -1,323 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -// TODO: -// - [x] Create base cases with HashMap -// - [x] Create on-disk size measurement -// - [x] Create different cases for the index. Ideal, 25% deletions, 80% deletions + compaction. -// - [ ] Create a benchmark for the get method -// - [x] Average over all valid values -// - [ ] Time to get a value that is not in the index -// - [ ] Create a benchmark for the new method (building the in-memory index) -// Optional: -// - [ ] Create in-memory size measurement (if possible) - -// Questions: -// How can I write out the file? Where should I put it? -// How can I take a argument to set the size of the index? - -use std::{collections::HashMap, io::Write, ops::Range, sync::Arc}; - -use arrow_array::{RecordBatch, UInt64Array}; -use arrow_schema::{DataType, Field, Schema}; -use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; - -use lance_core::utils::address::RowAddress; -use lance_core::utils::deletion::DeletionVector; -use lance_io::ReadBatchParams; -use lance_table::rowids::FragmentRowIdIndex; -use lance_table::{ - rowids::{RowIdIndex, RowIdSequence, write_row_ids}, - utils::stream::{RowIdAndDeletesConfig, apply_row_id_and_deletes}, -}; - -fn make_sequence(row_id_range: Range, deletions: usize) -> RowIdSequence { - let mut sequence = RowIdSequence::from(row_id_range); - - // Delete every other row - let delete_ids = sequence - .iter() - .step_by(2) - .take(deletions) - .collect::>(); - sequence.delete(delete_ids); - - sequence -} - -fn make_frag_sequences( - num_rows: u64, - num_frags: u64, - percent_deletion: f32, -) -> Vec<(u32, Arc)> { - let rows_per_frag = num_rows / num_frags; - let mut start = 0; - (0..num_frags) - .map(|i| { - let sequence = make_sequence( - start..(start + rows_per_frag), - (rows_per_frag as f32 * percent_deletion) as usize, - ); - start += rows_per_frag; - (i as u32, Arc::new(sequence)) - }) - .collect() -} - -// For range of values -// https://bheisler.github.io/criterion.rs/book/user_guide/benchmarking_with_inputs.html - -fn num_rows() -> u64 { - std::env::var("BENCH_NUM_ROWS") - .map(|s| s.parse().unwrap()) - .unwrap_or(1_000_000) -} - -struct SizeStats { - structure: String, - percent_deletions: f32, - size: u64, -} - -struct SizeStatsFile { - file: Option, -} - -impl SizeStatsFile { - fn new() -> Self { - if let Ok(path) = std::env::var("BENCH_SIZE_STATS_FILE") { - let mut file = std::fs::File::create(path).unwrap(); - // Header row - writeln!(file, "structure,percent_deletions,size").unwrap(); - Self { file: Some(file) } - } else { - Self { file: None } - } - } - - fn write_row(&mut self, stats: SizeStats) { - if let Some(file) = &mut self.file { - writeln!( - file, - "\"{}\",{},{}", - stats.structure, stats.percent_deletions, stats.size - ) - .unwrap(); - } - } -} - -fn bench_creation(c: &mut Criterion) { - let mut group = c.benchmark_group("row_id_index_creation"); - let mut stats_file = SizeStatsFile::new(); - - for percent_deletions in [0.0, 0.25, 0.5] { - let sequences = make_frag_sequences(num_rows(), 100, percent_deletions); - - let fragment_indices: Vec = sequences - .iter() - .map(|(frag_id, sequence)| FragmentRowIdIndex { - fragment_id: *frag_id, - row_id_sequence: sequence.clone(), - deletion_vector: Arc::new(DeletionVector::default()), - }) - .collect(); - - group.bench_with_input( - BenchmarkId::new("BuildIndex", percent_deletions), - &percent_deletions, - |b, _| { - b.iter(|| { - let _index = RowIdIndex::new(&fragment_indices).unwrap(); - }); - }, - ); - - // Measure size of index - { - let mut size = 0; - for (_frag_id, sequence) in &sequences { - size += write_row_ids(sequence).len() as u64; - } - let stats = SizeStats { - structure: "RowIdIndex".to_string(), - percent_deletions, - size, - }; - stats_file.write_row(stats); - } - - // TODO: we should compare tombstoned vs compacted. We don't mind the - // regression in the tombstoned case, but we want to see the improvement - // in the compacted case. - - // TODO: collect size of sequences when serialized - - // TODO: also show building a BTreeMap and HashMap - - let flat_data = sequences - .iter() - .map(|(frag_id, sequence)| { - let row_ids = sequence.iter().collect::>(); - let row_addresses = (0..sequence.len()) - .map(|i| RowAddress::new_from_parts(*frag_id, i as u32)) - .map(u64::from) - .collect::>(); - (row_ids, row_addresses) - }) - .collect::>(); - - // Size of flat data is just 16 bytes per row - let size = flat_data - .iter() - .map(|(ids, _addresses)| ids.len() * 16) - .sum::() as u64; - let stats = SizeStats { - structure: "FlatData".to_string(), - percent_deletions, - size, - }; - stats_file.write_row(stats); - - group.bench_with_input( - BenchmarkId::new("BuildHashMap", percent_deletions), - &percent_deletions, - |b, _| { - b.iter(|| { - let mut index = HashMap::new(); - index.extend(flat_data.iter().flat_map(|(ids, addresses)| { - ids.iter().copied().zip(addresses.iter().copied()) - })); - }); - }, - ); - } - - group.finish(); -} - -fn bench_get_single(c: &mut Criterion) { - let mut group = c.benchmark_group("row_id_index_get_single"); - - for percent_deletions in [0.0, 0.02, 0.25, 0.5, 0.8] { - let sequences = make_frag_sequences(num_rows(), 100, percent_deletions); - - let fragment_indices: Vec = sequences - .iter() - .map(|(frag_id, sequence)| FragmentRowIdIndex { - fragment_id: *frag_id, - row_id_sequence: sequence.clone(), - deletion_vector: Arc::new(DeletionVector::default()), - }) - .collect(); - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - let mut i = 0; - let total_rows: u64 = num_rows(); - let mut next_id = || { - let id = i; - i += 241861; - i %= total_rows; - id - }; - - group.bench_with_input( - BenchmarkId::new("GetIndex", percent_deletions), - &percent_deletions, - |b, _| { - b.iter(|| { - let _ = index.get(next_id()); - }); - }, - ); - - let flat_data = sequences - .iter() - .map(|(frag_id, sequence)| { - let row_ids = sequence.iter().collect::>(); - let row_addresses = (0..sequence.len()) - .map(|i| RowAddress::new_from_parts(*frag_id, i as u32)) - .map(u64::from) - .collect::>(); - (row_ids, row_addresses) - }) - .collect::>(); - - let index = - { - let mut index = HashMap::new(); - index.extend(flat_data.iter().flat_map(|(ids, addresses)| { - ids.iter().copied().zip(addresses.iter().copied()) - })); - index - }; - - group.bench_with_input( - BenchmarkId::new("GetHashMap", percent_deletions), - &percent_deletions, - |b, _| { - b.iter(|| { - for i in 0..num_rows() { - let _ = index.get(&i); - } - }); - }, - ); - } - - group.finish(); -} - -fn bench_apply_row_id(c: &mut Criterion) { - let mut group = c.benchmark_group("apply_row_id"); - - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new( - "value", - DataType::UInt64, - false, - )])), - vec![Arc::new(UInt64Array::from( - (0..num_rows()).collect::>(), - ))], - ) - .unwrap(); - - let config = RowIdAndDeletesConfig { - params: ReadBatchParams::default(), - with_row_id: true, - with_row_addr: false, - with_row_last_updated_at_version: false, - with_row_created_at_version: false, - deletion_vector: None, - row_id_sequence: None, - last_updated_at_sequence: None, - created_at_sequence: None, - make_deletions_null: false, - total_num_rows: num_rows() as u32, - }; - - group.bench_function("ApplyRowId", |b| { - let batch = batch.clone(); - b.iter(|| { - let _ = apply_row_id_and_deletes(batch.clone(), 0, 0, &config); - }); - }); - - group.finish(); -} - -#[cfg(target_os = "linux")] -criterion_group!( - name = benches; - config=Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new(100, pprof::criterion::Output::Flamegraph(None))); - targets=bench_creation, bench_get_single, bench_apply_row_id); -#[cfg(not(target_os = "linux"))] -criterion_group!( - benches, - bench_creation, - bench_get_single, - bench_apply_row_id -); -criterion_main!(benches); diff --git a/vendor/lance-table/build.rs b/vendor/lance-table/build.rs deleted file mode 100644 index 03216636..00000000 --- a/vendor/lance-table/build.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::io::Result; - -fn main() -> Result<()> { - println!("cargo:rerun-if-changed=protos"); - - #[cfg(feature = "protoc")] - // Use vendored protobuf compiler if requested. - unsafe { - std::env::set_var("PROTOC", protobuf_src::protoc()); - } - - let mut prost_build = prost_build::Config::new(); - prost_build.extern_path(".lance.file", "::lance_file::format::pb"); - prost_build.protoc_arg("--experimental_allow_proto3_optional"); - prost_build.enable_type_names(); - prost_build.compile_protos( - &[ - "./protos/table.proto", - "./protos/transaction.proto", - "./protos/rowids.proto", - ], - &["./protos"], - )?; - - Ok(()) -} diff --git a/vendor/lance-table/protos/AGENTS.md b/vendor/lance-table/protos/AGENTS.md deleted file mode 100644 index 23aef9fc..00000000 --- a/vendor/lance-table/protos/AGENTS.md +++ /dev/null @@ -1,18 +0,0 @@ -# Protobuf Guidelines - -Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. - -## Compatibility - -- All changes must be backwards compatible. Never re-use or change field numbers of existing fields. - -## Schema Design - -- Use `optional` when you need to distinguish "not set" from "zero value" — `optional` enables presence tracking (`has_*` methods) and maps to `Option` in Rust. Bare proto3 fields have no presence semantics: they always hold a value (defaulting to zero), so you cannot tell if the sender explicitly set them. -- Use structured message types (e.g., `BasePath`) instead of plain scalars, and scope fields to operation-specific messages (e.g., `InsertTransaction`) rather than generic top-level ones. -- Don't duplicate data across messages — store each fact once and derive relationships. Prefer parallel sequences over maps when keys already exist in another field. - -## Documentation - -- Document the semantic meaning of both present and absent states for `optional` fields — explain when each case applies. -- Use precise domain terminology in field descriptions — avoid ambiguous abbreviations or terms that collide with domain concepts. diff --git a/vendor/lance-table/protos/CLAUDE.md b/vendor/lance-table/protos/CLAUDE.md deleted file mode 100644 index 23aef9fc..00000000 --- a/vendor/lance-table/protos/CLAUDE.md +++ /dev/null @@ -1,18 +0,0 @@ -# Protobuf Guidelines - -Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. - -## Compatibility - -- All changes must be backwards compatible. Never re-use or change field numbers of existing fields. - -## Schema Design - -- Use `optional` when you need to distinguish "not set" from "zero value" — `optional` enables presence tracking (`has_*` methods) and maps to `Option` in Rust. Bare proto3 fields have no presence semantics: they always hold a value (defaulting to zero), so you cannot tell if the sender explicitly set them. -- Use structured message types (e.g., `BasePath`) instead of plain scalars, and scope fields to operation-specific messages (e.g., `InsertTransaction`) rather than generic top-level ones. -- Don't duplicate data across messages — store each fact once and derive relationships. Prefer parallel sequences over maps when keys already exist in another field. - -## Documentation - -- Document the semantic meaning of both present and absent states for `optional` fields — explain when each case applies. -- Use precise domain terminology in field descriptions — avoid ambiguous abbreviations or terms that collide with domain concepts. diff --git a/vendor/lance-table/protos/ann.proto b/vendor/lance-table/protos/ann.proto deleted file mode 100644 index c9d3b4dc..00000000 --- a/vendor/lance-table/protos/ann.proto +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.pb; - -import "table_identifier.proto"; -import "table.proto"; -import "index.proto"; - -// Serialized vector query parameters. -message VectorQueryProto { - // Query vector as Arrow IPC bytes (supports Float16, Float32, Float64, UInt8, etc.) - bytes query_vector_arrow_ipc = 1; - string column = 2; - uint32 k = 3; - optional float lower_bound = 4; - optional float upper_bound = 5; - optional uint32 minimum_nprobes = 6; - optional uint32 maximum_nprobes = 7; - optional uint32 ef = 8; - optional uint32 refine_factor = 9; - // Distance metric type. Absent means None (use the index's default metric). - optional lance.index.pb.VectorMetricType metric_type = 10; - bool use_index = 11; - optional float dist_q_c = 12; - optional int32 query_parallelism = 13; -} - -// Serializable form of ANNIvfSubIndexExec — the IVF sub-index search node. -// -// The prefilter child ExecutionPlan is serialized by DataFusion's codec -// automatically via children() / with_new_children(). The prefilter_type -// field tells the decoder which PreFilterSource variant to use when -// reconstructing from the deserialized child inputs. -message ANNIvfSubIndexExecProto { - enum PreFilterType { - NONE = 0; - FILTERED_ROW_IDS = 1; - SCALAR_INDEX_QUERY = 2; - } - - VectorQueryProto query = 1; - lance.datafusion.TableIdentifier table = 2; - repeated lance.table.IndexMetadata indices = 3; - PreFilterType prefilter_type = 4; -} - -// Serializable form of ANNIvfPartitionExec — the IVF centroid routing node. -message ANNIvfPartitionExecProto { - VectorQueryProto query = 1; - lance.datafusion.TableIdentifier table = 2; - repeated string index_uuids = 3; -} diff --git a/vendor/lance-table/protos/encodings_v2_0.proto b/vendor/lance-table/protos/encodings_v2_0.proto deleted file mode 100644 index acfc1508..00000000 --- a/vendor/lance-table/protos/encodings_v2_0.proto +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.encodings; - -import "google/protobuf/empty.proto"; - -// This file contains a specification for encodings that can be used -// to store and load Arrow data into a Lance file for the 2.0 format. It -// has been superseded by encodings21.proto which is used for the 2.1 format. -// -// # Types -// -// This file assumes the user wants to load data into Arrow arrays and -// explains how to map Arrow arrays into Lance files. Encodings are divided -// into "array encoding" (which maps to an Arrow array and may contain multiple -// buffers) and "buffer encoding" (which encodes a single buffer of data). -// -// # Encoding Tree -// -// Most encodings are layered on top of each other. These form a tree of -// encodings with a single root node. To encode an array you will typically -// start with the root node and then take the output from that root encoding -// and feed it into child encodings. The decoding process works in reverse. -// -// # Multi-column Encodings -// -// Some Arrow arrays will map to more than one column of Lance data. For -// example, struct arrays and list arrays. This file only contains encodings -// for a single column. However, it does describe how multi-column arrays can -// be encoded. - -// A pointer to a buffer in a Lance file -// -// A writer can place a buffer in three different locations. The buffer -// can go in the data page, in the column metadata, or in the file metadata. -// The writer is free to choose whatever is most appropriate (for example, a dictionary -// that is shared across all pages in a column will probably go in the column -// metadata). This specification does not dictate where the buffer should go. -message Buffer { - // The index of the buffer in the collection of buffers - uint32 buffer_index = 1; - // The collection holding the buffer - enum BufferType { - // The buffer is stored in the data page itself - page = 0; - // The buffer is stored in the column metadata - column = 1; - // The buffer is stored in the file metadata - file = 2; - }; - BufferType buffer_type = 2; -} - -// An encoding that adds nullability to another array encoding -// -// This can wrap any array encoding and add nullability information -message Nullable { - message NoNull { - ArrayEncoding values = 1; - } - message AllNull {} - message SomeNull { - ArrayEncoding validity = 1; - ArrayEncoding values = 2; - } - oneof nullability { - // The array has no nulls and there is a single buffer needed - NoNull no_nulls = 1; - // The array may have nulls and we need two buffers - SomeNull some_nulls = 2; - // All values are null (no buffers needed) - AllNull all_nulls = 3; - } -} - -// An array encoding for variable-length list fields -message List { - // An array containing the offsets into an items array. - // - // This array will have num_rows items and will never - // have nulls. - // - // If the list at index i is not null then offsets[i] will - // contain `base + len(list)` where `base` is defined as: - // i == 0: 0 - // i > 0: (offsets[i-1] % null_offset_adjustment) - // - // To help understand we can consider the following example list: - // [ [A, B], null, [], [C, D, E] ] - // - // The offsets will be [2, ?, 2, 5] - // - // If the incoming list at index i IS null then offsets[i] will - // contain `base + len(list) + null_offset_adjustment` where `base` - // is defined the same as above. - // - // To complete the above example let's assume that `null_offset_adjustment` - // is 7. Then the offsets will be [2, 9, 2, 5] - // - // If there are no nulls then the offsets we write here are exactly the - // same as the offsets in an Arrow list array (except we omit the leading - // 0 which is redundant) - // - // The reason we do this is so that reading a single list at index i only - // requires us to load the indices at i and i-1. - // - // If the offset at index i is greater than `null_offset_adjustment`` - // then the list at index i is null. - // - // Otherwise the length of the list is `offsets[i] - base` where - // base is defined the same as above. - // - // Let's consider our example offsets: [2, 9, 2, 5] - // - // We can take any range of lists and determine how many list items are - // referenced by the sublist. - // - // 0..3: [_, 5] -> items 0..5 (base = 0* and end is 5) - // 0..2: [_, 2] -> items 0..2 (base = 0* and end is 2) - // 0..1: [_, 9] -> items 0..2 (base = 0* and end is 9 % 7) - // 1..3: [2, 5] -> items 2..5 (base = 2 and end is 5) - // 1..2: [2, 2] -> items 2..2 (base = 2 and end is 2) - // 2..3: [9, 5] -> items 2..5 (base = 9 % 7 and end is 5) - // - // * When the start of our range is the 0th item the base is always 0 and we only - // need to load a single index from disk to determine the range. - // - // The data type of the offsets array is flexible and does not need - // to match the data type of the destination array. Please note that the offsets - // array is very likely to be efficiently encoded by bit packing deltas. - ArrayEncoding offsets = 1; - // If a list is null then we add this value to the offset - // - // This value must be greater than the length of the items so that - // (offset + null_offset_adjustment) is never used by a non-null list. - // - // Note that this value cannot be equal to the length of the items - // because then a page with a single list would store [ X ] and we - // couldn't know if that is a null list or a list with X items. - // - // Therefore, the best choice for this value is 1 + # of items. - // Choosing this will maximize the bit packing that we can apply to the offsets. - uint64 null_offset_adjustment = 2; - // How many items are referenced by these offsets. This is needed in - // order to determine which items pages map to this offsets page. - uint64 num_items = 3; -} - -// An array encoding for fixed-size list fields -message FixedSizeList { - /// The number of items in each list - uint32 dimension = 1; - /// True if the list is nullable - bool has_validity = 3; - /// The items in the list - ArrayEncoding items = 2; -} - -message Compression { - string scheme = 1; - optional int32 level = 2; -} - -// Fixed width items placed contiguously in a buffer -message Flat { - // the number of bits per value, must be greater than 0, does - // not need to be a multiple of 8 - uint64 bits_per_value = 1; - // the buffer of values - Buffer buffer = 2; - // The Compression message can specify the compression scheme (e.g. zstd) and any - // other information that is needed for decompression. - // - // If this array is compressed then the bits_per_value refers to the uncompressed - // data. - Compression compression = 3; -} - -// Compression algorithm where all values have a constant value -message Constant { - // The value (TODO: define encoding for literals?) - bytes value = 1; -} - -// Items are bitpacked in a buffer -message Bitpacked { - // the number of bits used for a value in the buffer - uint64 compressed_bits_per_value = 1; - - // the number of bits of the uncompressed value. e.g. for a u32, this will be 32 - uint64 uncompressed_bits_per_value = 2; - - // The items in the list - Buffer buffer = 3; - - // Whether or not a sign bit is included in the bitpacked value - bool signed = 4; -} - -// Items are bitpacked in a buffer -message BitpackedForNonNeg { - // the number of bits used for a value in the buffer - uint64 compressed_bits_per_value = 1; - - // the number of bits of the uncompressed value. e.g. for a u32, this will be 32 - uint64 uncompressed_bits_per_value = 2; - - // The items in the list - Buffer buffer = 3; -} - -// Opaque bitpacking variant where the bits per value are stored inline in the chunks themselves -message InlineBitpacking { - // the number of bits of the uncompressed value. e.g. for a u32, this will be 32 - uint64 uncompressed_bits_per_value = 2; -} - -// Transparent bitpacking variant where the number of bits per value is fixed through the whole buffer -message OutOfLineBitpacking { - // the number of bits of the uncompressed value. e.g. for a u32, this will be 32 - uint64 uncompressed_bits_per_value = 2; - // The number of compressed bits per value, fixed across the entire buffer - uint64 compressed_bits_per_value = 3; -} - -// An array encoding for shredded structs that will never be null -// -// There is no actual data in this column. -// -// TODO: Struct validity bitmaps will be placed here. -message SimpleStruct {} - -// An array encoding for binary fields -message Binary { - ArrayEncoding indices = 1; - ArrayEncoding bytes = 2; - uint64 null_adjustment = 3; -} - -message Variable { - uint32 bits_per_offset = 1; -} - -message Fsst { - ArrayEncoding binary = 1; - bytes symbol_table = 2; -} - -// An array encoding for dictionary-encoded fields -message Dictionary { - ArrayEncoding indices = 1; - ArrayEncoding items = 2; - uint32 num_dictionary_items = 3; -} - -message PackedStruct { - repeated ArrayEncoding inner = 1; - Buffer buffer = 2; -} - -message PackedStructFixedWidthMiniBlock { - ArrayEncoding Flat = 1; - repeated uint32 bits_per_values = 2; -} - -message FixedSizeBinary { - ArrayEncoding bytes = 1; - uint32 byte_width = 2; -} - -message Block { - string scheme = 1; -} - -// Run-Length Encoding for miniblock format -message Rle { - // Number of bits per value (8, 16, 32, 64, or 128) - uint64 bits_per_value = 1; -} - -// Byte Stream Split encoding for floating point values -message ByteStreamSplit { - // Number of bits per value (32 for float, 64 for double) - uint64 bits_per_value = 1; -} - -// General miniblock encoding - wraps another miniblock encoding with compression -message GeneralMiniBlock { - // The inner miniblock encoding (e.g., Rle, Bitpacked, etc.) - ArrayEncoding inner = 1; - // The compression scheme to apply to the miniblock buffers - Compression compression = 2; -} - -// Encodings that decode into an Arrow array -message ArrayEncoding { - oneof array_encoding { - Flat flat = 1; - Nullable nullable = 2; - FixedSizeList fixed_size_list = 3; - List list = 4; - SimpleStruct struct = 5; - Binary binary = 6; - Dictionary dictionary = 7; - Fsst fsst = 8; - PackedStruct packed_struct = 9; - Bitpacked bitpacked = 10; - FixedSizeBinary fixed_size_binary = 11; - BitpackedForNonNeg bitpacked_for_non_neg = 12; - Constant constant = 13; - InlineBitpacking inline_bitpacking = 14; - OutOfLineBitpacking out_of_line_bitpacking = 15; - Variable variable = 16; - PackedStructFixedWidthMiniBlock packed_struct_fixed_width_mini_block = 17; - Block block = 18; - Rle rle = 19; - GeneralMiniBlock general_mini_block = 20; - ByteStreamSplit byte_stream_split = 21; - } -} - -// Wraps a column with a zone map index that can be used -// to apply pushdown filters -message ZoneIndex { - uint32 rows_per_zone = 1; - Buffer zone_map_buffer = 2; - ColumnEncoding inner = 3; -} - -// Marks a column as blob data. It will contain a packed struct -// with fields position and size (u64) -message Blob { - ColumnEncoding inner = 1; -} - -// Encodings that describe a column of values -message ColumnEncoding { - oneof column_encoding { - // No special encoding, just column values - google.protobuf.Empty values = 1; - ZoneIndex zone_index = 2; - Blob blob = 3; - } -} diff --git a/vendor/lance-table/protos/encodings_v2_1.proto b/vendor/lance-table/protos/encodings_v2_1.proto deleted file mode 100644 index 46fd012f..00000000 --- a/vendor/lance-table/protos/encodings_v2_1.proto +++ /dev/null @@ -1,511 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.encodings21; - -// This file contains a specification for encodings that can be used -// to store and load Arrow data into a Lance file for the 2.1 format. -// -// # Types -// -// This file assumes the user wants to load data into Arrow arrays and -// explains how to map Arrow arrays into Lance files. Encodings are divided -// into "structural encodings" (which are used to encode the structure of the -// data such as any list or struct layers) and "compressive encodings" (which -// are used to compress the actual data values). -// -// # Standardized Interpretation of Counting Terms -// -// When working with 2.1 encodings we have a number of different "counting terms" and it can be -// difficult to understand what we mean when we are talking about a "number of values". Here is -// a standard interpretation of these terms: -// -// To understand these definitions consider a data type FIXED_SIZE_LIST>. -// -// A "value" is an abstract term when we aren't being specific. -// -// - num_rows: This is the highest level counting term. A single row includes everything in the -// fixed size list. This is what the user asks for when they asks for a range of rows. -// - num_elements: The number of elements is the number of rows multiplied by the dimension of any -// fixed size list wrappers. This is what you get when you flatten the FSL layer and -// is the starting point for structural encoding. Note that an element can be a list -// value or a single primitive value. -// - num_items: The number of items is the number of values in the repetition and definition vectors -// after everything has been flattened. -// - num_visible_items: The number of visible items is the number of items after invisible items -// have been removed. Invisible items are rep/def levels that don't correspond to an -// actual value. - - -// # Structural Encodings -// -// The following message are used to describe the structural encoding of the -// data. In this document, we refer to these structural encodings as layouts. - -// Repetition and definition levels are described in more detail elsewhere. As we peel through -// the structure of an array we will encounter layers of struct and list. Each of these layers -// potentially adds a new level to the repetition and definition levels. This message describes -// the meaning of each layer. -enum RepDefLayer { - // Should never be used, included for debugging purporses and general protobuf best practice - REPDEF_UNSPECIFIED = 0; - // All values are valid (can be primitive or struct) - REPDEF_ALL_VALID_ITEM = 1; - // All list values are valid - REPDEF_ALL_VALID_LIST = 2; - // There are one or more null items (can be primitive or struct) - REPDEF_NULLABLE_ITEM = 3; - // A list layer with null lists but no empty lists - REPDEF_NULLABLE_LIST = 4; - // A list layer with empty lists but no null lists - REPDEF_EMPTYABLE_LIST = 5; - // A list layer with both empty lists and null lists - REPDEF_NULL_AND_EMPTY_LIST = 6; -} - -// A layout used for pages where the data is small -// -// In this case we can fit many values into a single disk sector and transposing buffers is -// expensive. As a result, we do not transpose the buffers but compress the data into small -// chunks (called mini blocks) which are roughly the size of a disk sector. -// -// The end result is a small amount of read amplification (since we must read an entire page -// at a time) but we have more flexibility in compression and do less work per value when -// compressing and decompressing in bulk. -message MiniBlockLayout { - // Description of the compression of repetition levels (e.g. how many bits per rep) - // - // Optional, if there is no repetition then this field is not present - CompressiveEncoding rep_compression = 1; - // Description of the compression of definition levels (e.g. how many bits per def) - // - // Optional, if there is no definition then this field is not present - CompressiveEncoding def_compression = 2; - // Description of the compression of values - CompressiveEncoding value_compression = 3; - // Description of the compression of the dictionary data - // - // Optional, if there is no dictionary then this field is not present - CompressiveEncoding dictionary = 4; - // Number of items in the dictionary - uint64 num_dictionary_items = 5; - // The meaning of each repdef layer, used to interpret repdef buffers correctly - repeated RepDefLayer layers = 6; - // The number of buffers in each mini-block, this is determined by the compression and does - // NOT include the repetition or definition buffers (the presence of these buffers can be determined - // by looking at the rep_compression and def_compression fields) - uint64 num_buffers = 7; - // The depth of the repetition index. - // - // If there is repetition then the depth must be at least 1. If there are many layers - // of repetition then deeper repetition indices will support deeper nested random access. For - // example, given 5 layers of repetition then the repetition index depth must be at least - // 3 to support access like `rows[50][17][3]`. - // - // We require `repetition_index_depth + 1` u64 values per mini-block to store the repetition - // index if the `repetition_index_depth` is greater than 0. The +1 is because we need to store - // the number of "leftover items" at the end of the chunk. Otherwise, we wouldn't have any way - // to know if the final item in a chunk is valid or not. - uint32 repetition_index_depth = 8; - // The page already records how many rows are in the page. For mini-block we also need to know how - // many "items" are in the page. A row and an item are the same thing unless the page has lists. - uint64 num_items = 9; - - // Since Lance 2.2, miniblocks have larger chunk sizes (>= 64KB) - bool has_large_chunk = 10; -} - -// A layout used for pages where the data is large -// -// In this case the cost of transposing the data is relatively small (compared to the cost of writing the data) -// and so we just zip the buffers together -message FullZipLayout { - // The number of bits of repetition info (0 if there is no repetition) - uint32 bits_rep = 1; - // The number of bits of definition info (0 if there is no definition) - uint32 bits_def = 2; - // The number of bits of value info - // - // Note: we use bits here (and not bytes) for consistency with other encodings. However, in practice, - // there is never a reason to use a bits per value that is not a multiple of 8. The complexity is not - // worth the small savings in space since this encoding is typically used with large values already. - oneof details { - // If this is a fixed width block then we need to have a fixed number of bits per value - uint32 bits_per_value = 3; - // If this is a variable width block then we need to have a fixed number of bits per offset - uint32 bits_per_offset = 4; - } - // The number of items in the page - uint32 num_items = 5; - // The number of visible items in the page - uint32 num_visible_items = 6; - // Description of the compression of values - CompressiveEncoding value_compression = 7; - // The meaning of each repdef layer, used to interpret repdef buffers correctly - repeated RepDefLayer layers = 8; -} - -// A layout used for pages where all (visible) values are the same scalar value. -// -// This generalizes the prior AllNullLayout semantics for file_version >= 2.2. -// -// There may be buffers of repetition and definition information if required in order -// to interpret what kind of nulls are present / which items are visible. -message ConstantLayout { - // The meaning of each repdef layer, used to interpret repdef buffers correctly - repeated RepDefLayer layers = 5; - - // Inline fixed-width scalar value bytes. - // - // This MUST only be used for types where a single non-null element is represented by a single - // fixed-width Arrow value buffer (i.e. no offsets buffer, no child data). - // - // Constraints: - // - MUST be absent for an all-null page - // - MUST be <= 32 bytes if present - optional bytes inline_value = 6; - - // Optional compression algorithm used for the repetition buffer. - // If absent, repetition levels are stored as raw u16 values. - CompressiveEncoding rep_compression = 7; - // Optional compression algorithm used for the definition buffer. - // If absent, definition levels are stored as raw u16 values. - CompressiveEncoding def_compression = 8; - // Number of values in repetition buffer after decompression. - uint64 num_rep_values = 9; - // Number of values in definition buffer after decompression. - uint64 num_def_values = 10; -} - -// A layout where large binary data is encoded externally and only -// the descriptions (position + size) are placed in the page -// -// Repdef information is stored in the descriptions. A description with a size of -// 0 and a position of 0 is an empty value. A description with a size of 0 and a -// non-zero position is a null value and the position is the repdef value. -message BlobLayout { - // The inner layout used to store the descriptions - PageLayout inner_layout = 1; - // The meaning of each repdef layer, used to interpret repdef buffers correctly - // - // The inner layout's repdef layers will always be 1 all valid item layer - repeated RepDefLayer layers = 2; -} - -// Describes the structural encoding of a page -message PageLayout { - oneof layout { - // A layout used for pages where the data is small - MiniBlockLayout mini_block_layout = 1; - // A layout used for pages where all (visible) values are the same scalar value or null. - ConstantLayout constant_layout = 2; - // A layout used for pages where the data is large - FullZipLayout full_zip_layout = 3; - // A layout where large binary data is encoded externally - // and only the descriptions are put in the page - BlobLayout blob_layout = 4; - } -} - -// # Compressive Encodings -// -// These encodings describe how an array is compressed. An encoding may split an -// array into multiple buffers. The buffers can then be compressed further (and split -// into yet more buffers). The entire process forms a tree of encodings with the root -// of the tree being the initial array and the leaves being the final compressed buffers. -// -// # Data blocks and buffers -// -// Data blocks are a simplified version of arrays and represent a collection of buffers grouped -// with some kind of interpretation. Data blocks are the input and output of compressive encodings. -// There are different kinds of data blocks: -// - Fixed width data blocks (e.g. u8, u16, ...) -// - Variable width data blocks (e.g. strings, binary) -// - Struct data blocks (note: this is for packed structs, normal structs are encoded in the structural encoding) -// -// In addition, leaf encodings may output "buffers". These are fully compressed buffers of data that -// are stored in the page and no longer compressed. - -enum CompressionScheme { - COMPRESSION_ALGORITHM_UNSPECIFIED = 0; - COMPRESSION_ALGORITHM_LZ4 = 1; - COMPRESSION_ALGORITHM_ZSTD = 2; -} - -// Compression applied to a single buffer of data -// -// A buffer is the leaf of the compression tree. Unlike data blocks, which can -// be further compressed with a variety of techniques, a buffer cannot be understood -// in any particular way. -// -// A general compression scheme may be applied to a buffer. This is something like -// zstd, lz4, etc. The entire buffer is compressed as a single unit. If this happens -// then any parent encoding becomes opaque, even if it would normally be transparent. -// -// This is a leaf, no further compression is applied to the data. -message BufferCompression { - // A general compression scheme to apply to the buffer - CompressionScheme scheme = 1; - // The compression level - // - // Optional, if not present a scheme-specific default value will be used. - // - // Interpretation of this value depends on the compression scheme. Generally, larger - // values indicate more compression at the expense of more CPU time. - optional int32 level = 2; -} - -// Fixed width items placed contiguously in a single buffer -// -// This is a leaf encoding, there is no compression applied to the data. -// -// This is a transparent encoding by definition. -// -// The input is a fixed-width data block. -// The output is a single buffer. -message Flat { - // the number of bits per value, must be greater than 0, does - // not need to be a multiple of 8 - uint64 bits_per_value = 1; - // The compression applied to the data - optional BufferCompression data = 2; -} - -// Variable width items have the values stored in one buffer and the -// offsets are output as a data block that may be further compressed. -// -// This is a partial leaf encoding. Values are not compressed but -// the offsets may be further compressed. -// -// This is a transparent encoding by definition. -// -// The input is a variable-width data block. -// The output is a single fixed-width data block (the offsets) and -// a single buffer (the values) -message Variable { - // Describes how the offsets data block is compressed - CompressiveEncoding offsets = 1; - // The compression applied to the values - optional BufferCompression values = 2; -} - -// Compression algorithm where all values have a constant value (encoded in the description) -// -// This is a leaf encoding, there is no compression applied to the data. -// -// The input can be any kind of data block. -// There is no output. -message Constant { - // The value (TODO: define encoding for literals?) - optional bytes value = 1; -} - -// A compression scheme in which a single fixed-width block is "packed" into -// a smaller fixed-width block values where each value has fewer bits. -// -// This is typically done by throwing away the most significant bits of each value when -// those bits are all the same. -// -// In this scheme the number of bits per value is fixed across the entire buffer and stored -// in this message. -// -// This is a transparent encoding. -// -// The input is a fixed-width data block. -// The output is a single fixed-width data block. -message OutOfLineBitpacking { - // the number of bits of the uncompressed value. e.g. for a u32, this will be 32 - uint64 uncompressed_bits_per_value = 1; - // The compression used to store the bitpacked values data block - CompressiveEncoding values = 3; -} - -// Bitpacking variant where the bits per value are stored inline in the chunks themselves -// -// This variation of bitpacking allows for the number of bits per value to change throughout the -// buffer, which makes the compression more robust to outliers. -// -// This is an opaque encoding. -// -// The input is a fixed-width data block. -// The output is a single buffer. -message InlineBitpacking { - // the number of bits of the uncompressed value. e.g. for a u32, this will be 32 - uint64 uncompressed_bits_per_value = 1; - // The compression applied to the values - optional BufferCompression values = 2; -} - -// A compression scheme for variable-width data -// -// A small dictionary (referred to as a "symbol table") is used to compress the values. -// In this scheme there is a single symbol table for the entire page and it is stored in the -// encoding description itself. -// -// This is a transparent encoding. -// -// The input is a variable-width data block. -// The output is a single variable-width data block. -message Fsst { - // The FSST symbol table - bytes symbol_table = 1; - // The compression used to store the compressed values data block - CompressiveEncoding values = 2; -} - -// A compression scheme where common values are stored in a dictionary and the values are -// encoded as indices into the dictionary. -// -// This is an opaque encoding unless the dictionary is considered metadata. -// -// The input is a any kind of data block. -// There are two outputs: -// - A data block of the same kind as the input (the dictionary) -// - A fixed-width data block containing the indices into the dictionary. -message Dictionary { - // The compression used to store the indices data block - CompressiveEncoding indices = 1; - // The compression used to store the dictionary items data block - CompressiveEncoding items = 2; - // The number of items in the dictionary - uint32 num_dictionary_items = 3; -} - -// A compression scheme where runs of common values are encoded as a single value and a count -// -// This is an opaque encoding unless the run lengths are considered metadata. -// -// The input is a single data block of any kind. -// There are two outputs: -// - A data block of the same kind as the input (the run values) -// - A fixed-width data block containing the lengths of the runs -message Rle { - // The compression used to store the run values data block - CompressiveEncoding values = 1; - // The compression used to store the run lengths data block - CompressiveEncoding run_lengths = 2; -} - -// Converts a fixed-size-list of values into a flattened list of values -// -// This encoding does not actually compress the data, it just flattens out the FSL layers. -// -// This is a transparent encoding. -// -// The input is a single block of fixed-width data (with a wide width and few items) -// The output is a single block of fixed-width data (with a narrow width and many items) -message FixedSizeList { - // The number of items in this layer of FSL - uint64 items_per_value = 1; - // Whether or not there is a validity buffer - bool has_validity = 3; - // The compression used to store the flattened values data block - CompressiveEncoding values = 2; -} - -// Packs a struct containing only fixed-width children into a single fixed-width data block -// -// The children are concatenated row by row and stored as a single fixed-width buffer. This is -// the legacy packed struct representation and remains available for backwards compatibility. -message PackedStruct { - // The number of bits contributed by each child field in the packed row - repeated uint64 bits_per_value = 1; - // The compression used to store the packed fixed-width values - CompressiveEncoding values = 2; -} - -// Variable-width packed struct encoding (2.2 extension) -// -// Each child value is compressed independently before being transposed into -// a row-major layout. This preserves per-field compression boundaries at the -// cost of disabling mini-block compression. Readers must prefer this field -// when present and fall back to the legacy encoding otherwise. -message VariablePackedStruct { - // Per-field encoding metadata in struct order - repeated FieldEncoding fields = 1; - - // Encoding description for a single child field - message FieldEncoding { - // Compression applied to individual field values before transposition - CompressiveEncoding value = 1; - oneof layout { - // Bit width of each compressed value (when fixed width) - uint64 bits_per_value = 2; - // Bit width of the length prefix for variable-width compressed values - uint64 bits_per_length = 3; - } - } -} - -// A compression scheme that wraps the underlying data with general compression -// -// Note: The application of wrapped compression will depend on the layout of the data. -// If we apply it to mini-block data then we compress entire mini-blocks. If we apply -// it to full-zip data then we compress each value individually. -// -// Note: Wrapped compression is somewhat unique at the moment as it is applied to the -// output of the inner encoding and not the input like all other compressive encodings. -// -// Note: General compression can usually be applied in two spots. We can apply -// it to individual buffers or we can apply it here, to the entire array. -// -// For example, let's say we are storing mini-blocks of strings and we are using -// FSST and bitpacking the offsets. We have something like this... -// -// WRAPPED(†3) -> FSST -> VARIABLE -(offsets)-> INLINE_BITPACKING -(data)-> FLAT -> BUFFER (†1) -// -(data)-> BUFFER (†2) -// -// General compression can be applied at †1, †2, or †3 (or any combination of these). -// -// If we apply it at †1 then we apply it just to the bitpacked offsets -// If we apply it at †2 then we apply it just to the FSST compressed data -// If we apply it at †3 then we apply it to the entire mini-block (both offsets and data) -// -// The input is a single data block of any kind. -// The output is a single data block of the same kind as the input. -message General { - // The compression to apply to the values - BufferCompression compression = 1; - // The compression used to store the output data block - CompressiveEncoding values = 3; -} - -// A compression scheme where fixed-width values are transposed into a series of byte streams -// -// This is commonly used for floating point values where the upper bits (the mantissa) have a -// significantly different meaning than the lower bits. By splitting the values into byte streams -// we group the mantissa bits together and the exponent bits together. The end result is typically -// more compressible. -// -// Note that this encoding is mostly useful when combined with other encodings. It does not do any -// compression on its own. -// -// This is an opaque encoding. -// -// The input is a fixed-width data block -// The output is a single fixed-width data block -message ByteStreamSplit { - // The compression used to store the values - CompressiveEncoding values = 1; -} - -// An encoding that compresses a data block into buffers -message CompressiveEncoding { - oneof compression { - Flat flat = 1; - Variable variable = 2; - Constant constant = 3; - OutOfLineBitpacking out_of_line_bitpacking = 4; - InlineBitpacking inline_bitpacking = 5; - Fsst fsst = 6; - Dictionary dictionary = 7; - Rle rle = 8; - ByteStreamSplit byte_stream_split = 9; - General general = 10; - FixedSizeList fixed_size_list = 11; - PackedStruct packed_struct = 12; - VariablePackedStruct variable_packed_struct = 13; - } -} diff --git a/vendor/lance-table/protos/file.proto b/vendor/lance-table/protos/file.proto deleted file mode 100644 index cbc65864..00000000 --- a/vendor/lance-table/protos/file.proto +++ /dev/null @@ -1,207 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.file; - -// A file descriptor that describes the contents of a Lance file -message FileDescriptor { - // The schema of the file - Schema schema = 1; - // The number of rows in the file - uint64 length = 2; -} - -// A schema which describes the data type of each of the columns -message Schema { - // All fields in this file, including the nested fields. - repeated lance.file.Field fields = 1; - // Schema metadata. - map metadata = 5; -} - -// Metadata of one Lance file. -message Metadata { - // 4 was used for StatisticsMetadata in the past, but has been moved to - // prevent a bug in older readers. - reserved 4; - - // Position of the manifest in the file. If it is zero, the manifest is stored - // externally. - uint64 manifest_position = 1; - - // Logical offsets of each chunk group, i.e., number of the rows in each - // chunk. - repeated int32 batch_offsets = 2; - - // The file position that page table is stored. - // - // A page table is a matrix of N x M x 2, where N = num_fields, and M = - // num_batches. Each cell in the table is a pair of of the page. Both position and length are int64 values. The - // of all the pages in the same column are then - // contiguously stored. - // - // Every field that is a part of the file will have a run in the page table. - // This includes struct columns, which will have a run of length 0 since - // they don't store any actual data. - // - // For example, for the column 5 and batch 4, we have: - // ```text - // position = page_table[5][4][0]; - // length = page_table[5][4][1]; - // ``` - uint64 page_table_position = 3; - - message StatisticsMetadata { - // The schema of the statistics. - // - // This might be empty, meaning there are no statistics. It also might not - // contain statistics for every field. - repeated Field schema = 1; - - // The field ids of the statistics leaf fields. - // - // This plays a similar role to the `fields` field in the DataFile message. - // Each of these field ids corresponds to a field in the stats_schema. There - // is one per column in the stats page table. - repeated int32 fields = 2; - - // The file position of the statistics page table - // - // The page table is a matrix of N x 2, where N = length of stats_fields. - // This is the same layout as the main page table, except there is always - // only one batch. - // - // For example, to get the stats column 5, we have: - // ```text - // position = stats_page_table[5][0]; - // length = stats_page_table[5][1]; - // ``` - uint64 page_table_position = 3; - } - - StatisticsMetadata statistics = 5; -} // Metadata - -// Supported encodings. -enum Encoding { - // Invalid encoding. - NONE = 0; - // Plain encoding. - PLAIN = 1; - // Var-length binary encoding. - VAR_BINARY = 2; - // Dictionary encoding. - DICTIONARY = 3; - // Run-length encoding. - RLE = 4; -} - -// Dictionary field metadata -message Dictionary { - /// The file offset for storing the dictionary value. - /// It is only valid if encoding is DICTIONARY. - /// - /// The logic type presents the value type of the column, i.e., string value. - int64 offset = 1; - - /// The length of dictionary values. - int64 length = 2; -} - -// Field metadata for a column. -message Field { - enum Type { - PARENT = 0; - REPEATED = 1; - LEAF = 2; - } - Type type = 1; - - // Fully qualified name. - string name = 2; - /// Field Id. - /// - /// See the comment in `DataFile.fields` for how field ids are assigned. - int32 id = 3; - /// Parent Field ID. If not set, this is a top-level column. - int32 parent_id = 4; - - // Logical types, support parameterized Arrow Type. - // - // PARENT types will always have logical type "struct". - // - // REPEATED types may have logical types: - // * "list" - // * "large_list" - // * "list.struct" - // * "large_list.struct" - // The final two are used if the list values are structs, and therefore the - // field is both implicitly REPEATED and PARENT. - // - // LEAF types may have logical types: - // * "null" - // * "bool" - // * "int8" / "uint8" - // * "int16" / "uint16" - // * "int32" / "uint32" - // * "int64" / "uint64" - // * "halffloat" / "float" / "double" - // * "string" / "large_string" - // * "binary" / "large_binary" - // * "date32:day" - // * "date64:ms" - // * "decimal:128:{precision}:{scale}" / "decimal:256:{precision}:{scale}" - // * "time:{unit}" / "timestamp:{unit}" / "duration:{unit}", where unit is - // "s", "ms", "us", "ns" - // * "dict:{value_type}:{index_type}:false" - string logical_type = 5; - // If this field is nullable. - bool nullable = 6; - - // optional field metadata (e.g. extension type name/parameters) - map metadata = 10; - - bool unenforced_primary_key = 12; - - // Position of this field in the primary key (1-based). - // 0 means the field is part of the primary key but uses schema field id for ordering. - // When set to a positive value, primary key fields are ordered by this position. - uint32 unenforced_primary_key_position = 13; - - // Reserved for future use. Use unenforced_clustering_key_position instead. - bool unenforced_clustering_key = 14; - - // Position of this field in the clustering key (1-based). - // 0 means the field is not part of the clustering key. - uint32 unenforced_clustering_key_position = 15; - - // DEPRECATED ---------------------------------------------------------------- - - // Deprecated: Only used in V1 file format. V2 uses variable encodings defined - // per page. - // - // The global encoding to use for this field. - Encoding encoding = 7; - - // Deprecated: Only used in V1 file format. V2 dynamically chooses when to - // do dictionary encoding and keeps the dictionary in the data files. - // - // The file offset for storing the dictionary value. - // It is only valid if encoding is DICTIONARY. - // - // The logic type presents the value type of the column, i.e., string value. - Dictionary dictionary = 8; - - // Deprecated: optional extension type name, use metadata field - // ARROW:extension:name - string extension_name = 9; - - // Field number 11 was previously `string storage_class`. - // Keep it reserved so older manifests remain compatible while new writers - // avoid reusing the slot. - reserved 11; - reserved "storage_class"; -} diff --git a/vendor/lance-table/protos/file2.proto b/vendor/lance-table/protos/file2.proto deleted file mode 100644 index da0b1d5e..00000000 --- a/vendor/lance-table/protos/file2.proto +++ /dev/null @@ -1,210 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.file.v2; - -import "google/protobuf/any.proto"; -import "google/protobuf/empty.proto"; - -// # Lance v2.X File Format -// -// The Lance file format is a barebones format for serializing columnar data -// into a file. -// -// * Each Lance file contains between 0 and 4Gi columns -// * Each column contains between 0 and 4Gi pages -// * Each page contains between 0 and 2^64 items -// * Different pages within a column can have different items counts -// * Columns may have up to 2^64 items -// * Different columns within a file can have different item counts -// -// The Lance file format does not have any notion of a type system or schemas. -// From the perspective of the file format all data is arbitrary buffers of -// bytes with an extensible metadata block to describe the data. It is up to -// the user to interpret these bytes meaningfully. -// -// Data buffers are written to the file first. These data buffers can be -// referenced from three different places in the file: -// -// * Page encodings can reference data buffers. This is the most common way -// that actual data is stored. -// * Column encodings can reference data buffers. For example, a column encoding -// may reference data buffer(s) containing statistics or dictionaries. -// * Finally, the global buffer offset table can reference data buffers. This -// is useful for storing data that is shared across multiple columns. -// This is also useful for global file metadata (e.g. a schema that describes -// the file) -// -// ## File Layout -// -// Note: the number of buffers (BN) is independent of the number of columns (CN) -// and pages. -// -// Buffers often need to be aligned. 64-byte alignment is common when -// working with SIMD operations. 4096-byte alignment is common when -// working with direct I/O. In order to ensure these buffers are aligned -// writers may need to insert padding before the buffers. -// -// If direct I/O is required then most (but not all) fields described -// below must be sector aligned. We have marked these fields with an -// asterisk for clarity. Readers should assume there will be optional -// padding inserted before these fields. -// -// All footer fields are unsigned integers written with little endian -// byte order. -// -// ├──────────────────────────────────┤ -// | Data Pages | -// | Data Buffer 0* | -// | ... | -// | Data Buffer BN* | -// ├──────────────────────────────────┤ -// | Column Metadatas | -// | |A| Column 0 Metadata* | -// | Column 1 Metadata* | -// | ... | -// | Column CN Metadata* | -// ├──────────────────────────────────┤ -// | Column Metadata Offset Table | -// | |B| Column 0 Metadata Position* | -// | Column 0 Metadata Size | -// | ... | -// | Column CN Metadata Position | -// | Column CN Metadata Size | -// ├──────────────────────────────────┤ -// | Global Buffers Offset Table | -// | |C| Global Buffer 0 Position* | -// | Global Buffer 0 Size | -// | ... | -// | Global Buffer GN Position | -// | Global Buffer GN Size | -// ├──────────────────────────────────┤ -// | Footer | -// | A u64: Offset to column meta 0 | -// | B u64: Offset to CMO table | -// | C u64: Offset to GBO table | -// | u32: Number of global bufs | -// | u32: Number of columns | -// | u16: Major version | -// | u16: Minor version | -// | "LANC" | -// ├──────────────────────────────────┤ -// -// File Layout-End -// -// ## Data Pages -// -// A lot of flexibility is provided in how data is stored. A page's buffers do -// not strictly need to be contiguous on the disk. However, it is recommended -// that buffers within a page be grouped together for best performance. -// -// Data pages should be large. The only time a page should be written to disk -// is when the writer needs to flush the page to disk because it has accumulated -// too much data. Pages are not read in sequential order and if pages are too -// small then the seek overhead (or request overhead) will be problematic. We -// generally advise that pages be at least 8MB or larger. -// -// ## Encodings -// -// Specific encodings are not part of this minimal format. They are provided -// by extensions. Readers and writers should be designed so that encodings can -// be easily added and removed. Ideally, they should allow for this without -// requiring recompilation through some kind of plugin system. - -// The deferred encoding is used to place the encoding itself in a different -// part of the file. This is most commonly used to allow encodings to be shared -// across different columns. For example, when writing a file with thousands of -// columns, where many pages have the exact same encoding, it can be useful -// to cut down on the size of the metadata by using a deferred encoding. -message DeferredEncoding { - // Location of the buffer containing the encoding. - // - // * If sharing encodings across columns then this will be in a global buffer - // * If sharing encodings across pages within a column this could be in a - // column metadata buffer. - // * This could also be a page buffer if the encoding is not shared, needs - // to be written before the file ends, and the encoding is too large to load - // unless we first determine the page needs to be read. This combination - // seems unusual. - uint64 buffer_location = 1; - uint64 buffer_length = 2; -} - -// The encoding is placed directly in the metadata section -message DirectEncoding { - // The bytes that make up the encoding embedded directly in the metadata - // - // This is the most common approach. - bytes encoding = 1; -} - -// An encoding stores the information needed to decode a column or page -// -// For example, it could describe if the page is using bit packing, and how many bits -// there are in each individual value. -// -// At the column level it can be used to wrap columns with dictionaries or statistics. -message Encoding { - oneof location { - // The encoding is stored elsewhere and not part of this protobuf message - DeferredEncoding indirect = 1; - // The encoding is stored within this protobuf message - DirectEncoding direct = 2; - // There is no encoding information - google.protobuf.Empty none = 3; - } -} - -// ## Metadata - -// Each column has a metadata block that is placed at the end of the file. -// These may be read individually to allow for column projection. -message ColumnMetadata { - - // This describes a page of column data. - message Page { - // The file offsets for each of the page buffers - // - // The number of buffers is variable and depends on the encoding. There - // may be zero buffers (e.g. constant encoded data) in which case this - // could be empty. - repeated uint64 buffer_offsets = 1; - // The size (in bytes) of each of the page buffers - // - // This field will have the same length as `buffer_offsets` and - // may be empty. - repeated uint64 buffer_sizes = 2; - // Logical length (e.g. # rows) of the page - uint64 length = 3; - // The encoding used to encode the page - Encoding encoding = 4; - // The priority of the page - // - // For tabular data this will be the top-level row number of the first row - // in the page (and top-level rows should not split across pages). - uint64 priority = 5; - } - // Encoding information about the column itself. This typically describes - // how to interpret the column metadata buffers. For example, it could - // describe how statistics or dictionaries are stored in the column metadata. - Encoding encoding = 1; - // The pages in the column - repeated Page pages = 2; - // The file offsets of each of the column metadata buffers - // - // There may be zero buffers. - repeated uint64 buffer_offsets = 3; - // The size (in bytes) of each of the column metadata buffers - // - // This field will have the same length as `buffer_offsets` and - // may be empty. - repeated uint64 buffer_sizes = 4; -} // Metadata-End - -// ## Where is the rest? -// -// This file format is extremely minimal. It is a building block for -// creating more useful readers and writers and not terribly useful by itself. -// Other protobuf files will describe how this can be extended. \ No newline at end of file diff --git a/vendor/lance-table/protos/filtered_read.proto b/vendor/lance-table/protos/filtered_read.proto deleted file mode 100644 index d81f6b02..00000000 --- a/vendor/lance-table/protos/filtered_read.proto +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.datafusion; - -import "table_identifier.proto"; - -message U64Range { - uint64 start = 1; - uint64 end = 2; -} - -message ProjectionProto { - repeated int32 field_ids = 1; - bool with_row_id = 2; - bool with_row_addr = 3; - bool with_row_last_updated_at_version = 4; - bool with_row_created_at_version = 5; - BlobHandlingProto blob_handling = 6; -} - -message BlobHandlingProto { - oneof mode { - // All blobs read as binary - bool all_binary = 1; - // Blobs as descriptions, other binary as binary (default) - bool blobs_descriptions = 2; - // All binary columns as descriptions - bool all_descriptions = 3; - // Specific blobs read as binary, rest as descriptions (non-blob binary stays binary) - FieldIdSet some_blobs_binary = 4; - // Specific columns as binary, all other binary as descriptions - FieldIdSet some_binary = 5; - } -} - -message FieldIdSet { - repeated uint32 field_ids = 1; -} - -message FilteredReadThreadingModeProto { - oneof mode { - uint64 one_partition_multiple_threads = 1; - uint64 multiple_partitions = 2; - } -} - -// Serializable form of FilteredReadOptions. -message FilteredReadOptionsProto { - optional U64Range scan_range_before_filter = 1; - optional U64Range scan_range_after_filter = 2; - bool with_deleted_rows = 3; - optional uint32 batch_size = 4; - optional uint64 fragment_readahead = 5; - repeated uint64 fragment_ids = 6; - ProjectionProto projection = 7; - optional bytes refine_filter_substrait = 8; - optional bytes full_filter_substrait = 9; - FilteredReadThreadingModeProto threading_mode = 10; - optional uint64 io_buffer_size_bytes = 11; - // Arrow IPC schema for decoding Substrait filters (may be wider than projection). - optional bytes filter_schema_ipc = 12; -} - -// Serializable form of FilteredReadPlan (planned/distributed mode). -// RowAddrTreeMap serialized via its built-in serialize_into/deserialize_from. -// Per-fragment filters are Substrait-encoded and deduplicated. -message FilteredReadPlanProto { - bytes row_addr_tree_map = 1; - optional U64Range scan_range_after_filter = 2; - // Arrow IPC schema for decoding Substrait filters (matches the schema used at encode time). - optional bytes filter_schema_ipc = 3; - // Per-fragment filter mapping. Key is fragment id, value is a list index into - // filter_expressions. Multiple fragments can share the same list index when - // they have the same filter, avoiding duplicate Substrait encoding. - map fragment_filter_ids = 4; - // Deduplicated Substrait-encoded filter expressions. Each entry is referenced - // by one or more values in fragment_filter_ids. - repeated bytes filter_expressions = 5; -} - -// Top-level wrapper for FilteredReadExec serialization. -message FilteredReadExecProto { - TableIdentifier table = 1; - FilteredReadOptionsProto options = 2; - // FilteredRead has two modes - // Plan-then-execute (distributed): The planner creates a FilteredReadPlan and sends it to a remote executor. - // Plan-and-execute (local): The executor creates the plan itself at execution time. - optional FilteredReadPlanProto plan = 3; - // Note: FilteredReadExec.index_input (child ExecutionPlan) is NOT serialized here. - // DataFusion's PhysicalExtensionCodec handles child plans automatically: it walks - // the plan tree via children() / with_new_children(), serializes each node, and - // passes deserialized children back as the `inputs` parameter in try_decode. - // This means any ExecutionPlan in the tree (including index_input) must also - // implement try_encode/try_decode in the PhysicalExtensionCodec. - // TODO: implement serialize/deserialize for lance-specific index input ExecutionPlans. -} diff --git a/vendor/lance-table/protos/index.proto b/vendor/lance-table/protos/index.proto deleted file mode 100644 index ea21c703..00000000 --- a/vendor/lance-table/protos/index.proto +++ /dev/null @@ -1,249 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.index.pb; - -import "google/protobuf/any.proto"; - -// The type of an index. -enum IndexType { - // Vector index - VECTOR = 0; -} - -message Index { - // The unique index name in the dataset. - string name = 1; - - // Columns to be used to build the index. - repeated string columns = 2; - - // The version of the dataset this index was built from. - uint64 dataset_version = 3; - - // The [`IndexType`] of the index. - IndexType index_type = 4; - - /// Index implementation details. - oneof implementation { - VectorIndex vector_index = 5; - } -} - -message Tensor { - enum DataType { - BFLOAT16 = 0; - FLOAT16 = 1; - FLOAT32 = 2; - FLOAT64 = 3; - UINT8 = 4; - UINT16 = 5; - UINT32 = 6; - UINT64 = 7; - } - - DataType data_type = 1; - - // Data shape, [dim1, dim2, ...] - repeated uint32 shape = 2; - - // Data buffer - bytes data = 3; -} - -// Inverted Index File Metadata. -message IVF { - // Centroids of partitions. `dimension * num_partitions` of float32s. - // - // Deprecated, use centroids_tensor instead. - repeated float centroids = 1; // [deprecated = true]; - - // File offset of each partition. - repeated uint64 offsets = 2; - - // Number of records in the partition. - repeated uint32 lengths = 3; - - // Tensor of centroids. `num_partitions * dimension` of float32s. - Tensor centroids_tensor = 4; - - // KMeans loss. - optional double loss = 5; -} - -// Product Quantization. -message PQ { - // The number of bits to present a centroid. - uint32 num_bits = 1; - - // Number of sub vectors. - uint32 num_sub_vectors = 2; - - // Vector dimension - uint32 dimension = 3; - - // Codebook. `dimension * 2 ^ num_bits` of float32s. - repeated float codebook = 4; - - // Tensor of codebook. `2 ^ num_bits * dimension` of floats. - Tensor codebook_tensor = 5; -} - -// Transform type -enum TransformType { - OPQ = 0; -} - -// A transform matrix to apply to a vector or vectors. -message Transform { - // The file offset the matrix is stored - uint64 position = 1; - - // Data shape of the matrix, [rows, cols]. - repeated uint32 shape = 2; - - // Transform type. - TransformType type = 3; -} - -// Flat Index -message Flat {} - -// DiskAnn Index -message DiskAnn { - // Graph spec version - uint32 spec = 1; - - // Graph file - string filename = 2; - - // r parameter - uint32 r = 3; - - // alpha parameter - float alpha = 4; - - // L parameter - uint32 L = 5; - - /// Entry points to the graph - repeated uint64 entries = 6; -} - -// One stage in the vector index pipeline. -message VectorIndexStage { - oneof stage { - // Flat index - Flat flat = 1; - // `IVF` - Inverted File - IVF ivf = 2; - // Product Quantization - PQ pq = 3; - // Transformer - Transform transform = 4; - // DiskANN - DiskAnn diskann = 5; - } -} - -// Metric Type for Vector Index -enum VectorMetricType { - // L2 (Euclidean) Distance - L2 = 0; - - // Cosine Distance - Cosine = 1; - - // Dot Product - Dot = 2; - - // Hamming Distance - Hamming = 3; -} - -// Vector Index Metadata -message VectorIndex { - // Index specification version. - uint32 spec_version = 1; - - // Vector dimension; - uint32 dimension = 2; - - // Composed vector index stages. - // - // For example, `IVF_PQ` index type can be expressed as: - // - // ```text - // let stages = vec![Ivf{}, PQ{num_bits: 8, num_sub_vectors: 16}] - // ``` - repeated VectorIndexStage stages = 3; - - // Vector distance metrics type - VectorMetricType metric_type = 4; -} - -// Details for vector indexes, stored in the manifest's index_details field. -message VectorIndexDetails { - VectorMetricType metric_type = 1; - - // The target number of vectors per partition. - // 0 means unset. - uint64 target_partition_size = 2; - - // Optional HNSW index configuration. If set, the index has an HNSW layer. - optional HnswParameters hnsw_index_config = 3; - - message ProductQuantization { - uint32 num_bits = 1; - uint32 num_sub_vectors = 2; - } - message ScalarQuantization { - uint32 num_bits = 1; - } - message RabitQuantization { - enum RotationType { - FAST = 0; - MATRIX = 1; - } - uint32 num_bits = 1; - RotationType rotation_type = 2; - } - - // No quantization; vectors are stored as-is. - message FlatCompression {} - - oneof compression { - ProductQuantization pq = 4; - ScalarQuantization sq = 5; - RabitQuantization rq = 6; - FlatCompression flat = 8; - } - - // Runtime hints: optional build preferences that don't affect index structure. - // Keys use reverse-DNS namespacing (e.g., "lance.ivf.max_iters", "lancedb.accelerator"). - // Unrecognized keys must be silently ignored by all runtimes. - map runtime_hints = 9; -} - -// Hierarchical Navigable Small World (HNSW) parameters, used as an optional configuration for IVF indexes. -message HnswParameters { - // The maximum number of outgoing edges per node in the HNSW graph. Higher values - // means more connections, better recall, but more memory and slower builds. - // Referred to as "M" in the HNSW literature. - uint32 max_connections = 1; - // "construction exploration factor": The size of the dynamic list used during - // index construction. - uint32 construction_ef = 2; - // The maximum number of levels in the HNSW graph. - uint32 max_level = 3; -} - -message JsonIndexDetails { - string path = 1; - google.protobuf.Any target_details = 2; -} -message BloomFilterIndexDetails {} - -message RTreeIndexDetails {} \ No newline at end of file diff --git a/vendor/lance-table/protos/index_old.proto b/vendor/lance-table/protos/index_old.proto deleted file mode 100644 index 601aa268..00000000 --- a/vendor/lance-table/protos/index_old.proto +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.table; - -// NOTE: Do *NOT* add new index details here. Add them to the index.proto file instead. -// This file is in the lance.table package namespace while the index.proto file is in the -// lance.index package namespace. -// -// These are only here for forward compatibility. Older versions of Lance expect btree indexes -// to have lance.table in the package namespace. -// -// If you need to modify these messages (e.g. to add new fields to btree or bitmap) then -// it is ok to modify them here. - -// Currently many of these are empty messages because all needed details are either hard-coded (e.g. -// filenames) or stored in the index itself. However, we may want to add more details in the -// future, in particular we can add details that may be useful for planning queries (e.g. don't -// force us to load the index until we know we can make use of it) - -message BTreeIndexDetails {} -message BitmapIndexDetails {} -message LabelListIndexDetails {} -message NGramIndexDetails {} -message ZoneMapIndexDetails {} -message InvertedIndexDetails { - // Marking this field as optional as old versions of the index store blank details and we - // need to make sure we have a proper optional field to detect this. - optional string base_tokenizer = 1; - string language = 2; - bool with_position = 3; - optional uint32 max_token_length = 4; - bool lower_case = 5; - bool stem = 6; - bool remove_stop_words = 7; - bool ascii_folding = 8; - uint32 min_ngram_length = 9; - uint32 max_ngram_length = 10; - bool prefix_only = 11; -} diff --git a/vendor/lance-table/protos/license_header.txt b/vendor/lance-table/protos/license_header.txt deleted file mode 100644 index 893e9a80..00000000 --- a/vendor/lance-table/protos/license_header.txt +++ /dev/null @@ -1,2 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors diff --git a/vendor/lance-table/protos/rowids.proto b/vendor/lance-table/protos/rowids.proto deleted file mode 100644 index 3039cbf2..00000000 --- a/vendor/lance-table/protos/rowids.proto +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.table; -// TODO: what would it take to store this in a LanceV2 file? -// Or would flatbuffers be better for this? - -/// A sequence of row IDs. This is split up into one or more segments, -/// each of which can be encoded in different ways. The encodings are optimized -/// for values that are sorted, which will often be the case with row ids. -/// They also have optimized forms depending on how sparse the values are. -message RowIdSequence { - repeated U64Segment segments = 1; -} - -/// Different ways to encode a sequence of u64 values. -message U64Segment { - /// A range of u64 values. - message Range { - /// The start of the range, inclusive. - uint64 start = 1; - /// The end of the range, exclusive. - uint64 end = 2; - } - - /// A range of u64 values with holes. - message RangeWithHoles { - /// The start of the range, inclusive. - uint64 start = 1; - /// The end of the range, exclusive. - uint64 end = 2; - /// The holes in the range, as a sorted array of values; - /// Binary search can be used to check whether a value is a hole and should - /// be skipped. This can also be used to count the number of holes before a - /// given value, if you need to find the logical offset of a value in the - /// segment. - EncodedU64Array holes = 3; - } - - /// A range of u64 values with a bitmap. - message RangeWithBitmap { - /// The start of the range, inclusive. - uint64 start = 1; - /// The end of the range, exclusive. - uint64 end = 2; - /// A bitmap of the values in the range. The bitmap is a sequence of bytes, - /// where each byte represents 8 values. The first byte represents values - /// start to start + 7, the second byte represents values start + 8 to - /// start + 15, and so on. The most significant bit of each byte represents - /// the first value in the range, and the least significant bit represents - /// the last value in the range. If the bit is set, the value is in the - /// range; if it is not set, the value is not in the range. - bytes bitmap = 3; - } - - oneof segment { - /// When the values are sorted and contiguous. - Range range = 1; - /// When the values are sorted but have a few gaps. - RangeWithHoles range_with_holes = 2; - /// When the values are sorted but have many gaps. - RangeWithBitmap range_with_bitmap = 3; - /// When the values are sorted but are sparse. - EncodedU64Array sorted_array = 4; - /// A general array of values, which is not sorted. - EncodedU64Array array = 5; - } -} // RowIdSegment - -/// A basic bitpacked array of u64 values. -message EncodedU64Array { - message U16Array { - uint64 base = 1; - /// The deltas are stored as 16-bit unsigned integers. - /// (protobuf doesn't support 16-bit integers, so we use bytes instead) - bytes offsets = 2; - } - - message U32Array { - uint64 base = 1; - /// The deltas are stored as 32-bit unsigned integers. - /// (we use bytes instead of uint32 to avoid overhead of varint encoding) - bytes offsets = 2; - } - - message U64Array { - /// (We use bytes instead of uint64 to avoid overhead of varint encoding) - bytes values = 2; - } - - oneof array { - U16Array u16_array = 1; - U32Array u32_array = 2; - U64Array u64_array = 3; - } -} - -/// A sequence of dataset versions. Similar to RowIdSequence but tracks -/// version runs. It uses RLE (Run-Length Encoding) to efficiently -// represent consecutive rows with the same version. -message RowDatasetVersionSequence { - repeated RowDatasetVersionRun runs = 1; -} - -/// A run of rows with the same version. -message RowDatasetVersionRun { - /// The number of consecutive rows with the same version. - U64Segment span = 1; - - uint64 version = 2; -} diff --git a/vendor/lance-table/protos/table.proto b/vendor/lance-table/protos/table.proto deleted file mode 100644 index d298809d..00000000 --- a/vendor/lance-table/protos/table.proto +++ /dev/null @@ -1,717 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.table; - -import "google/protobuf/any.proto"; -import "google/protobuf/timestamp.proto"; -import "file.proto"; - -/* - -Format: - -+----------------------------------------+ -| Encoded Column 0, Chunk 0 | - ... -| Encoded Column M, Chunk N - 1 | -| Encoded Column M, Chunk N | -| Indices ... | -| Chunk Position (M x N x 8) | -| Manifest (Optional) | -| Metadata | -| i64: metadata position | -| MAJOR_VERSION | MINOR_VERSION | "LANC" | -+----------------------------------------+ - */ - -// UUID type. encoded as 16 bytes. -message UUID { - bytes uuid = 1; -} - -// Manifest is a global section shared between all the files. -message Manifest { - // All fields of the dataset, including the nested fields. - repeated lance.file.Field fields = 1; - - // Schema metadata. - map schema_metadata = 5; - - // Fragments of the dataset. - repeated DataFragment fragments = 2; - - // Snapshot version number. - uint64 version = 3; - - // The file position of the version auxiliary data. - // * It is not inheritable between versions. - // * It is not loaded by default during query. - uint64 version_aux_data = 4; - - message WriterVersion { - // The name of the library that created this file. - string library = 1; - // The version of the library that created this file. Because we cannot assume - // that the library is semantically versioned, this is a string. However, if it - // is semantically versioned, it should be a valid semver string without any 'v' - // prefix. For example: `2.0.0`, `2.0.0-rc.1`. - // - // For forward compatibility with older readers, when writing new manifests this - // field should contain only the core version (major.minor.patch) without any - // prerelease or build metadata. The prerelease/build info should be stored in - // the separate prerelease and build_metadata fields instead. - string version = 2; - // Optional semver prerelease identifier. - // - // This field stores the prerelease portion of a semantic version separately - // from the core version number. For example, if the full version is "2.0.0-rc.1", - // the version field would contain "2.0.0" and prerelease would contain "rc.1". - // - // This separation ensures forward compatibility: older readers can parse the - // clean version field without errors, while newer readers can reconstruct the - // full semantic version by combining version, prerelease, and build_metadata. - // - // If absent, the version field is used as-is. - optional string prerelease = 3; - // Optional semver build metadata. - // - // This field stores the build metadata portion of a semantic version separately - // from the core version number. For example, if the full version is - // "2.0.0-rc.1+build.123", the version field would contain "2.0.0", prerelease - // would contain "rc.1", and build_metadata would contain "build.123". - // - // If absent, no build metadata is present. - optional string build_metadata = 4; - } - - // The version of the writer that created this file. - // - // This information may be used to detect whether the file may have known bugs - // associated with that writer. - WriterVersion writer_version = 13; - - // If present, the file position of the index metadata. - optional uint64 index_section = 6; - - // Version creation Timestamp, UTC timezone - google.protobuf.Timestamp timestamp = 7; - - // Optional version tag - string tag = 8; - - // Feature flags for readers. - // - // A bitmap of flags that indicate which features are required to be able to - // read the table. If a reader does not recognize a flag that is set, it - // should not attempt to read the dataset. - // - // Known flags: - // * 1: deletion files are present - // * 2: row ids are stable and stored as part of the fragment metadata. - // * 4: use v2 format (deprecated) - // * 8: table config is present - uint64 reader_feature_flags = 9; - - // Feature flags for writers. - // - // A bitmap of flags that indicate which features must be used when writing to the - // dataset. If a writer does not recognize a flag that is set, it should not attempt to - // write to the dataset. - // - // The flag identities are the same as for reader_feature_flags, but the values of - // reader_feature_flags and writer_feature_flags are not required to be identical. - uint64 writer_feature_flags = 10; - - // The highest fragment ID that has been used so far. - // - // This ID is not guaranteed to be present in the current version, but it may - // have been used in previous versions. - // - // For a single fragment, will be zero. For no fragments, will be absent. - optional uint32 max_fragment_id = 11; - - // Path to the transaction file, relative to `{root}/_transactions`. The file at that - // location contains a wire-format serialized Transaction message representing the - // transaction that created this version. - // - // This string field "transaction_file" may be empty if no transaction file was written. - // - // The path format is "{read_version}-{uuid}.txn" where {read_version} is the version of - // the table the transaction read from (serialized to decimal with no padding digits), - // and {uuid} is a hyphen-separated UUID. - string transaction_file = 12; - - // The file position of the transaction content. None if transaction is empty - // This transaction content begins with the transaction content length as u32 - // If the transaction proto message has a length of `len`, the message ends at `len` + 4 - optional uint64 transaction_section = 21; - - // The next unused row id. If zero, then the table does not have any rows. - // - // This is only used if the "stable_row_ids" feature flag is set. - uint64 next_row_id = 14; - - message DataStorageFormat { - // The format of the data files (e.g. "lance") - string file_format = 1; - // The max format version of the data files. The format of the version can vary by - // file_format and is not required to follow semver. - // - // Every file in this version of the dataset has the same file_format version. - string version = 2; - } - - // The data storage format - // - // This specifies what format is used to store the data files. - DataStorageFormat data_format = 15; - - // Table config. - // - // Keys with the prefix "lance." are reserved for the Lance library. Other - // libraries may wish to similarly prefix their configuration keys - // appropriately. - map config = 16; - - // Metadata associated with the table. - // - // This is a key-value map that can be used to store arbitrary metadata - // associated with the table. - // - // This is different than configuration, which is used to tell libraries how - // to read, write, or manage the table. - // - // This is different than schema metadata, which is used to describe the - // data itself and is attached to the output schema of scans. - map table_metadata = 19; - - // Field number 17 (`blob_dataset_version`) was used for a secondary blob dataset. - reserved 17; - reserved "blob_dataset_version"; - - // The base paths of data files. - // - // This is used to determine the base path of a data file. In common cases data file paths are under current dataset base path. - // But for shallow cloning, importing file and other multi-tier storage cases, the actual data files could be outside of the current dataset. - // This field is used with the `base_id` in `lance.file.File` and `lance.file.DeletionFile`. - // - // For example, if we have a dataset with base path `s3://bucket/dataset`, we have a DataFile with base_id 0, we get the actual data file path by: - // base_paths[id = 0] + /data/ + file.path - // the key(a.k.a index) starts from 0, increased by 1 for each new base path. - repeated BasePath base_paths = 18; - - // The branch of the dataset. None means main branch. - optional string branch = 20; -} // Manifest - -// external dataset base path -message BasePath { - uint32 id = 1; - // This is an alias name of the base path, it is optional. - // When we use shallow clone and the target version is a tag, the tag name will be set here. - optional string name = 2; - // Flag indicating whether this path is a dataset root path or file directory: - // - true: Path is a dataset root (actual files under subdirectories like `data`, '_deletions') - // - false: Path is a direct file directory (scenario like importing files) - bool is_dataset_root = 3; - // Note: This absolute path will be directly used by Path:parse(), - string path = 4; -} - -// Auxiliary Data attached to a version. -// Only load on-demand. -message VersionAuxData { - // key-value metadata. - map metadata = 3; -} - -// Metadata describing an index. -message IndexMetadata { - // Unique ID of an index. It is unique across all the dataset versions. - UUID uuid = 1; - - // The columns to build the index. These refer to file.Field.id. - repeated int32 fields = 2; - - // Index name. Must be unique within one dataset version. - string name = 3; - - // The version of the dataset this index was built from. - uint64 dataset_version = 4; - - // A bitmap of the included fragment ids. - // - // This may by used to determine how much of the dataset is covered by the - // index. This information can be retrieved from the dataset by looking at - // the dataset at `dataset_version`. However, since the old version may be - // deleted while the index is still in use, this information is also stored - // in the index. - // - // The bitmap is stored as a 32-bit Roaring bitmap. - bytes fragment_bitmap = 5; - - // Details, specific to the index type, which are needed to load / interpret the index - // - // Indices should avoid putting large amounts of information in this field, as it will - // bloat the manifest. - // - // Indexes are plugins, and so the format of the details message is flexible and not fully - // defined by the table format. However, there are some conventions that should be followed: - // - // - When Lance APIs refer to indexes they will use the type URL of the index details as the - // identifier for the index type. If a user provides a simple string identifier like - // "btree" then it will be converted to "/lance.table.BTreeIndexDetails" - // - Type URLs comparisons are case-insensitive. Thereform an index must have a unique type - // URL ignoring case. - google.protobuf.Any index_details = 6; - - // The minimum lance version that this index is compatible with. - optional int32 index_version = 7; - - // Timestamp when the index was created (UTC timestamp in milliseconds since epoch) - // - // This field is optional for backward compatibility. For existing indices created before - // this field was added, this will be None/null. - optional uint64 created_at = 8; - - // The base path index of the data file. Used when the file is imported or referred from another dataset. - // Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file. - optional uint32 base_id = 9; - - // List of files and their sizes for this index segment. - // This enables skipping HEAD calls when opening indices and allows reporting - // of index sizes without extra IO. - // If this is empty, the index files sizes are unknown. - repeated IndexFile files = 10; -} - -// Metadata about a single file within an index segment. -message IndexFile { - // Path relative to the index directory (e.g., "index.idx", "auxiliary.idx") - string path = 1; - // Size of the file in bytes - uint64 size_bytes = 2; -} - -// Index Section, containing a list of index metadata for one dataset version. -message IndexSection { - repeated IndexMetadata indices = 1; -} - -// A DataFragment is a set of files which represent the different columns of the same -// rows. If column exists in the schema of a dataset, but the file for that column does -// not exist within a DataFragment of that dataset, that column consists entirely of -// nulls. -message DataFragment { - // The ID of a DataFragment is unique within a dataset. - uint64 id = 1; - - repeated DataFile files = 2; - - // File that indicates which rows, if any, should be considered deleted. - DeletionFile deletion_file = 3; - - // TODO: What's the simplest way we can allow an inline tombstone bitmap? - - // A serialized RowIdSequence message (see rowids.proto). - // - // These are the row ids for the fragment, in order of the rows as they appear. - // That is, if a fragment has 3 rows, and the row ids are [1, 42, 3], then the - // first row is row 1, the second row is row 42, and the third row is row 3. - oneof row_id_sequence { - // If small (< 200KB), the row ids are stored inline. - bytes inline_row_ids = 5; - // Otherwise, stored as part of a file. - ExternalFile external_row_ids = 6; - } // row_id_sequence - - oneof last_updated_at_version_sequence { - // If small (< 200KB), the row latest updated versions are stored inline. - bytes inline_last_updated_at_versions = 7; - // Otherwise, stored as part of a file. - ExternalFile external_last_updated_at_versions = 8; - } // last_updated_at_version_sequence - - oneof created_at_version_sequence { - // If small (< 200KB), the row created at versions are stored inline. - bytes inline_created_at_versions = 9; - // Otherwise, stored as part of a file. - ExternalFile external_created_at_versions = 10; - } // created_at_version_sequence - - // Number of original rows in the fragment, this includes rows that are now marked with - // deletion tombstones. To compute the current number of rows, subtract - // `deletion_file.num_deleted_rows` from this value. - uint64 physical_rows = 4; -} - -message DataFile { - // Path to the root relative to the dataset's URI. - string path = 1; - // The ids of the fields/columns in this file. - // - // When a DataFile object is created in memory, every value in fields is assigned -1 by - // default. An object with a value in fields of -1 must not be stored to disk. -2 is - // used for "tombstoned", meaning a field that is no longer in use. This is often - // because the original field id was reassigned to a different data file. - // - // In Lance v1 IDs are assigned based on position in the file, offset by the max - // existing field id in the table (if any already). So when a fragment is first created - // with one file of N columns, the field ids will be 1, 2, ..., N. If a second fragment - // is created with M columns, the field ids will be N+1, N+2, ..., N+M. - // - // In Lance v1 there is one field for each field in the input schema, this includes - // nested fields (both struct and list). Fixed size list fields have only a single - // field id (these are not considered nested fields in Lance v1). - // - // This allows column indices to be calculated from field IDs and the input schema. - // - // In Lance v2 the field IDs generally follow the same pattern but there is no - // way to calculate the column index from the field ID. This is because a given - // field could be encoded in many different ways, some of which occupy a different - // number of columns. For example, a struct field could be encoded into N + 1 columns - // or it could be encoded into a single packed column. To determine column indices - // the column_indices property should be used instead. - // - // In Lance v1 these ids must be sorted but might not always be contiguous. - repeated int32 fields = 2; - // The top-level column indices for each field in the file. - // - // If the data file is version 1 then this property will be empty - // - // Otherwise there must be one entry for each field in `fields`. - // - // Some fields may not correspond to a top-level column in the file. In these cases - // the index will -1. - // - // For example, consider the schema: - // - // - dimension: packed-struct (0): - // - x: u32 (1) - // - y: u32 (2) - // - path: `list` (3) - // - embedding: `fsl<768>` (4) - // - fp64 - // - borders: `fsl<4>` (5) - // - simple-struct (6) - // - margin: fp64 (7) - // - padding: fp64 (8) - // - // One possible column indices array could be: - // [0, -1, -1, 1, 3, 4, 5, 6, 7] - // - // This reflects quite a few phenomenon: - // - The packed struct is encoded into a single column and there is no top-level column - // for the x or y fields - // - The variable sized list is encoded into two columns - // - The embedding is encoded into a single column (common for FSL of primitive) and there - // is not "FSL column" - // - The borders field actually does have an "FSL column" - // - // The column indices table may not have duplicates (other than -1) - repeated int32 column_indices = 3; - // The major file version used to create the file - uint32 file_major_version = 4; - // The minor file version used to create the file - // - // If both `file_major_version` and `file_minor_version` are set to 0, - // then this is a version 0.1 or version 0.2 file. - uint32 file_minor_version = 5; - - // The known size of the file on disk in bytes. - // - // This is used to quickly find the footer of the file. - // - // When this is zero, it should be interpreted as "unknown". - uint64 file_size_bytes = 6; - - // The base path index of the data file. Used when the file is imported or referred from another dataset. - // Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file. - optional uint32 base_id = 7; -} // DataFile - -// Deletion File -// -// The path of the deletion file is constructed as: -// {root}/_deletions/{fragment_id}-{read_version}-{id}.{extension} -// where {extension} depends on DeletionFileType. -message DeletionFile { - // Type of deletion file, intended as a way to increase efficiency of the storage of deleted row - // offsets. If there are sparsely deleted rows, then ARROW_ARRAY is the most efficient. If there - // are densely deleted rows, then BITMAP is the most efficient. - enum DeletionFileType { - // A single Int32Array of deleted row offsets, stored as an Arrow IPC file with one batch and - // one column. Has a .arrow extension. - ARROW_ARRAY = 0; - // A Roaring Bitmap of deleted row offsets. Has a .bin extension. - BITMAP = 1; - } - - // Type of deletion file. - DeletionFileType file_type = 1; - // The version of the dataset this deletion file was built from. - uint64 read_version = 2; - // An opaque id used to differentiate this file from others written by concurrent - // writers. - uint64 id = 3; - // The number of rows that are marked as deleted. - uint64 num_deleted_rows = 4; - // The base path index of the deletion file. Used when the file is imported or referred from another - // dataset. Lance uses it as key of the base_paths field in Manifest to determine the actual base - // path of the deletion file. - optional uint32 base_id = 7; -} // DeletionFile - -message ExternalFile { - // Path to the file, relative to the root of the table. - string path = 1; - // The byte offset in the file where the data starts. - uint64 offset = 2; - // The size of the data in the file, in bytes. - uint64 size = 3; -} - -// VectorIndexDetails and HnswParameters (formerly HnswIndexDetails) moved to index.proto - -message FragmentReuseIndexDetails { - - oneof content { - // if < 200KB, store the content inline, otherwise store the InlineContent bytes in external file - InlineContent inline = 1; - ExternalFile external = 2; - } - - message InlineContent { - repeated Version versions = 1; - } - - message FragmentDigest { - uint64 id = 1; - - uint64 physical_rows = 2; - - uint64 num_deleted_rows = 3; - } - - // A summarized version of the RewriteGroup information in a Rewrite transaction - message Group { - // A roaring treemap of the changed row addresses. - // When combined with the old fragment IDs and new fragment IDs, - // it can recover the full mapping of old row addresses to either new row addresses or deleted. - // this mapping can then be used to remap indexes or satisfy index queries for the new unindexed fragments. - bytes changed_row_addrs = 1; - - repeated FragmentDigest old_fragments = 2; - - repeated FragmentDigest new_fragments = 3; - } - - message Version { - // The dataset_version at the time the index adds this version entry - uint64 dataset_version = 1; - - repeated Group groups = 3; - } -} - -// ============================================================================ -// MemWAL Index Types -// ============================================================================ - -// Shard manifest containing epoch-based fencing and WAL state. -// Each shard has exactly one active writer at any time. -message ShardManifest { - // Shard identifier (UUID v4). - UUID shard_id = 11; - - // Manifest version number. - // Matches the version encoded in the filename. - uint64 version = 1; - - // Shard spec ID this shard was created with. - // Set at shard creation and immutable thereafter. - // A value of 0 indicates a manually-created shard not governed by any spec. - uint32 shard_spec_id = 10; - - // Computed shard field values as raw Arrow scalar bytes, keyed by shard - // field id. The byte encoding follows Arrow's little-endian convention: - // int32 is 4 LE bytes, utf8 is raw UTF-8 bytes, etc. The receiver looks - // up the result_type from the ShardingSpec to interpret each value. - repeated ShardFieldEntry shard_field_entries = 14; - - // Writer fencing token - monotonically increasing. - // A writer must increment this when claiming the shard. - uint64 writer_epoch = 2; - - // The most recent WAL entry position that has been flushed to a MemTable. - // During recovery, replay starts from replay_after_wal_entry_position + 1. - // WAL positions are 1-based, so the default value 0 unambiguously means - // "no flush has ever stamped this shard" and recovery replays from 1. - uint64 replay_after_wal_entry_position = 3; - - // The most recent WAL entry position observed at the time the manifest was - // updated. WAL positions are 1-based; default 0 means no entry has been - // written yet. This is a hint, not authoritative - recovery must list - // files to find actual state. - uint64 wal_entry_position_last_seen = 4; - - // Next generation ID to create (incremented after each MemTable flush). - uint64 current_generation = 6; - - // Field 7 removed: merged_generation moved to MemWalIndexDetails.merged_generations - // which is the authoritative source for merge progress. - - // List of flushed MemTable generations and their directory paths. - repeated FlushedGeneration flushed_generations = 8; -} - -// A shard field value stored as raw Arrow scalar bytes. -message ShardFieldEntry { - // Shard field id (matches ShardingField.field_id in the ShardingSpec). - string field_id = 1; - - // Raw Arrow scalar value bytes in little-endian encoding. - // The data type is determined by the result_type of the matching ShardingField. - bytes value = 2; -} - -// A flushed MemTable generation and its storage location. -message FlushedGeneration { - // Generation number. - uint64 generation = 1; - - // Directory name relative to the shard directory. - string path = 2; -} - -// A shard's merged generation, used in MemWalIndexDetails. -message MergedGeneration { - // Shard identifier (UUID v4). - UUID shard_id = 1; - - // Last generation merged to base table for this shard. - uint64 generation = 2; -} - -// Tracks which merged generation a base table index has been rebuilt to cover. -// Used to determine whether to read from flushed MemTable indexes or base table. -message IndexCatchupProgress { - // Name of the base table index (must match an entry in maintained_indexes). - string index_name = 1; - - // Per-shard progress: the generation up to which this index covers. - // If a shard is not present, the index is assumed to be fully caught up - // (i.e., caught_up_generation >= merged_generation for that shard). - repeated MergedGeneration caught_up_generations = 2; -} - -// Index details for MemWAL Index, stored in IndexMetadata.index_details. -// This is the centralized structure for all MemWAL metadata: -// - Configuration (sharding specs, indexes to maintain) -// - Merge progress (merged generations per shard) -// - Shard state snapshots -// -// Writers read this index to get configuration before writing. -// Readers may use shard snapshots in this index as a point-in-time -// optimization. Readers that need the latest shard set should list shard -// directories in storage and read each shard's latest manifest. -// A background process updates the index periodically to keep shard snapshots current. -// -// Shard snapshots are stored as a Lance file with one row per shard. -// The schema records shard discovery fields. Full mutable shard state remains -// authoritative in the shard manifest files. -// shard_id: utf8 -// shard_spec_id: uint32 -// shard_field_{field_id}: typed per the matching ShardingField.result_type -message MemWalIndexDetails { - // Snapshot timestamp (Unix timestamp in milliseconds). - int64 snapshot_ts_millis = 1; - - // Number of shards in the snapshot. - // Used to determine storage format without reading the snapshot data. - uint32 num_shards = 2; - - // Inline shard snapshots for small shard counts. - // When num_shards <= threshold (implementation-defined, e.g., 100), - // snapshots are stored inline as serialized bytes. - // Format: Lance file bytes with the shard snapshot schema. - optional bytes inline_snapshots = 3; - - // Sharding specs defining how to derive shard identifiers. - // This configuration determines how rows are partitioned into shards. - repeated ShardingSpec sharding_specs = 7; - - // Indexes from the base table to maintain in MemTables. - // These are index names referencing indexes defined on the base table. - // The primary key btree index is always maintained implicitly and - // should not be listed here. - // - // For vector indexes, MemTables inherit quantization parameters (PQ codebook, - // SQ params) from the base table index to ensure distance comparability. - repeated string maintained_indexes = 8; - - // Last generation merged to base table for each shard. - // This is updated atomically with merge-insert data commits, enabling - // conflict resolution when multiple mergers operate concurrently. - // - // Note: This is separate from shard snapshots because: - // 1. merged_generations is updated by mergers (atomic with data commit) - // 2. shard snapshots are updated by background index builder - repeated MergedGeneration merged_generations = 9; - - // Per-index catchup progress tracking. - // When data is merged to the base table, base table indexes are rebuilt - // asynchronously. This field tracks which generation each index covers. - // - // For indexed queries, if an index's caught_up_generation < merged_generation, - // readers should use flushed MemTable indexes for the gap instead of - // scanning unindexed data in the base table. - // - // If an index is not present in this list, it is assumed to be fully caught up. - repeated IndexCatchupProgress index_catchup = 10; - - // Default ShardWriter configuration values for this MemWAL index. - // - // A free-form string map persisted so that every writer — across - // processes and restarts — starts from the same default writer - // configuration. These are defaults only: an individual writer may - // still override any value at runtime in its own ShardWriterConfig - // (which is not persisted). - map writer_config_defaults = 11; -} - -// Sharding spec definition. -message ShardingSpec { - // Unique identifier for this spec within the index. - // IDs are never reused. - uint32 spec_id = 1; - - // Sharding field definitions that determine how to compute shard identifiers. - repeated ShardingField fields = 2; -} - -// Sharding field definition. -message ShardingField { - // Unique string identifier for this shard field. - string field_id = 1; - - // Field IDs referencing source columns in the schema. - repeated int32 source_ids = 2; - - // Well-known shard transform name (e.g., "identity", "year", "bucket"). - // Mutually exclusive with expression. - optional string transform = 3; - - // DataFusion SQL expression for custom logic. - // Mutually exclusive with transform. - optional string expression = 4; - - // Output type of the shard value (Arrow type name). - string result_type = 5; - - // Transform parameters (e.g., num_buckets for bucket transform). - map parameters = 6; -} diff --git a/vendor/lance-table/protos/table_identifier.proto b/vendor/lance-table/protos/table_identifier.proto deleted file mode 100644 index 3a471455..00000000 --- a/vendor/lance-table/protos/table_identifier.proto +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -package lance.datafusion; - -// Identifies a Lance dataset for remote reconstruction. -// -// Two modes: -// 1. uri + serialized_manifest (fast): remote executor skips manifest read. -// 2. uri + version + etag (lightweight): remote executor loads manifest from storage. -message TableIdentifier { - string uri = 1; - uint64 version = 2; - optional string manifest_etag = 3; - optional bytes serialized_manifest = 4; - map storage_options = 5; -} diff --git a/vendor/lance-table/protos/transaction.proto b/vendor/lance-table/protos/transaction.proto deleted file mode 100644 index e72e9502..00000000 --- a/vendor/lance-table/protos/transaction.proto +++ /dev/null @@ -1,354 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -import "file.proto"; -import "table.proto"; -import "google/protobuf/any.proto"; - -package lance.table; - -// A transaction represents the changes to a dataset. -// -// This has two purposes: -// 1. When retrying a commit, the transaction can be used to re-build an updated -// manifest. -// 2. When there's a conflict, this can be used to determine whether the other -// transaction is compatible with this one. -message Transaction { - // The version of the dataset this transaction was built from. - // - // For example, for a delete transaction this means the version of the dataset - // that was read from while evaluating the deletion predicate. - uint64 read_version = 1; - - // The UUID that unique identifies a transaction. - string uuid = 2; - - // Optional version tag. - string tag = 3; - - // Optional properties for the transaction - // __lance_commit_message is a reserved key - map transaction_properties = 4; - - // Add new rows to the dataset. - message Append { - // The new fragments to append. - // - // Fragment IDs are not yet assigned. - repeated DataFragment fragments = 1; - } - - // Mark rows as deleted. - message Delete { - // The fragments to update - // - // The fragment IDs will match existing fragments in the dataset. - repeated DataFragment updated_fragments = 1; - // The fragments to delete entirely. - repeated uint64 deleted_fragment_ids = 2; - // The predicate that was evaluated - // - // This may be used to determine whether the delete would have affected - // files written by a concurrent transaction. - string predicate = 3; - } - - // Create or overwrite the entire dataset. - message Overwrite { - // The new fragments - // - // Fragment IDs are not yet assigned. - repeated DataFragment fragments = 1; - // The new schema - repeated lance.file.Field schema = 2; - // Schema metadata. - map schema_metadata = 3; - // Key-value pairs to merge with existing config. - map config_upsert_values = 4; - // The base paths to be added for the initial dataset creation - repeated BasePath initial_bases = 5; - } - - // Add or replace a new secondary index. - // - // This is also used to remove an index (we are replacing it with nothing) - // - // - new_indices: the modified indices, empty if dropping indices only - // - removed_indices: the indices that are being replaced - message CreateIndex { - repeated IndexMetadata new_indices = 1; - repeated IndexMetadata removed_indices = 2; - } - - // An operation that rewrites but does not change the data in the table. These - // kinds of operations just rearrange data. - message Rewrite { - // The old fragments that are being replaced - // - // DEPRECATED: use groups instead. - // - // These should all have existing fragment IDs. - repeated DataFragment old_fragments = 1; - // The new fragments - // - // DEPRECATED: use groups instead. - // - // These fragments IDs are not yet assigned. - repeated DataFragment new_fragments = 2; - - // During a rewrite an index may be rewritten. We only serialize the UUID - // since a rewrite should not change the other index parameters. - message RewrittenIndex { - // The id of the index that will be replaced - UUID old_id = 1; - // the id of the new index - UUID new_id = 2; - // the new index details - google.protobuf.Any new_index_details = 3; - // the version of the new index - uint32 new_index_version = 4; - // Files in the new index with their sizes. - // Empty if file sizes are not available (e.g. older writers). - repeated IndexFile new_index_files = 5; - } - - // A group of rewrite files that are all part of the same rewrite. - message RewriteGroup { - // The old fragment that is being replaced - // - // This should have an existing fragment ID. - repeated DataFragment old_fragments = 1; - // The new fragment - // - // The ID should have been reserved by an earlier - // reserve operation - repeated DataFragment new_fragments = 2; - } - - // Groups of files that have been rewritten - repeated RewriteGroup groups = 3; - // Indices that have been rewritten - repeated RewrittenIndex rewritten_indices = 4; - } - - // An operation that merges in a new column, altering the schema. - message Merge { - // The updated fragments - // - // These should all have existing fragment IDs. - repeated DataFragment fragments = 1; - // The new schema - repeated lance.file.Field schema = 2; - // Schema metadata. - map schema_metadata = 3; - } - - // An operation that projects a subset of columns, altering the schema. - message Project { - // The new schema - repeated lance.file.Field schema = 1; - } - - // An operation that restores a dataset to a previous version. - message Restore { - // The version to restore to - uint64 version = 1; - } - - // An operation that reserves fragment ids for future use in - // a rewrite operation. - message ReserveFragments { - uint32 num_fragments = 1; - } - - // An operation that clones a dataset. - message Clone { - // - true: Performs a metadata-only clone (copies manifest without data files). - // The cloned dataset references original data through `base_paths`, - // suitable for experimental scenarios or rapid metadata migration. - // - false: Performs a full deep clone using the underlying object storage's native - // copy API (e.g., S3 CopyObject, GCS rewrite). This leverages server-side - // bulk copy operations to bypass download/upload bottlenecks, achieving - // near-linear speedup for large datasets (typically 3-10x faster than - // manual file transfers). The operation maintains atomicity and data - // integrity guarantees provided by the storage backend. - bool is_shallow = 1; - // the reference name in the source dataset - // in most cases it should be the branch or tag name in the source dataset - optional string ref_name = 2; - // the version of the source dataset for cloning - uint64 ref_version = 3; - // the absolute base path of the source dataset for cloning - string ref_path = 4; - // if the target dataset is a branch, this is the branch name of the target dataset - optional string branch_name = 5; - } - - // Exact set of key hashes for conflict detection. - // Used when the number of inserted rows is small. - message ExactKeySetFilter { - // 64-bit hashes of the inserted row keys. - repeated uint64 key_hashes = 1; - } - - // Bloom filter for key existence tests. - // Used when the number of rows is large. - message BloomFilter { - // Bitset backing the bloom filter (SBBF format). - bytes bitmap = 1; - // Number of bits in the bitmap. - uint32 num_bits = 2; - // Number of items the filter was sized for. - // Used for intersection validation (filters with different sizes cannot be compared). - // Default: 8192 - uint64 number_of_items = 3; - // False positive probability the filter was sized for. - // Used for intersection validation (filters with different parameters cannot be compared). - // Default: 0.00057 - double probability = 4; - } - - // A filter for checking key existence in set of rows inserted by a merge insert operation. - // Only created when the merge insert's ON columns match the schema's unenforced primary key. - // The presence of this filter indicates strict primary key conflict detection should be used. - // Can use either an exact set (for small row counts) or a Bloom filter (for large row counts). - message KeyExistenceFilter { - // Field IDs of columns participating in the key (must match unenforced primary key). - repeated int32 field_ids = 1; - // The underlying data structure storing the key hashes. - oneof data { - // Exact set of key hashes (used for small number of rows). - ExactKeySetFilter exact = 2; - // Bloom filter (used for large number of rows). - BloomFilter bloom = 3; - } - } - - // Serialized as sorted distinct local physical row offsets within the fragment (0-based). - message UInt32List { - repeated uint32 values = 1; - } - - // An operation that updates rows but does not add or remove rows. - message Update { - // The fragments that have been removed. These are fragments where all rows - // have been updated and moved to a new fragment. - repeated uint64 removed_fragment_ids = 1; - // The fragments that have been updated. - repeated DataFragment updated_fragments = 2; - // The new fragments where updated rows have been moved to. - repeated DataFragment new_fragments = 3; - // The ids of the fields that have been modified. - repeated uint32 fields_modified = 4; - /// List of MemWAL shard generations to mark as merged after this transaction - repeated MergedGeneration merged_generations = 5; - /// The fields that used to judge whether to preserve the new frag's id into - /// the frag bitmap of the specified indices. - repeated uint32 fields_for_preserving_frag_bitmap = 6; - // The mode of update - UpdateMode update_mode = 7; - // Filter for checking existence of keys in newly inserted rows, used for conflict detection. - // Only tracks keys from INSERT operations during merge insert, not updates. - optional KeyExistenceFilter inserted_rows = 8; - // Per-fragment physical row offsets that matched an update_columns hash join (RewriteColumns). - map updated_fragment_offsets = 9; - } - - // The mode of update operation - enum UpdateMode { - - /// rows are deleted in current fragments and rewritten in new fragments. - /// This is most optimal when the majority of columns are being rewritten - /// or only a few rows are being updated. - REWRITE_ROWS = 0; - - /// within each fragment, columns are fully rewritten and inserted as new data files. - /// Old versions of columns are tombstoned. This is most optimal when most rows are affected - /// but a small subset of columns are affected. - REWRITE_COLUMNS = 1; - } - - // An entry for a map update. If value is not set, the key will be removed from the map. - message UpdateMapEntry { - // The key of the map entry to update. - string key = 1; - // The value to set for the key. - optional string value = 2; - } - - message UpdateMap { - repeated UpdateMapEntry update_entries = 1; - // If true, the map will be replaced entirely with the new entries. - // If false, the new entries will be merged with the existing map. - bool replace = 2; - } - - // An operation that updates the table config, table metadata, schema metadata, - // or field metadata. - message UpdateConfig { - UpdateMap config_updates = 6; - UpdateMap table_metadata_updates = 7; - UpdateMap schema_metadata_updates = 8; - map field_metadata_updates = 9; - - // Deprecated ------------------------------- - map upsert_values = 1; - repeated string delete_keys = 2; - map schema_metadata = 3; - map field_metadata = 4; - - message FieldMetadataUpdate { - map metadata = 5; - } - } - - message DataReplacementGroup { - uint64 fragment_id = 1; - DataFile new_file = 2; - } - - // An operation that replaces the data in a region of the table with new data. - message DataReplacement { - repeated DataReplacementGroup replacements = 1; - } - - // Update the merged generations in MemWAL index. - // This operation is used during merge-insert to atomically record which - // generations have been merged to the base table. - message UpdateMemWalState { - // Shards and generations being marked as merged. - repeated MergedGeneration merged_generations = 1; - } - - // An operation that updates base paths in the dataset. - message UpdateBases { - // The new base paths to add to the manifest. - repeated BasePath new_bases = 1; - } - - // The operation of this transaction. - oneof operation { - Append append = 100; - Delete delete = 101; - Overwrite overwrite = 102; - CreateIndex create_index = 103; - Rewrite rewrite = 104; - Merge merge = 105; - Restore restore = 106; - ReserveFragments reserve_fragments = 107; - Update update = 108; - Project project = 109; - UpdateConfig update_config = 110; - DataReplacement data_replacement = 111; - UpdateMemWalState update_mem_wal_state = 112; - Clone clone = 113; - UpdateBases update_bases = 114; - } - - // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. - reserved 200, 202; - reserved "blob_append", "blob_overwrite"; -} diff --git a/vendor/lance-table/src/feature_flags.rs b/vendor/lance-table/src/feature_flags.rs deleted file mode 100644 index 096f0da7..00000000 --- a/vendor/lance-table/src/feature_flags.rs +++ /dev/null @@ -1,184 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Feature flags - -use crate::format::Manifest; -use lance_core::{Error, Result}; - -/// Fragments may contain deletion files, which record the tombstones of -/// soft-deleted rows. -pub const FLAG_DELETION_FILES: u64 = 1; -/// Row ids are stable for both moves and updates. Fragments contain an index -/// mapping row ids to row addresses. -pub const FLAG_STABLE_ROW_IDS: u64 = 2; -/// Files are written with the new v2 format (this flag is no longer used) -pub const FLAG_USE_V2_FORMAT_DEPRECATED: u64 = 4; -/// Table config is present -pub const FLAG_TABLE_CONFIG: u64 = 8; -/// Dataset uses multiple base paths (for shallow clones or multi-base datasets) -pub const FLAG_BASE_PATHS: u64 = 16; -/// Disable writing transaction file under _transaction/, this flag is set when we only want to write inline transaction in manifest -pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 32; -/// The first bit that is unknown as a feature flag -pub const FLAG_UNKNOWN: u64 = 64; - -/// Set the reader and writer feature flags in the manifest based on the contents of the manifest. -pub fn apply_feature_flags( - manifest: &mut Manifest, - enable_stable_row_id: bool, - disable_transaction_file: bool, -) -> Result<()> { - // Reset flags - manifest.reader_feature_flags = 0; - manifest.writer_feature_flags = 0; - - let has_deletion_files = manifest - .fragments - .iter() - .any(|frag| frag.deletion_file.is_some()); - if has_deletion_files { - // Both readers and writers need to be able to read deletion files - manifest.reader_feature_flags |= FLAG_DELETION_FILES; - manifest.writer_feature_flags |= FLAG_DELETION_FILES; - } - - // If any fragment has row ids, they must all have row ids. - let has_row_ids = manifest - .fragments - .iter() - .any(|frag| frag.row_id_meta.is_some()); - if has_row_ids || enable_stable_row_id { - if !manifest - .fragments - .iter() - .all(|frag| frag.row_id_meta.is_some()) - { - return Err(Error::invalid_input("All fragments must have row ids")); - } - manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; - manifest.writer_feature_flags |= FLAG_STABLE_ROW_IDS; - } - - // Test whether any table metadata has been set - if !manifest.config.is_empty() { - manifest.writer_feature_flags |= FLAG_TABLE_CONFIG; - } - - // Check if this dataset uses multiple base paths (for shallow clones or multi-base datasets) - if !manifest.base_paths.is_empty() { - manifest.reader_feature_flags |= FLAG_BASE_PATHS; - manifest.writer_feature_flags |= FLAG_BASE_PATHS; - } - - if disable_transaction_file { - manifest.writer_feature_flags |= FLAG_DISABLE_TRANSACTION_FILE; - } - Ok(()) -} - -pub fn can_read_dataset(reader_flags: u64) -> bool { - reader_flags < FLAG_UNKNOWN -} - -pub fn can_write_dataset(writer_flags: u64) -> bool { - writer_flags < FLAG_UNKNOWN -} - -pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool { - writer_flags & FLAG_USE_V2_FORMAT_DEPRECATED != 0 -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::format::BasePath; - - #[test] - fn test_read_check() { - assert!(can_read_dataset(0)); - assert!(can_read_dataset(super::FLAG_DELETION_FILES)); - assert!(can_read_dataset(super::FLAG_STABLE_ROW_IDS)); - assert!(can_read_dataset(super::FLAG_USE_V2_FORMAT_DEPRECATED)); - assert!(can_read_dataset(super::FLAG_TABLE_CONFIG)); - assert!(can_read_dataset(super::FLAG_BASE_PATHS)); - assert!(can_read_dataset(super::FLAG_DISABLE_TRANSACTION_FILE)); - assert!(can_read_dataset( - super::FLAG_DELETION_FILES - | super::FLAG_STABLE_ROW_IDS - | super::FLAG_USE_V2_FORMAT_DEPRECATED - )); - assert!(!can_read_dataset(super::FLAG_UNKNOWN)); - } - - #[test] - fn test_write_check() { - assert!(can_write_dataset(0)); - assert!(can_write_dataset(super::FLAG_DELETION_FILES)); - assert!(can_write_dataset(super::FLAG_STABLE_ROW_IDS)); - assert!(can_write_dataset(super::FLAG_USE_V2_FORMAT_DEPRECATED)); - assert!(can_write_dataset(super::FLAG_TABLE_CONFIG)); - assert!(can_write_dataset(super::FLAG_BASE_PATHS)); - assert!(can_write_dataset(super::FLAG_DISABLE_TRANSACTION_FILE)); - assert!(can_write_dataset( - super::FLAG_DELETION_FILES - | super::FLAG_STABLE_ROW_IDS - | super::FLAG_USE_V2_FORMAT_DEPRECATED - | super::FLAG_TABLE_CONFIG - | super::FLAG_BASE_PATHS - )); - assert!(!can_write_dataset(super::FLAG_UNKNOWN)); - } - - #[test] - fn test_base_paths_feature_flags() { - use crate::format::{DataStorageFormat, Manifest}; - use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Schema; - use std::collections::HashMap; - use std::sync::Arc; - // Create a basic schema for testing - let arrow_schema = ArrowSchema::new(vec![ArrowField::new( - "test_field", - arrow_schema::DataType::Int64, - false, - )]); - let schema = Schema::try_from(&arrow_schema).unwrap(); - // Test 1: Normal dataset (no base_paths) should not have FLAG_BASE_PATHS - let mut normal_manifest = Manifest::new( - schema.clone(), - Arc::new(vec![]), - DataStorageFormat::default(), - HashMap::new(), // Empty base_paths - ); - apply_feature_flags(&mut normal_manifest, false, false).unwrap(); - assert_eq!(normal_manifest.reader_feature_flags & FLAG_BASE_PATHS, 0); - assert_eq!(normal_manifest.writer_feature_flags & FLAG_BASE_PATHS, 0); - // Test 2: Dataset with base_paths (shallow clone or multi-base) should have FLAG_BASE_PATHS - let mut base_paths: HashMap = HashMap::new(); - base_paths.insert( - 1, - BasePath::new( - 1, - "file:///path/to/original".to_string(), - Some("test_ref".to_string()), - true, - ), - ); - let mut multi_base_manifest = Manifest::new( - schema, - Arc::new(vec![]), - DataStorageFormat::default(), - base_paths, - ); - apply_feature_flags(&mut multi_base_manifest, false, false).unwrap(); - assert_ne!( - multi_base_manifest.reader_feature_flags & FLAG_BASE_PATHS, - 0 - ); - assert_ne!( - multi_base_manifest.writer_feature_flags & FLAG_BASE_PATHS, - 0 - ); - } -} diff --git a/vendor/lance-table/src/format.rs b/vendor/lance-table/src/format.rs deleted file mode 100644 index 842c76f1..00000000 --- a/vendor/lance-table/src/format.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use arrow_buffer::ToByteSlice; -use uuid::Uuid; - -mod fragment; -mod index; -mod manifest; -mod transaction; - -pub use crate::rowids::version::{ - RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, -}; -pub use fragment::*; -pub use index::{IndexFile, IndexMetadata, index_metadata_codec, list_index_files_with_sizes}; - -pub use manifest::{ - BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, SelfDescribingFileReader, - WriterVersion, is_detached_version, -}; -pub use transaction::Transaction; - -use lance_core::{Error, Result}; - -// In 0.36.1 we renamed Index to IndexMetadata because Index conflicted too much with the -// Index trait. This is left in for backward compatibility. -#[deprecated(since = "0.36.1", note = "Use IndexMetadata instead")] -pub type Index = IndexMetadata; - -/// Protobuf definitions for Lance Format -pub mod pb { - #![allow(clippy::all)] - #![allow(non_upper_case_globals)] - #![allow(non_camel_case_types)] - #![allow(non_snake_case)] - #![allow(unused)] - #![allow(improper_ctypes)] - #![allow(clippy::upper_case_acronyms)] - #![allow(clippy::use_self)] - include!(concat!(env!("OUT_DIR"), "/lance.table.rs")); -} - -/// These version/magic values are written at the end of manifest files (e.g. versions/1.version) -pub const MAJOR_VERSION: i16 = 0; -pub const MINOR_VERSION: i16 = 1; -pub const MAGIC: &[u8; 4] = b"LANC"; - -impl TryFrom<&pb::Uuid> for Uuid { - type Error = Error; - - fn try_from(p: &pb::Uuid) -> Result { - if p.uuid.len() != 16 { - return Err(Error::invalid_input( - "Protobuf UUID is malformed".to_string(), - )); - } - let mut buf: [u8; 16] = [0; 16]; - buf.copy_from_slice(p.uuid.to_byte_slice()); - Ok(Self::from_bytes(buf)) - } -} - -impl From<&Uuid> for pb::Uuid { - fn from(value: &Uuid) -> Self { - Self { - uuid: value.into_bytes().to_vec(), - } - } -} diff --git a/vendor/lance-table/src/format/fragment.rs b/vendor/lance-table/src/format/fragment.rs deleted file mode 100644 index dc5c94b3..00000000 --- a/vendor/lance-table/src/format/fragment.rs +++ /dev/null @@ -1,841 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::collections::HashMap; -use std::num::NonZero; -use std::sync::Arc; - -use deepsize::DeepSizeOf; -use lance_core::Error; -use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; -use lance_file::version::LanceFileVersion; -use lance_io::utils::CachedFileSize; -use object_store::path::Path; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -use crate::format::pb; - -use crate::rowids::version::{ - RowDatasetVersionMeta, created_at_version_meta_to_pb, last_updated_at_version_meta_to_pb, -}; -use lance_core::datatypes::Schema; -use lance_core::error::Result; - -/// Lance Data File -/// -/// A data file is one piece of file storing data. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] -pub struct DataFile { - /// Relative path of the data file to dataset root. - pub path: String, - /// The ids of fields in this file. - /// - /// When identical across many fragments (common case), multiple `DataFile` - /// instances share a single heap allocation via `Arc`, significantly - /// reducing manifest memory for large tables. - pub fields: Arc<[i32]>, - /// The offsets of the fields listed in `fields`, empty in v1 files - /// - /// Note that -1 is a possibility and it indices that the field has - /// no top-level column in the file. - /// - /// Columns that lack a field id may still exist as extra entries in - /// `column_indices`; such columns are ignored by field-id–based projection. - /// For example, some fields, such as blob fields, occupy multiple - /// columns in the file but only have a single field id. - pub column_indices: Arc<[i32]>, - /// The major version of the file format used to write this file. - pub file_major_version: u32, - /// The minor version of the file format used to write this file. - pub file_minor_version: u32, - - /// The size of the file in bytes, if known. - pub file_size_bytes: CachedFileSize, - - /// The base path of the datafile, when the datafile is outside the dataset. - pub base_id: Option, -} - -// Custom Serialize: convert Arc<[i32]> to slice for transparent JSON output -impl Serialize for DataFile { - fn serialize(&self, serializer: S) -> std::result::Result { - use serde::ser::SerializeStruct; - let mut s = serializer.serialize_struct("DataFile", 7)?; - s.serialize_field("path", &self.path)?; - s.serialize_field("fields", self.fields.as_ref())?; - s.serialize_field("column_indices", self.column_indices.as_ref())?; - s.serialize_field("file_major_version", &self.file_major_version)?; - s.serialize_field("file_minor_version", &self.file_minor_version)?; - s.serialize_field("file_size_bytes", &self.file_size_bytes)?; - s.serialize_field("base_id", &self.base_id)?; - s.end() - } -} - -// Custom Deserialize: read Vec and convert to Arc<[i32]> -impl<'de> Deserialize<'de> for DataFile { - fn deserialize>(deserializer: D) -> std::result::Result { - #[derive(Deserialize)] - struct DataFileHelper { - path: String, - fields: Vec, - #[serde(default)] - column_indices: Vec, - #[serde(default)] - file_major_version: u32, - #[serde(default)] - file_minor_version: u32, - file_size_bytes: CachedFileSize, - base_id: Option, - } - - let helper = DataFileHelper::deserialize(deserializer)?; - Ok(Self { - path: helper.path, - fields: Arc::from(helper.fields), - column_indices: Arc::from(helper.column_indices), - file_major_version: helper.file_major_version, - file_minor_version: helper.file_minor_version, - file_size_bytes: helper.file_size_bytes, - base_id: helper.base_id, - }) - } -} - -impl DataFile { - pub fn new( - path: impl Into, - fields: Vec, - column_indices: Vec, - file_major_version: u32, - file_minor_version: u32, - file_size_bytes: Option>, - base_id: Option, - ) -> Self { - Self { - path: path.into(), - fields: Arc::from(fields), - column_indices: Arc::from(column_indices), - file_major_version, - file_minor_version, - file_size_bytes: file_size_bytes.into(), - base_id, - } - } - - /// Create a new `DataFile` with the expectation that fields and column_indices will be set later - pub fn new_unstarted( - path: impl Into, - file_major_version: u32, - file_minor_version: u32, - ) -> Self { - Self { - path: path.into(), - fields: Arc::from([]), - column_indices: Arc::from([]), - file_major_version, - file_minor_version, - file_size_bytes: Default::default(), - base_id: None, - } - } - - pub fn new_legacy_from_fields( - path: impl Into, - fields: Vec, - base_id: Option, - ) -> Self { - Self::new( - path, - fields, - vec![], - MAJOR_VERSION as u32, - MINOR_VERSION as u32, - None, - base_id, - ) - } - - pub fn new_legacy( - path: impl Into, - schema: &Schema, - file_size_bytes: Option>, - base_id: Option, - ) -> Self { - let mut field_ids = schema.field_ids(); - field_ids.sort(); - Self::new( - path, - field_ids, - vec![], - MAJOR_VERSION as u32, - MINOR_VERSION as u32, - file_size_bytes, - base_id, - ) - } - - pub fn schema(&self, full_schema: &Schema) -> Schema { - full_schema.project_by_ids(&self.fields, false) - } - - pub fn is_legacy_file(&self) -> bool { - self.file_major_version == 0 && self.file_minor_version < 3 - } - - pub fn validate(&self, base_path: &Path) -> Result<()> { - if self.is_legacy_file() { - if !self.fields.windows(2).all(|w| w[0] < w[1]) { - return Err(Error::corrupt_file( - base_path.clone().join(self.path.clone()), - "contained unsorted or duplicate field ids", - )); - } - } else if self.column_indices.len() < self.fields.len() { - // Every recorded field id must have a column index, but not every column needs - // to be associated with a field id (extra columns are allowed). - return Err(Error::corrupt_file( - base_path.clone().join(self.path.clone()), - "contained fewer column_indices than fields", - )); - } - Ok(()) - } -} - -impl From<&DataFile> for pb::DataFile { - fn from(df: &DataFile) -> Self { - Self { - path: df.path.clone(), - fields: df.fields.to_vec(), - column_indices: df.column_indices.to_vec(), - file_major_version: df.file_major_version, - file_minor_version: df.file_minor_version, - file_size_bytes: df.file_size_bytes.get().map_or(0, |v| v.get()), - base_id: df.base_id, - } - } -} - -impl TryFrom for DataFile { - type Error = Error; - - fn try_from(proto: pb::DataFile) -> Result { - Ok(Self { - path: proto.path, - fields: Arc::from(proto.fields), - column_indices: Arc::from(proto.column_indices), - file_major_version: proto.file_major_version, - file_minor_version: proto.file_minor_version, - file_size_bytes: CachedFileSize::new(proto.file_size_bytes), - base_id: proto.base_id, - }) - } -} - -/// Interns repeated data so that fragments with identical content share a -/// single heap allocation via `Arc`. -/// -/// At 20M fragments the deduplication typically saves multiple GB of heap -/// because every fragment in a homogeneous table carries the same field list, -/// and post-compaction fragments share identical version metadata bytes. -/// -/// Uses a `Vec`-based linear scan when the cache is small (<=16 entries) -/// and upgrades to `HashMap` for larger caches. In the common homogeneous -/// case (1-3 unique values), linear scan avoids per-fragment hashing overhead. -#[derive(Default)] -pub struct DataFileFieldInterner { - fields: InternCache, - column_indices: InternCache, - inline_bytes: InternCache, -} - -/// A cache that uses linear scan for small sizes and HashMap for large. -/// The threshold is chosen so that scan + compare is cheaper than hash for -/// typical payload sizes (20-200 bytes). -enum InternCache { - Small(Vec>), - Large(HashMap, ()>), -} - -const INTERN_CACHE_UPGRADE_THRESHOLD: usize = 16; - -impl Default for InternCache { - fn default() -> Self { - Self::Small(Vec::new()) - } -} - -impl InternCache { - fn intern(&mut self, v: Vec) -> Arc<[T]> { - match self { - Self::Small(entries) => { - for existing in entries.iter() { - if existing.as_ref() == v.as_slice() { - return existing.clone(); - } - } - let arc: Arc<[T]> = Arc::from(v); - entries.push(arc.clone()); - if entries.len() > INTERN_CACHE_UPGRADE_THRESHOLD { - let mut map = HashMap::with_capacity(entries.len()); - for e in entries.drain(..) { - map.insert(e, ()); - } - *self = Self::Large(map); - } - arc - } - Self::Large(map) => { - if let Some((existing, _)) = map.get_key_value(v.as_slice()) { - existing.clone() - } else { - let arc: Arc<[T]> = Arc::from(v); - map.insert(arc.clone(), ()); - arc - } - } - } - } -} - -impl DataFileFieldInterner { - /// Intern a `RowDatasetVersionMeta`, deduplicating inline byte payloads. - /// Accepts the protobuf oneof value directly to avoid an intermediate - /// `Arc<[u8]>` allocation that would need to be `.to_vec()`'d for the key lookup. - fn intern_last_updated_version_meta( - cache: &mut InternCache, - pb: pb::data_fragment::LastUpdatedAtVersionSequence, - ) -> Result { - match pb { - pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(data) => { - Ok(RowDatasetVersionMeta::Inline(cache.intern(data))) - } - pb::data_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( - file, - ) => Ok(RowDatasetVersionMeta::External(ExternalFile { - path: file.path, - offset: file.offset, - size: file.size, - })), - } - } - - /// Intern a `RowDatasetVersionMeta`, deduplicating inline byte payloads. - fn intern_created_version_meta( - cache: &mut InternCache, - pb: pb::data_fragment::CreatedAtVersionSequence, - ) -> Result { - match pb { - pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => { - Ok(RowDatasetVersionMeta::Inline(cache.intern(data))) - } - pb::data_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => { - Ok(RowDatasetVersionMeta::External(ExternalFile { - path: file.path, - offset: file.offset, - size: file.size, - })) - } - } - } - - /// Convert a protobuf `DataFile`, interning `fields` and `column_indices`. - pub fn intern_data_file(&mut self, proto: pb::DataFile) -> Result { - Ok(DataFile { - path: proto.path, - fields: self.fields.intern(proto.fields), - column_indices: self.column_indices.intern(proto.column_indices), - file_major_version: proto.file_major_version, - file_minor_version: proto.file_minor_version, - file_size_bytes: CachedFileSize::new(proto.file_size_bytes), - base_id: proto.base_id, - }) - } - - /// Convert a protobuf `DataFragment`, interning fields and version metadata. - pub fn intern_fragment(&mut self, p: pb::DataFragment) -> Result { - let physical_rows = if p.physical_rows > 0 { - Some(p.physical_rows as usize) - } else { - None - }; - let last_updated_at_version_meta = p - .last_updated_at_version_sequence - .map(|pb| Self::intern_last_updated_version_meta(&mut self.inline_bytes, pb)) - .transpose()?; - let created_at_version_meta = p - .created_at_version_sequence - .map(|pb| Self::intern_created_version_meta(&mut self.inline_bytes, pb)) - .transpose()?; - Ok(Fragment { - id: p.id, - files: p - .files - .into_iter() - .map(|f| self.intern_data_file(f)) - .collect::>()?, - deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?, - row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?, - physical_rows, - last_updated_at_version_meta, - created_at_version_meta, - }) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -#[serde(rename_all = "lowercase")] -pub enum DeletionFileType { - Array, - Bitmap, -} - -impl DeletionFileType { - // TODO: pub(crate) - pub fn suffix(&self) -> &str { - match self { - Self::Array => "arrow", - Self::Bitmap => "bin", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct DeletionFile { - pub read_version: u64, - pub id: u64, - pub file_type: DeletionFileType, - /// Number of deleted rows in this file. If None, this is unknown. - pub num_deleted_rows: Option, - pub base_id: Option, -} - -impl TryFrom for DeletionFile { - type Error = Error; - - fn try_from(value: pb::DeletionFile) -> Result { - let file_type = match value.file_type { - 0 => DeletionFileType::Array, - 1 => DeletionFileType::Bitmap, - _ => { - return Err(Error::not_supported_source( - "Unknown deletion file type".into(), - )); - } - }; - let num_deleted_rows = if value.num_deleted_rows == 0 { - None - } else { - Some(value.num_deleted_rows as usize) - }; - Ok(Self { - read_version: value.read_version, - id: value.id, - file_type, - num_deleted_rows, - base_id: value.base_id, - }) - } -} - -/// A reference to a part of a file. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct ExternalFile { - pub path: String, - pub offset: u64, - pub size: u64, -} - -/// Metadata about location of the row id sequence. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub enum RowIdMeta { - Inline(Vec), - External(ExternalFile), -} - -impl TryFrom for RowIdMeta { - type Error = Error; - - fn try_from(value: pb::data_fragment::RowIdSequence) -> Result { - match value { - pb::data_fragment::RowIdSequence::InlineRowIds(data) => Ok(Self::Inline(data)), - pb::data_fragment::RowIdSequence::ExternalRowIds(file) => { - Ok(Self::External(ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - })) - } - } - } -} - -/// Data fragment. -/// -/// A fragment is a set of files which represent the different columns of the same rows. -/// If column exists in the schema, but the related file does not exist, treat this column as `nulls`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct Fragment { - /// Fragment ID - pub id: u64, - - /// Files within the fragment. - pub files: Vec, - - /// Optional file with deleted local row offsets. - #[serde(skip_serializing_if = "Option::is_none")] - pub deletion_file: Option, - - /// RowIndex - #[serde(skip_serializing_if = "Option::is_none")] - pub row_id_meta: Option, - - /// Original number of rows in the fragment. If this is None, then it is - /// unknown. This is only optional for legacy reasons. All new tables should - /// have this set. - pub physical_rows: Option, - - /// Last updated at version metadata - #[serde(skip_serializing_if = "Option::is_none")] - pub last_updated_at_version_meta: Option, - - /// Created at version metadata - #[serde(skip_serializing_if = "Option::is_none")] - pub created_at_version_meta: Option, -} - -impl Fragment { - pub fn new(id: u64) -> Self { - Self { - id, - files: vec![], - deletion_file: None, - row_id_meta: None, - physical_rows: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - } - } - - pub fn num_rows(&self) -> Option { - match (self.physical_rows, &self.deletion_file) { - // Known fragment length, no deletion file. - (Some(len), None) => Some(len), - // Known fragment length, but don't know deletion file size. - ( - Some(len), - Some(DeletionFile { - num_deleted_rows: Some(num_deleted_rows), - .. - }), - ) => Some(len - num_deleted_rows), - _ => None, - } - } - - pub fn from_json(json: &str) -> Result { - let fragment: Self = serde_json::from_str(json)?; - Ok(fragment) - } - - /// Create a `Fragment` with one DataFile - pub fn with_file_legacy( - id: u64, - path: &str, - schema: &Schema, - physical_rows: Option, - ) -> Self { - Self { - id, - files: vec![DataFile::new_legacy(path, schema, None, None)], - deletion_file: None, - physical_rows, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - } - } - - pub fn with_file( - mut self, - path: impl Into, - field_ids: Vec, - column_indices: Vec, - version: &LanceFileVersion, - file_size_bytes: Option>, - ) -> Self { - let (major, minor) = version.to_numbers(); - let data_file = DataFile::new( - path, - field_ids, - column_indices, - major, - minor, - file_size_bytes, - None, - ); - self.files.push(data_file); - self - } - - pub fn with_physical_rows(mut self, physical_rows: usize) -> Self { - self.physical_rows = Some(physical_rows); - self - } - - pub fn add_file( - &mut self, - path: impl Into, - field_ids: Vec, - column_indices: Vec, - version: &LanceFileVersion, - file_size_bytes: Option>, - ) { - let (major, minor) = version.to_numbers(); - self.files.push(DataFile::new( - path, - field_ids, - column_indices, - major, - minor, - file_size_bytes, - None, - )); - } - - /// Add a new [`DataFile`] to this fragment. - pub fn add_file_legacy(&mut self, path: &str, schema: &Schema) { - self.files - .push(DataFile::new_legacy(path, schema, None, None)); - } - - // True if this fragment is made up of legacy v1 files, false otherwise - pub fn has_legacy_files(&self) -> bool { - // If any file in a fragment is legacy then all files in the fragment must be - self.files[0].is_legacy_file() - } - - // Helper method to infer the Lance version from a set of fragments - // - // Returns None if there are no data files - // Returns an error if the data files have different versions - pub fn try_infer_version(fragments: &[Self]) -> Result> { - // Otherwise we need to check the actual file versions - // Determine version from first file - let Some(sample_file) = fragments - .iter() - .find(|f| !f.files.is_empty()) - .map(|f| &f.files[0]) - else { - return Ok(None); - }; - let file_version = LanceFileVersion::try_from_major_minor( - sample_file.file_major_version, - sample_file.file_minor_version, - )?; - // Ensure all files match - for frag in fragments { - for file in &frag.files { - let this_file_version = LanceFileVersion::try_from_major_minor( - file.file_major_version, - file.file_minor_version, - )?; - if file_version != this_file_version { - return Err(Error::invalid_input(format!( - "All data files must have the same version. Detected both {} and {}", - file_version, this_file_version - ))); - } - } - } - Ok(Some(file_version)) - } -} - -impl TryFrom for Fragment { - type Error = Error; - - fn try_from(p: pb::DataFragment) -> Result { - let physical_rows = if p.physical_rows > 0 { - Some(p.physical_rows as usize) - } else { - None - }; - Ok(Self { - id: p.id, - files: p - .files - .into_iter() - .map(DataFile::try_from) - .collect::>()?, - deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?, - row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?, - physical_rows, - last_updated_at_version_meta: p - .last_updated_at_version_sequence - .map(RowDatasetVersionMeta::try_from) - .transpose()?, - created_at_version_meta: p - .created_at_version_sequence - .map(RowDatasetVersionMeta::try_from) - .transpose()?, - }) - } -} - -impl From<&Fragment> for pb::DataFragment { - fn from(f: &Fragment) -> Self { - let deletion_file = f.deletion_file.as_ref().map(|f| { - let file_type = match f.file_type { - DeletionFileType::Array => pb::deletion_file::DeletionFileType::ArrowArray, - DeletionFileType::Bitmap => pb::deletion_file::DeletionFileType::Bitmap, - }; - pb::DeletionFile { - read_version: f.read_version, - id: f.id, - file_type: file_type.into(), - num_deleted_rows: f.num_deleted_rows.unwrap_or_default() as u64, - base_id: f.base_id, - } - }); - - let row_id_sequence = f.row_id_meta.as_ref().map(|m| match m { - RowIdMeta::Inline(data) => pb::data_fragment::RowIdSequence::InlineRowIds(data.clone()), - RowIdMeta::External(file) => { - pb::data_fragment::RowIdSequence::ExternalRowIds(pb::ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - }) - } - }); - let last_updated_at_version_sequence = - last_updated_at_version_meta_to_pb(&f.last_updated_at_version_meta); - let created_at_version_sequence = created_at_version_meta_to_pb(&f.created_at_version_meta); - Self { - id: f.id, - files: f.files.iter().map(pb::DataFile::from).collect(), - deletion_file, - row_id_sequence, - physical_rows: f.physical_rows.unwrap_or_default() as u64, - last_updated_at_version_sequence, - created_at_version_sequence, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use arrow_schema::{ - DataType, Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema, - }; - use object_store::path::Path; - use serde_json::{Value, json}; - - #[test] - fn test_new_fragment() { - let path = "foobar.lance"; - - let arrow_schema = ArrowSchema::new(vec![ - ArrowField::new( - "s", - DataType::Struct(ArrowFields::from(vec![ - ArrowField::new("si", DataType::Int32, false), - ArrowField::new("sb", DataType::Binary, true), - ])), - true, - ), - ArrowField::new("bool", DataType::Boolean, true), - ]); - let schema = Schema::try_from(&arrow_schema).unwrap(); - let fragment = Fragment::with_file_legacy(123, path, &schema, Some(10)); - - assert_eq!(123, fragment.id); - assert_eq!( - fragment.files, - vec![DataFile::new_legacy_from_fields( - path.to_string(), - vec![0, 1, 2, 3], - None, - )] - ) - } - - #[test] - fn test_roundtrip_fragment() { - let mut fragment = Fragment::new(123); - let schema = ArrowSchema::new(vec![ArrowField::new("x", DataType::Float16, true)]); - fragment.add_file_legacy("foobar.lance", &Schema::try_from(&schema).unwrap()); - fragment.deletion_file = Some(DeletionFile { - read_version: 123, - id: 456, - file_type: DeletionFileType::Array, - num_deleted_rows: Some(10), - base_id: None, - }); - - let proto = pb::DataFragment::from(&fragment); - let fragment2 = Fragment::try_from(proto).unwrap(); - assert_eq!(fragment, fragment2); - - fragment.deletion_file = None; - let proto = pb::DataFragment::from(&fragment); - let fragment2 = Fragment::try_from(proto).unwrap(); - assert_eq!(fragment, fragment2); - } - - #[test] - fn test_to_json() { - let mut fragment = Fragment::new(123); - let schema = ArrowSchema::new(vec![ArrowField::new("x", DataType::Float16, true)]); - fragment.add_file_legacy("foobar.lance", &Schema::try_from(&schema).unwrap()); - fragment.deletion_file = Some(DeletionFile { - read_version: 123, - id: 456, - file_type: DeletionFileType::Array, - num_deleted_rows: Some(10), - base_id: None, - }); - - let json = serde_json::to_string(&fragment).unwrap(); - - let value: Value = serde_json::from_str(&json).unwrap(); - assert_eq!( - value, - json!({ - "id": 123, - "files":[ - {"path": "foobar.lance", "fields": [0], "column_indices": [], - "file_major_version": MAJOR_VERSION, "file_minor_version": MINOR_VERSION, - "file_size_bytes": null, "base_id": null } - ], - "deletion_file": {"read_version": 123, "id": 456, "file_type": "array", - "num_deleted_rows": 10, "base_id": null}, - "physical_rows": None::}), - ); - - let frag2 = Fragment::from_json(&json).unwrap(); - assert_eq!(fragment, frag2); - } - - #[test] - fn data_file_validate_allows_extra_columns() { - let data_file = DataFile { - path: "foo.lance".to_string(), - fields: Arc::from([1, 2]), - // One extra column without a field id mapping - column_indices: Arc::from([0, 1, 2]), - file_major_version: MAJOR_VERSION as u32, - file_minor_version: MINOR_VERSION as u32, - file_size_bytes: Default::default(), - base_id: None, - }; - - let base_path = Path::from("base"); - data_file - .validate(&base_path) - .expect("validation should allow extra columns without field ids"); - } -} diff --git a/vendor/lance-table/src/format/index.rs b/vendor/lance-table/src/format/index.rs deleted file mode 100644 index 945d8364..00000000 --- a/vendor/lance-table/src/format/index.rs +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Metadata for index - -use std::collections::HashMap; -use std::sync::Arc; - -use chrono::{DateTime, Utc}; -use deepsize::DeepSizeOf; -use futures::StreamExt; -use lance_io::object_store::ObjectStore; -use object_store::path::Path; -use roaring::RoaringBitmap; -use uuid::Uuid; - -use super::pb; -use lance_core::{Error, Result}; - -/// Metadata about a single file within an index segment. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct IndexFile { - /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx") - pub path: String, - /// Size of the file in bytes - pub size_bytes: u64, -} - -/// Index metadata -#[derive(Debug, Clone, PartialEq)] -pub struct IndexMetadata { - /// Unique ID across all dataset versions. - pub uuid: Uuid, - - /// Fields to build the index. - pub fields: Vec, - - /// Human readable index name - pub name: String, - - /// The version of the dataset this index was last updated on - /// - /// This is set when the index is created (based on the version used to train the index) - /// This is updated when the index is updated or remapped - pub dataset_version: u64, - - /// The fragment ids this index covers. - /// - /// This may contain fragment ids that no longer exist in the dataset. - /// - /// If this is None, then this is unknown. - pub fragment_bitmap: Option, - - /// Metadata specific to the index type - /// - /// This is an Option because older versions of Lance may not have this defined. However, it should always - /// be present in newer versions. - pub index_details: Option>, - - /// The index version. - pub index_version: i32, - - /// Timestamp when the index was created - /// - /// This field is optional for backward compatibility. For existing indices created before - /// this field was added, this will be None. - pub created_at: Option>, - - /// The base path index of the index files. Used when the index is imported or referred from another dataset. - /// Lance uses it as key of the base_paths field in Manifest to determine the actual base path of the index files. - pub base_id: Option, - - /// List of files and their sizes for this index segment. - /// This enables skipping HEAD calls when opening indices and provides - /// visibility into index storage size via describe_indices(). - /// This is None if the file sizes are unknown. This happens for indices created - /// before this field was added. - pub files: Option>, -} - -impl IndexMetadata { - pub fn effective_fragment_bitmap( - &self, - existing_fragments: &RoaringBitmap, - ) -> Option { - let fragment_bitmap = self.fragment_bitmap.as_ref()?; - Some(fragment_bitmap & existing_fragments) - } - - /// Returns a map of relative file paths to their sizes. - /// Returns an empty map if file information is not available. - pub fn file_size_map(&self) -> HashMap { - self.files - .as_ref() - .map(|files| { - files - .iter() - .map(|f| (f.path.clone(), f.size_bytes)) - .collect() - }) - .unwrap_or_default() - } - - /// Returns the total size of all files in this index segment in bytes. - /// Returns None if file information is not available. - pub fn total_size_bytes(&self) -> Option { - self.files - .as_ref() - .map(|files| files.iter().map(|f| f.size_bytes).sum()) - } - - /// Returns the set of fragments which are part of the fragment bitmap - /// but no longer in the dataset. - pub fn deleted_fragment_bitmap( - &self, - existing_fragments: &RoaringBitmap, - ) -> Option { - let fragment_bitmap = self.fragment_bitmap.as_ref()?; - Some(fragment_bitmap - existing_fragments) - } -} - -impl DeepSizeOf for IndexMetadata { - fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize { - self.uuid.as_bytes().deep_size_of_children(context) - + self.fields.deep_size_of_children(context) - + self.name.deep_size_of_children(context) - + self.dataset_version.deep_size_of_children(context) - + self - .fragment_bitmap - .as_ref() - .map(|fragment_bitmap| fragment_bitmap.serialized_size()) - .unwrap_or(0) - + self.files.deep_size_of_children(context) - } -} - -impl TryFrom for IndexMetadata { - type Error = Error; - - fn try_from(proto: pb::IndexMetadata) -> Result { - let fragment_bitmap = if proto.fragment_bitmap.is_empty() { - None - } else { - Some(RoaringBitmap::deserialize_from( - &mut proto.fragment_bitmap.as_slice(), - )?) - }; - - let files = if proto.files.is_empty() { - None - } else { - Some( - proto - .files - .into_iter() - .map(|f| IndexFile { - path: f.path, - size_bytes: f.size_bytes, - }) - .collect(), - ) - }; - - Ok(Self { - uuid: proto.uuid.as_ref().map(Uuid::try_from).ok_or_else(|| { - Error::invalid_input("uuid field does not exist in Index metadata".to_string()) - })??, - name: proto.name, - fields: proto.fields, - dataset_version: proto.dataset_version, - fragment_bitmap, - index_details: proto.index_details.map(Arc::new), - index_version: proto.index_version.unwrap_or_default(), - created_at: proto.created_at.map(|ts| { - DateTime::from_timestamp_millis(ts as i64) - .expect("Invalid timestamp in index metadata") - }), - base_id: proto.base_id, - files, - }) - } -} - -impl From<&IndexMetadata> for pb::IndexMetadata { - fn from(idx: &IndexMetadata) -> Self { - let mut fragment_bitmap = Vec::new(); - if let Some(bitmap) = &idx.fragment_bitmap - && let Err(e) = bitmap.serialize_into(&mut fragment_bitmap) - { - // In theory, this should never error. But if we do, just - // recover gracefully. - log::error!("Failed to serialize fragment bitmap: {}", e); - fragment_bitmap.clear(); - } - - let files = idx - .files - .as_ref() - .map(|files| { - files - .iter() - .map(|f| pb::IndexFile { - path: f.path.clone(), - size_bytes: f.size_bytes, - }) - .collect() - }) - .unwrap_or_default(); - - Self { - uuid: Some((&idx.uuid).into()), - name: idx.name.clone(), - fields: idx.fields.clone(), - dataset_version: idx.dataset_version, - fragment_bitmap, - index_details: idx - .index_details - .as_ref() - .map(|details| details.as_ref().clone()), - index_version: Some(idx.index_version), - created_at: idx.created_at.map(|dt| dt.timestamp_millis() as u64), - base_id: idx.base_id, - files, - } - } -} - -/// Returns a [`CacheCodec`](lance_core::cache::CacheCodec) for `Vec`. -/// -/// Uses `pb::IndexSection` (which wraps `repeated IndexMetadata`) as the wire -/// format, reusing the existing `TryFrom`/`From` conversions. -/// -/// Uses [`CacheCodec::new`](lance_core::cache::CacheCodec::new) because the -/// orphan rule prevents `impl CacheCodecImpl for Vec`. -type ArcAny = Arc; - -fn serialize_index_metadata( - any: &ArcAny, - writer: &mut dyn std::io::Write, -) -> lance_core::Result<()> { - use prost::Message; - let vec = any - .downcast_ref::>() - .expect("index_metadata_codec: wrong type (this is a bug in the cache layer)"); - let section = pb::IndexSection { - indices: vec.iter().map(pb::IndexMetadata::from).collect(), - }; - writer.write_all(§ion.encode_to_vec())?; - Ok(()) -} - -fn deserialize_index_metadata(data: &bytes::Bytes) -> lance_core::Result { - use prost::Message; - let section = pb::IndexSection::decode(data.as_ref())?; - let indices: Vec = section - .indices - .into_iter() - .map(IndexMetadata::try_from) - .collect::>()?; - Ok(Arc::new(indices)) -} - -pub fn index_metadata_codec() -> lance_core::cache::CacheCodec { - lance_core::cache::CacheCodec::new(serialize_index_metadata, deserialize_index_metadata) -} - -/// List all files in an index directory with their sizes. -/// -/// Returns a list of `IndexFile` structs containing relative paths and sizes. -/// This is used to capture file metadata after index creation/modification. -pub async fn list_index_files_with_sizes( - object_store: &ObjectStore, - index_dir: &Path, -) -> Result> { - let mut files = Vec::new(); - let mut stream = object_store.read_dir_all(index_dir, None); - while let Some(meta) = stream.next().await { - let meta = meta?; - // Get relative path by stripping the index_dir prefix - let relative_path = meta - .location - .as_ref() - .strip_prefix(index_dir.as_ref()) - .map(|s| s.trim_start_matches('/').to_string()) - .unwrap_or_else(|| meta.location.filename().unwrap_or("").to_string()); - files.push(IndexFile { - path: relative_path, - size_bytes: meta.size, - }); - } - Ok(files) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - /// Demonstrates the pattern a disk-backed cache backend would use: - /// serialize entries to bytes, store in a key-value map, then - /// deserialize on retrieval. - #[test] - fn test_index_metadata_codec_roundtrip() { - let codec = index_metadata_codec(); - - let original = vec![ - IndexMetadata { - uuid: Uuid::new_v4(), - name: "my_index".to_string(), - fields: vec![0, 1], - dataset_version: 42, - fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2, 3])), - index_details: None, - index_version: 1, - created_at: None, - base_id: None, - files: Some(vec![IndexFile { - path: "index.idx".to_string(), - size_bytes: 1024, - }]), - }, - IndexMetadata { - uuid: Uuid::new_v4(), - name: "second_index".to_string(), - fields: vec![2], - dataset_version: 43, - fragment_bitmap: None, - index_details: None, - index_version: 2, - created_at: None, - base_id: Some(7), - files: None, - }, - ]; - - // Simulate a disk-backed store: HashMap> - let mut store: HashMap> = HashMap::new(); - - // Serialize into the store - let key = "dataset/v42/Vec".to_string(); - let mut buf = Vec::new(); - let entry: Arc = Arc::new(original.clone()); - codec.serialize(&entry, &mut buf).unwrap(); - store.insert(key.clone(), buf); - - // Deserialize from the store - let bytes = store.get(&key).unwrap(); - let recovered = codec - .deserialize(&bytes::Bytes::copy_from_slice(bytes)) - .unwrap(); - let recovered = recovered - .downcast::>() - .expect("downcast should succeed"); - - assert_eq!(original.len(), recovered.len()); - for (orig, rec) in original.iter().zip(recovered.iter()) { - assert_eq!(orig.uuid, rec.uuid); - assert_eq!(orig.name, rec.name); - assert_eq!(orig.fields, rec.fields); - assert_eq!(orig.dataset_version, rec.dataset_version); - assert_eq!(orig.fragment_bitmap, rec.fragment_bitmap); - assert_eq!(orig.index_version, rec.index_version); - assert_eq!(orig.base_id, rec.base_id); - assert_eq!(orig.files, rec.files); - } - } -} diff --git a/vendor/lance-table/src/format/manifest.rs b/vendor/lance-table/src/format/manifest.rs deleted file mode 100644 index d2b5f2d3..00000000 --- a/vendor/lance-table/src/format/manifest.rs +++ /dev/null @@ -1,1490 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use async_trait::async_trait; -use chrono::prelude::*; -use deepsize::DeepSizeOf; -use lance_file::datatypes::{Fields, FieldsWithMeta, populate_schema_dictionary}; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::version::{LEGACY_FORMAT_VERSION, LanceFileVersion}; -use lance_io::traits::{ProtoStruct, Reader}; -use object_store::path::Path; -use prost::Message; -use prost_types::Timestamp; -use std::collections::{BTreeMap, HashMap}; -use std::ops::Range; -use std::sync::Arc; - -use super::Fragment; -use crate::feature_flags::{FLAG_STABLE_ROW_IDS, has_deprecated_v2_feature_flag}; -use crate::format::fragment::DataFileFieldInterner; -use crate::format::pb; -use lance_core::cache::LanceCache; -use lance_core::datatypes::Schema; -use lance_core::{Error, Result}; -use lance_io::object_store::{ObjectStore, ObjectStoreRegistry}; -use lance_io::utils::read_struct; - -/// Manifest of a dataset -/// -/// * Schema -/// * Version -/// * Fragments. -/// * Indices. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct Manifest { - /// Dataset schema. - pub schema: Schema, - - /// Dataset version - pub version: u64, - - /// Branch name, None if the dataset is the main branch. - pub branch: Option, - - /// Version of the writer library that wrote this manifest. - pub writer_version: Option, - - /// Fragments, the pieces to build the dataset. - /// - /// This list is stored in order, sorted by fragment id. However, the fragment id - /// sequence may have gaps. - pub fragments: Arc>, - - /// The file position of the version aux data. - pub version_aux_data: usize, - - /// The file position of the index metadata. - pub index_section: Option, - - /// The creation timestamp with nanosecond resolution as 128-bit integer - pub timestamp_nanos: u128, - - /// An optional string tag for this version - pub tag: Option, - - /// The reader flags - pub reader_feature_flags: u64, - - /// The writer flags - pub writer_feature_flags: u64, - - /// The max fragment id used so far - /// None means never set, Some(0) means max ID used so far is 0 - pub max_fragment_id: Option, - - /// The path to the transaction file, relative to the root of the dataset - pub transaction_file: Option, - - /// The file position of the inline transaction content inside the manifest - pub transaction_section: Option, - - /// Precomputed logic offset of each fragment - /// accelerating the fragment search using offset ranges. - fragment_offsets: Vec, - - /// The max row id used so far. - pub next_row_id: u64, - - /// The storage format of the data files. - pub data_storage_format: DataStorageFormat, - - /// Table configuration. - pub config: HashMap, - - /// Table metadata. - /// - /// This is a key-value map that can be used to store arbitrary metadata - /// associated with the table. This is different than configuration, which - /// is used to tell libraries how to read, write, or manage the table. - pub table_metadata: HashMap, - - /* external base paths */ - pub base_paths: HashMap, -} - -// We use the most significant bit to indicate that a transaction is detached -pub const DETACHED_VERSION_MASK: u64 = 0x8000_0000_0000_0000; - -pub fn is_detached_version(version: u64) -> bool { - version & DETACHED_VERSION_MASK != 0 -} - -fn compute_fragment_offsets(fragments: &[Fragment]) -> Vec { - fragments - .iter() - .map(|f| f.num_rows().unwrap_or_default()) - .chain([0]) // Make the last offset to be the full-length of the dataset. - .scan(0_usize, |offset, len| { - let start = *offset; - *offset += len; - Some(start) - }) - .collect() -} - -#[derive(Default)] -pub struct ManifestSummary { - pub total_fragments: u64, - pub total_data_files: u64, - pub total_files_size: u64, - pub total_deletion_files: u64, - pub total_data_file_rows: u64, - pub total_deletion_file_rows: u64, - pub total_rows: u64, -} - -impl From for BTreeMap { - fn from(summary: ManifestSummary) -> Self { - let mut stats_map = Self::new(); - stats_map.insert( - "total_fragments".to_string(), - summary.total_fragments.to_string(), - ); - stats_map.insert( - "total_data_files".to_string(), - summary.total_data_files.to_string(), - ); - stats_map.insert( - "total_files_size".to_string(), - summary.total_files_size.to_string(), - ); - stats_map.insert( - "total_deletion_files".to_string(), - summary.total_deletion_files.to_string(), - ); - stats_map.insert( - "total_data_file_rows".to_string(), - summary.total_data_file_rows.to_string(), - ); - stats_map.insert( - "total_deletion_file_rows".to_string(), - summary.total_deletion_file_rows.to_string(), - ); - stats_map.insert("total_rows".to_string(), summary.total_rows.to_string()); - stats_map - } -} - -impl Manifest { - pub fn new( - schema: Schema, - fragments: Arc>, - data_storage_format: DataStorageFormat, - base_paths: HashMap, - ) -> Self { - let fragment_offsets = compute_fragment_offsets(&fragments); - - Self { - schema, - version: 1, - branch: None, - writer_version: Some(WriterVersion::default()), - fragments, - version_aux_data: 0, - index_section: None, - timestamp_nanos: 0, - tag: None, - reader_feature_flags: 0, - writer_feature_flags: 0, - max_fragment_id: None, - transaction_file: None, - transaction_section: None, - fragment_offsets, - next_row_id: 0, - data_storage_format, - config: HashMap::new(), - table_metadata: HashMap::new(), - base_paths, - } - } - - pub fn new_from_previous( - previous: &Self, - schema: Schema, - fragments: Arc>, - ) -> Self { - let fragment_offsets = compute_fragment_offsets(&fragments); - - Self { - schema, - version: previous.version + 1, - branch: previous.branch.clone(), - writer_version: Some(WriterVersion::default()), - fragments, - version_aux_data: 0, - index_section: None, // Caller should update index if they want to keep them. - timestamp_nanos: 0, // This will be set on commit - tag: None, - reader_feature_flags: 0, // These will be set on commit - writer_feature_flags: 0, // These will be set on commit - max_fragment_id: previous.max_fragment_id, - transaction_file: None, - transaction_section: None, - fragment_offsets, - next_row_id: previous.next_row_id, - data_storage_format: previous.data_storage_format.clone(), - config: previous.config.clone(), - table_metadata: previous.table_metadata.clone(), - base_paths: previous.base_paths.clone(), - } - } - - /// Performs a shallow_clone of the manifest entirely in memory without: - /// - Any persistent storage operations - /// - Modifications to the original data - /// - If the shallow clone is for branch, ref_name is the source branch - pub fn shallow_clone( - &self, - ref_name: Option, - ref_path: String, - ref_base_id: u32, - branch_name: Option, - transaction_file: String, - ) -> Self { - let cloned_fragments = self - .fragments - .as_ref() - .iter() - .map(|fragment| { - let mut cloned_fragment = fragment.clone(); - for file in &mut cloned_fragment.files { - if file.base_id.is_none() { - file.base_id = Some(ref_base_id); - } - } - - if let Some(deletion) = &mut cloned_fragment.deletion_file - && deletion.base_id.is_none() - { - deletion.base_id = Some(ref_base_id); - } - cloned_fragment - }) - .collect::>(); - - Self { - schema: self.schema.clone(), - version: self.version, - branch: branch_name, - writer_version: self.writer_version.clone(), - fragments: Arc::new(cloned_fragments), - version_aux_data: self.version_aux_data, - index_section: None, // These will be set on commit - timestamp_nanos: self.timestamp_nanos, - tag: None, - reader_feature_flags: 0, // These will be set on commit - writer_feature_flags: 0, // These will be set on commit - max_fragment_id: self.max_fragment_id, - transaction_file: Some(transaction_file), - transaction_section: None, - fragment_offsets: self.fragment_offsets.clone(), - next_row_id: self.next_row_id, - data_storage_format: self.data_storage_format.clone(), - config: self.config.clone(), - base_paths: { - let mut base_paths = self.base_paths.clone(); - let base_path = BasePath::new(ref_base_id, ref_path, ref_name, true); - base_paths.insert(ref_base_id, base_path); - base_paths - }, - table_metadata: self.table_metadata.clone(), - } - } - - /// Return the `timestamp_nanos` value as a Utc DateTime - pub fn timestamp(&self) -> DateTime { - let nanos = self.timestamp_nanos % 1_000_000_000; - let seconds = ((self.timestamp_nanos - nanos) / 1_000_000_000) as i64; - Utc.from_utc_datetime( - &DateTime::from_timestamp(seconds, nanos as u32) - .unwrap_or_default() - .naive_utc(), - ) - } - - /// Set the `timestamp_nanos` value from a Utc DateTime - pub fn set_timestamp(&mut self, nanos: u128) { - self.timestamp_nanos = nanos; - } - - /// Get a mutable reference to the config - pub fn config_mut(&mut self) -> &mut HashMap { - &mut self.config - } - - /// Get a mutable reference to the table metadata - pub fn table_metadata_mut(&mut self) -> &mut HashMap { - &mut self.table_metadata - } - - /// Get a mutable reference to the schema metadata - pub fn schema_metadata_mut(&mut self) -> &mut HashMap { - &mut self.schema.metadata - } - - /// Get a mutable reference to the field metadata for a specific field id - /// - /// Returns None if the field does not exist in the schema. - pub fn field_metadata_mut(&mut self, field_id: i32) -> Option<&mut HashMap> { - self.schema - .field_by_id_mut(field_id) - .map(|field| &mut field.metadata) - } - - /// Set the `config` from an iterator - #[deprecated(note = "Use config_mut() for direct access to config HashMap")] - pub fn update_config(&mut self, upsert_values: impl IntoIterator) { - self.config.extend(upsert_values); - } - - /// Delete `config` keys using a slice of keys - #[deprecated(note = "Use config_mut() for direct access to config HashMap")] - pub fn delete_config_keys(&mut self, delete_keys: &[&str]) { - self.config - .retain(|key, _| !delete_keys.contains(&key.as_str())); - } - - /// Replaces the schema metadata with the given key-value pairs. - #[deprecated(note = "Use schema_metadata_mut() for direct access to schema metadata HashMap")] - pub fn replace_schema_metadata(&mut self, new_metadata: HashMap) { - self.schema.metadata = new_metadata; - } - - /// Replaces the metadata of the field with the given id with the given key-value pairs. - /// - /// If the field does not exist in the schema, this is a no-op. - #[deprecated( - note = "Use field_metadata_mut(field_id) for direct access to field metadata HashMap" - )] - pub fn replace_field_metadata( - &mut self, - field_id: i32, - new_metadata: HashMap, - ) -> Result<()> { - if let Some(field) = self.schema.field_by_id_mut(field_id) { - field.metadata = new_metadata; - Ok(()) - } else { - Err(Error::invalid_input(format!( - "Field with id {} does not exist for replace_field_metadata", - field_id - ))) - } - } - - /// Check the current fragment list and update the high water mark - pub fn update_max_fragment_id(&mut self) { - // If there are no fragments, don't update max_fragment_id - if self.fragments.is_empty() { - return; - } - - let max_fragment_id = self - .fragments - .iter() - .map(|f| f.id) - .max() - .unwrap() // Safe because we checked fragments is not empty - .try_into() - .unwrap(); - - match self.max_fragment_id { - None => { - // First time being set - self.max_fragment_id = Some(max_fragment_id); - } - Some(current_max) => { - // Only update if the computed max is greater than current - // This preserves the high water mark even when fragments are deleted - if max_fragment_id > current_max { - self.max_fragment_id = Some(max_fragment_id); - } - } - } - } - - /// Return the max fragment id. - /// Note this does not support recycling of fragment ids. - /// - /// This will return None if there are no fragments and max_fragment_id was never set. - pub fn max_fragment_id(&self) -> Option { - if let Some(max_id) = self.max_fragment_id { - // Return the stored high water mark - Some(max_id.into()) - } else { - // Not yet set, compute from fragment list - self.fragments.iter().map(|f| f.id).max() - } - } - - /// Get the max used field id - /// - /// This is different than [Schema::max_field_id] because it also considers - /// the field ids in the data files that have been dropped from the schema. - pub fn max_field_id(&self) -> i32 { - let schema_max_id = self.schema.max_field_id().unwrap_or(-1); - let fragment_max_id = self - .fragments - .iter() - .flat_map(|f| f.files.iter().flat_map(|file| file.fields.iter())) - .max() - .copied(); - let fragment_max_id = fragment_max_id.unwrap_or(-1); - schema_max_id.max(fragment_max_id) - } - - /// Return the fragments that are newer than the given manifest. - /// Note this does not support recycling of fragment ids. - pub fn fragments_since(&self, since: &Self) -> Result> { - if since.version >= self.version { - return Err(Error::invalid_input(format!( - "fragments_since: given version {} is newer than manifest version {}", - since.version, self.version - ))); - } - let start = since.max_fragment_id(); - Ok(self - .fragments - .iter() - .filter(|&f| start.map(|s| f.id > s).unwrap_or(true)) - .cloned() - .collect()) - } - - /// Find the fragments that contain the rows, identified by the offset range. - /// - /// Note that the offsets are the logical offsets of rows, not row IDs. - /// - /// - /// Parameters - /// ---------- - /// range: `Range` - /// Offset range - /// - /// Returns - /// ------- - /// Vec<(usize, Fragment)> - /// A vector of `(starting_offset_of_fragment, fragment)` pairs. - /// - pub fn fragments_by_offset_range(&self, range: Range) -> Vec<(usize, &Fragment)> { - let start = range.start; - let end = range.end; - let idx = self - .fragment_offsets - .binary_search(&start) - .unwrap_or_else(|idx| idx - 1); - - let mut fragments = vec![]; - for i in idx..self.fragments.len() { - if self.fragment_offsets[i] >= end - || self.fragment_offsets[i] + self.fragments[i].num_rows().unwrap_or_default() - <= start - { - break; - } - fragments.push((self.fragment_offsets[i], &self.fragments[i])); - } - - fragments - } - - /// Whether the dataset uses stable row ids. - pub fn uses_stable_row_ids(&self) -> bool { - self.reader_feature_flags & FLAG_STABLE_ROW_IDS != 0 - } - - /// Creates a serialized copy of the manifest, suitable for IPC or temp storage - /// and can be used to create a dataset - pub fn serialized(&self) -> Vec { - let pb_manifest: pb::Manifest = self.into(); - pb_manifest.encode_to_vec() - } - - pub fn should_use_legacy_format(&self) -> bool { - self.data_storage_format.version == LEGACY_FORMAT_VERSION - } - - /// Get the summary information of a manifest. - /// - /// This function calculates various statistics about the manifest, including: - /// - total_files_size: Total size of all data files in bytes - /// - total_fragments: Total number of fragments in the dataset - /// - total_data_files: Total number of data files across all fragments - /// - total_deletion_files: Total number of deletion files - /// - total_data_file_rows: Total number of rows in data files - /// - total_deletion_file_rows: Total number of deleted rows in deletion files - /// - total_rows: Total number of rows in the dataset - pub fn summary(&self) -> ManifestSummary { - // Calculate total fragments - let mut summary = - self.fragments - .iter() - .fold(ManifestSummary::default(), |mut summary, f| { - // Count data files in the current fragment - summary.total_data_files += f.files.len() as u64; - // Sum the number of rows for the current fragment (if available) - if let Some(num_rows) = f.num_rows() { - summary.total_rows += num_rows as u64; - } - // Sum file sizes for all data files in the current fragment (if available) - for data_file in &f.files { - if let Some(size_bytes) = data_file.file_size_bytes.get() { - summary.total_files_size += size_bytes.get(); - } - } - // Check and count if the current fragment has a deletion file - if f.deletion_file.is_some() { - summary.total_deletion_files += 1; - } - // Sum the number of deleted rows from the deletion file (if available) - if let Some(deletion_file) = &f.deletion_file - && let Some(num_deleted) = deletion_file.num_deleted_rows - { - summary.total_deletion_file_rows += num_deleted as u64; - } - summary - }); - summary.total_fragments = self.fragments.len() as u64; - summary.total_data_file_rows = summary.total_rows + summary.total_deletion_file_rows; - - summary - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct BasePath { - pub id: u32, - pub name: Option, - pub is_dataset_root: bool, - /// The full URI string (e.g., "s3://bucket/path") - pub path: String, -} - -impl BasePath { - /// Create a new BasePath - /// - /// # Arguments - /// - /// * `id` - Unique identifier for this base path - /// * `path` - Full URI string (e.g., "s3://bucket/path", "/local/path") - /// * `name` - Optional human-readable name for this base - /// * `is_dataset_root` - Whether this is the dataset root or a data-only base - pub fn new(id: u32, path: String, name: Option, is_dataset_root: bool) -> Self { - Self { - id, - name, - is_dataset_root, - path, - } - } - - /// Extract the object store path from this BasePath's URI. - /// - /// This is a synchronous operation that parses the URI without initializing an object store. - pub fn extract_path(&self, registry: Arc) -> Result { - ObjectStore::extract_path_from_uri(registry, &self.path) - } -} - -impl DeepSizeOf for BasePath { - fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize { - self.name.deep_size_of_children(context) - + self.path.deep_size_of_children(context) * 2 - + size_of::() - } -} - -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct WriterVersion { - pub library: String, - pub version: String, - pub prerelease: Option, - pub build_metadata: Option, -} - -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct DataStorageFormat { - pub file_format: String, - pub version: String, -} - -const LANCE_FORMAT_NAME: &str = "lance"; - -impl DataStorageFormat { - pub fn new(version: LanceFileVersion) -> Self { - Self { - file_format: LANCE_FORMAT_NAME.to_string(), - version: version.resolve().to_string(), - } - } - - pub fn lance_file_version(&self) -> Result { - self.version.parse::() - } -} - -impl Default for DataStorageFormat { - fn default() -> Self { - Self::new(LanceFileVersion::default()) - } -} - -impl From for DataStorageFormat { - fn from(pb: pb::manifest::DataStorageFormat) -> Self { - Self { - file_format: pb.file_format, - version: pb.version, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VersionPart { - Major, - Minor, - Patch, -} - -fn bump_version(version: &mut semver::Version, part: VersionPart) { - match part { - VersionPart::Major => { - version.major += 1; - version.minor = 0; - version.patch = 0; - } - VersionPart::Minor => { - version.minor += 1; - version.patch = 0; - } - VersionPart::Patch => { - version.patch += 1; - } - } -} - -impl WriterVersion { - /// Split a version string into clean version (major.minor.patch), prerelease, and build metadata. - /// - /// Returns None if the input is not a valid semver string. - /// - /// For example: - /// - "2.0.0-rc.1" -> Some(("2.0.0", Some("rc.1"), None)) - /// - "2.0.0-rc.1+build.123" -> Some(("2.0.0", Some("rc.1"), Some("build.123"))) - /// - "2.0.0+build.123" -> Some(("2.0.0", None, Some("build.123"))) - /// - "not-a-version" -> None - fn split_version(full_version: &str) -> Option<(String, Option, Option)> { - let mut parsed = semver::Version::parse(full_version).ok()?; - - let prerelease = if parsed.pre.is_empty() { - None - } else { - Some(parsed.pre.to_string()) - }; - - let build_metadata = if parsed.build.is_empty() { - None - } else { - Some(parsed.build.to_string()) - }; - - // Remove prerelease and build metadata to get clean version - parsed.pre = semver::Prerelease::EMPTY; - parsed.build = semver::BuildMetadata::EMPTY; - Some((parsed.to_string(), prerelease, build_metadata)) - } - - /// Try to parse the version string as a semver string. Returns None if - /// not successful. - #[deprecated(note = "Use `lance_lib_version()` instead")] - pub fn semver(&self) -> Option<(u32, u32, u32, Option<&str>)> { - // First split by '-' to separate the version from the pre-release tag - let (version_part, tag) = if let Some(dash_idx) = self.version.find('-') { - ( - &self.version[..dash_idx], - Some(&self.version[dash_idx + 1..]), - ) - } else { - (self.version.as_str(), None) - }; - - let mut parts = version_part.split('.'); - let major = parts.next().unwrap_or("0").parse().ok()?; - let minor = parts.next().unwrap_or("0").parse().ok()?; - let patch = parts.next().unwrap_or("0").parse().ok()?; - - Some((major, minor, patch, tag)) - } - - /// If the library is "lance", parse the version as semver and return it. - /// Returns None if the library is not "lance" or the version cannot be parsed as semver. - /// - /// This method reconstructs the full semantic version by combining the version field - /// with the prerelease and build_metadata fields (if present). For example: - /// - version="2.0.0" + prerelease=Some("rc.1") -> "2.0.0-rc.1" - /// - version="2.0.0" + prerelease=Some("rc.1") + build_metadata=Some("build.123") -> "2.0.0-rc.1+build.123" - pub fn lance_lib_version(&self) -> Option { - if self.library != "lance" { - return None; - } - - let mut version = semver::Version::parse(&self.version).ok()?; - - if let Some(ref prerelease) = self.prerelease { - version.pre = semver::Prerelease::new(prerelease).ok()?; - } - - if let Some(ref build_metadata) = self.build_metadata { - version.build = semver::BuildMetadata::new(build_metadata).ok()?; - } - - Some(version) - } - - #[deprecated( - note = "Use `lance_lib_version()` instead, which safely checks the library field and returns Option" - )] - #[allow(deprecated)] - pub fn semver_or_panic(&self) -> (u32, u32, u32, Option<&str>) { - self.semver() - .unwrap_or_else(|| panic!("Invalid writer version: {}", self.version)) - } - - /// Check if this is a Lance library version older than the given major/minor/patch. - /// - /// # Panics - /// - /// Panics if the library is not "lance" or the version cannot be parsed as semver. - #[deprecated(note = "Use `lance_lib_version()` and its `older_than` method instead.")] - pub fn older_than(&self, major: u32, minor: u32, patch: u32) -> bool { - let version = self - .lance_lib_version() - .expect("Not lance library or invalid version"); - let other = semver::Version { - major: major.into(), - minor: minor.into(), - patch: patch.into(), - pre: semver::Prerelease::EMPTY, - build: semver::BuildMetadata::EMPTY, - }; - version < other - } - - #[deprecated(note = "This is meant for testing and will be made private in future version.")] - pub fn bump(&self, part: VersionPart, keep_tag: bool) -> Self { - let mut version = self.lance_lib_version().expect("Should be lance version"); - bump_version(&mut version, part); - if !keep_tag { - version.pre = semver::Prerelease::EMPTY; - } - let (clean_version, prerelease, build_metadata) = Self::split_version(&version.to_string()) - .expect("Bumped version should be valid semver"); - Self { - library: self.library.clone(), - version: clean_version, - prerelease, - build_metadata, - } - } -} - -impl Default for WriterVersion { - #[cfg(not(test))] - fn default() -> Self { - let full_version = env!("CARGO_PKG_VERSION"); - let (version, prerelease, build_metadata) = - Self::split_version(full_version).expect("CARGO_PKG_VERSION should be valid semver"); - Self { - library: "lance".to_string(), - version, - prerelease, - build_metadata, - } - } - - // Unit tests always run as if they are in the next version. - #[cfg(test)] - #[allow(deprecated)] - fn default() -> Self { - let full_version = env!("CARGO_PKG_VERSION"); - let (version, prerelease, build_metadata) = - Self::split_version(full_version).expect("CARGO_PKG_VERSION should be valid semver"); - Self { - library: "lance".to_string(), - version, - prerelease, - build_metadata, - } - .bump(VersionPart::Patch, true) - } -} - -impl ProtoStruct for Manifest { - type Proto = pb::Manifest; -} - -impl From for BasePath { - fn from(p: pb::BasePath) -> Self { - Self::new(p.id, p.path, p.name, p.is_dataset_root) - } -} - -impl From for pb::BasePath { - fn from(p: BasePath) -> Self { - Self { - id: p.id, - name: p.name, - is_dataset_root: p.is_dataset_root, - path: p.path, - } - } -} - -impl TryFrom for Manifest { - type Error = Error; - - fn try_from(p: pb::Manifest) -> Result { - let timestamp_nanos = p.timestamp.map(|ts| { - let sec = ts.seconds as u128 * 1e9 as u128; - let nanos = ts.nanos as u128; - sec + nanos - }); - // We only use the writer version if it is fully set. - let writer_version = match p.writer_version { - Some(pb::manifest::WriterVersion { - library, - version, - prerelease, - build_metadata, - }) => Some(WriterVersion { - library, - version, - prerelease, - build_metadata, - }), - _ => None, - }; - let mut interner = DataFileFieldInterner::default(); - let fragments = Arc::new( - p.fragments - .into_iter() - .map(|f| interner.intern_fragment(f)) - .collect::>>()?, - ); - let fragment_offsets = compute_fragment_offsets(fragments.as_slice()); - let fields_with_meta = FieldsWithMeta { - fields: Fields(p.fields), - metadata: p.schema_metadata, - }; - - if FLAG_STABLE_ROW_IDS & p.reader_feature_flags != 0 - && !fragments.iter().all(|frag| frag.row_id_meta.is_some()) - { - return Err(Error::internal("All fragments must have row ids")); - } - - let data_storage_format = match p.data_format { - None => { - if let Some(inferred_version) = Fragment::try_infer_version(fragments.as_ref())? { - // If there are fragments, they are a better indicator - DataStorageFormat::new(inferred_version) - } else { - // No fragments to inspect, best we can do is look at writer flags - if has_deprecated_v2_feature_flag(p.writer_feature_flags) { - DataStorageFormat::new(LanceFileVersion::Stable) - } else { - DataStorageFormat::new(LanceFileVersion::Legacy) - } - } - } - Some(format) => DataStorageFormat::from(format), - }; - - let schema = Schema::from(fields_with_meta); - - Ok(Self { - schema, - version: p.version, - branch: p.branch, - writer_version, - version_aux_data: p.version_aux_data as usize, - index_section: p.index_section.map(|i| i as usize), - timestamp_nanos: timestamp_nanos.unwrap_or(0), - tag: if p.tag.is_empty() { None } else { Some(p.tag) }, - reader_feature_flags: p.reader_feature_flags, - writer_feature_flags: p.writer_feature_flags, - max_fragment_id: p.max_fragment_id, - fragments, - transaction_file: if p.transaction_file.is_empty() { - None - } else { - Some(p.transaction_file) - }, - transaction_section: p.transaction_section.map(|i| i as usize), - fragment_offsets, - next_row_id: p.next_row_id, - data_storage_format, - config: p.config, - table_metadata: p.table_metadata, - base_paths: p - .base_paths - .iter() - .map(|item| (item.id, item.clone().into())) - .collect(), - }) - } -} - -impl From<&Manifest> for pb::Manifest { - fn from(m: &Manifest) -> Self { - let timestamp_nanos = if m.timestamp_nanos == 0 { - None - } else { - let nanos = m.timestamp_nanos % 1e9 as u128; - let seconds = ((m.timestamp_nanos - nanos) / 1e9 as u128) as i64; - Some(Timestamp { - seconds, - nanos: nanos as i32, - }) - }; - let fields_with_meta: FieldsWithMeta = (&m.schema).into(); - Self { - fields: fields_with_meta.fields.0, - schema_metadata: m - .schema - .metadata - .iter() - .map(|(k, v)| (k.clone(), v.as_bytes().to_vec())) - .collect(), - version: m.version, - branch: m.branch.clone(), - writer_version: m - .writer_version - .as_ref() - .map(|wv| pb::manifest::WriterVersion { - library: wv.library.clone(), - version: wv.version.clone(), - prerelease: wv.prerelease.clone(), - build_metadata: wv.build_metadata.clone(), - }), - fragments: m.fragments.iter().map(pb::DataFragment::from).collect(), - table_metadata: m.table_metadata.clone(), - version_aux_data: m.version_aux_data as u64, - index_section: m.index_section.map(|i| i as u64), - timestamp: timestamp_nanos, - tag: m.tag.clone().unwrap_or_default(), - reader_feature_flags: m.reader_feature_flags, - writer_feature_flags: m.writer_feature_flags, - max_fragment_id: m.max_fragment_id, - transaction_file: m.transaction_file.clone().unwrap_or_default(), - next_row_id: m.next_row_id, - data_format: Some(pb::manifest::DataStorageFormat { - file_format: m.data_storage_format.file_format.clone(), - version: m.data_storage_format.version.clone(), - }), - config: m.config.clone(), - base_paths: m - .base_paths - .values() - .map(|base_path| pb::BasePath { - id: base_path.id, - name: base_path.name.clone(), - is_dataset_root: base_path.is_dataset_root, - path: base_path.path.clone(), - }) - .collect(), - transaction_section: m.transaction_section.map(|i| i as u64), - } - } -} - -#[async_trait] -pub trait SelfDescribingFileReader { - /// Open a file reader without any cached schema - /// - /// In this case the schema will first need to be loaded - /// from the file itself. - /// - /// When loading files from a dataset it is preferable to use - /// the fragment reader to avoid this overhead. - async fn try_new_self_described( - object_store: &ObjectStore, - path: &Path, - cache: Option<&LanceCache>, - ) -> Result - where - Self: Sized, - { - let reader = object_store.open(path).await?; - Self::try_new_self_described_from_reader(reader.into(), cache).await - } - - async fn try_new_self_described_from_reader( - reader: Arc, - cache: Option<&LanceCache>, - ) -> Result - where - Self: Sized; -} - -#[async_trait] -impl SelfDescribingFileReader for PreviousFileReader { - async fn try_new_self_described_from_reader( - reader: Arc, - cache: Option<&LanceCache>, - ) -> Result { - let metadata = Self::read_metadata(reader.as_ref(), cache).await?; - let manifest_position = metadata.manifest_position.ok_or(Error::internal(format!( - "Attempt to open file at {} as self-describing but it did not contain a manifest", - reader.path(), - )))?; - let mut manifest: Manifest = read_struct(reader.as_ref(), manifest_position).await?; - if manifest.should_use_legacy_format() { - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; - } - let schema = manifest.schema; - let max_field_id = schema.max_field_id().unwrap_or_default(); - Self::try_new_from_reader( - reader.path(), - reader.clone(), - Some(metadata), - schema, - 0, - 0, - max_field_id, - cache, - ) - .await - } -} - -#[cfg(test)] -mod tests { - use crate::format::{DataFile, DeletionFile, DeletionFileType}; - use std::num::NonZero; - - use super::*; - - use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Field; - - #[test] - fn test_writer_version() { - let wv = WriterVersion::default(); - assert_eq!(wv.library, "lance"); - - // Parse the actual cargo version to check if it has a pre-release tag - let cargo_version = env!("CARGO_PKG_VERSION"); - let expected_tag = if cargo_version.contains('-') { - Some(cargo_version.split('-').nth(1).unwrap()) - } else { - None - }; - - // Verify the version field contains only major.minor.patch - let version_parts: Vec<&str> = wv.version.split('.').collect(); - assert_eq!( - version_parts.len(), - 3, - "Version should be major.minor.patch" - ); - assert!( - !wv.version.contains('-'), - "Version field should not contain prerelease" - ); - - // Verify the prerelease field matches the expected tag - assert_eq!(wv.prerelease.as_deref(), expected_tag); - // Build metadata should be None for default version - assert_eq!(wv.build_metadata, None); - - // Verify lance_lib_version() reconstructs the full semver correctly - let version = wv.lance_lib_version().unwrap(); - assert_eq!( - version.major, - env!("CARGO_PKG_VERSION_MAJOR").parse::().unwrap() - ); - assert_eq!( - version.minor, - env!("CARGO_PKG_VERSION_MINOR").parse::().unwrap() - ); - assert_eq!( - version.patch, - // Unit tests run against (major,minor,patch + 1) - env!("CARGO_PKG_VERSION_PATCH").parse::().unwrap() + 1 - ); - assert_eq!(version.pre.as_str(), expected_tag.unwrap_or("")); - - for part in &[VersionPart::Major, VersionPart::Minor, VersionPart::Patch] { - let mut bumped_version = version.clone(); - bump_version(&mut bumped_version, *part); - assert!(version < bumped_version); - } - } - - #[test] - fn test_writer_version_split() { - // Test splitting version with prerelease - let (version, prerelease, build_metadata) = - WriterVersion::split_version("2.0.0-rc.1").unwrap(); - assert_eq!(version, "2.0.0"); - assert_eq!(prerelease, Some("rc.1".to_string())); - assert_eq!(build_metadata, None); - - // Test splitting version without prerelease - let (version, prerelease, build_metadata) = WriterVersion::split_version("2.0.0").unwrap(); - assert_eq!(version, "2.0.0"); - assert_eq!(prerelease, None); - assert_eq!(build_metadata, None); - - // Test splitting version with prerelease and build metadata - let (version, prerelease, build_metadata) = - WriterVersion::split_version("2.0.0-rc.1+build.123").unwrap(); - assert_eq!(version, "2.0.0"); - assert_eq!(prerelease, Some("rc.1".to_string())); - assert_eq!(build_metadata, Some("build.123".to_string())); - - // Test splitting version with only build metadata - let (version, prerelease, build_metadata) = - WriterVersion::split_version("2.0.0+build.123").unwrap(); - assert_eq!(version, "2.0.0"); - assert_eq!(prerelease, None); - assert_eq!(build_metadata, Some("build.123".to_string())); - - // Test with invalid version returns None - assert!(WriterVersion::split_version("not-a-version").is_none()); - } - - #[test] - fn test_writer_version_comparison_with_prerelease() { - let v1 = WriterVersion { - library: "lance".to_string(), - version: "2.0.0".to_string(), - prerelease: Some("rc.1".to_string()), - build_metadata: None, - }; - - let v2 = WriterVersion { - library: "lance".to_string(), - version: "2.0.0".to_string(), - prerelease: None, - build_metadata: None, - }; - - let semver1 = v1.lance_lib_version().unwrap(); - let semver2 = v2.lance_lib_version().unwrap(); - - // rc.1 should be less than the release version - assert!(semver1 < semver2); - } - - #[test] - fn test_writer_version_with_build_metadata() { - let v = WriterVersion { - library: "lance".to_string(), - version: "2.0.0".to_string(), - prerelease: Some("rc.1".to_string()), - build_metadata: Some("build.123".to_string()), - }; - - let semver = v.lance_lib_version().unwrap(); - assert_eq!(semver.to_string(), "2.0.0-rc.1+build.123"); - assert_eq!(semver.major, 2); - assert_eq!(semver.minor, 0); - assert_eq!(semver.patch, 0); - assert_eq!(semver.pre.as_str(), "rc.1"); - assert_eq!(semver.build.as_str(), "build.123"); - } - - #[test] - fn test_writer_version_non_semver() { - // Test that Lance library can have non-semver version strings - let v = WriterVersion { - library: "lance".to_string(), - version: "custom-build-v1".to_string(), - prerelease: None, - build_metadata: None, - }; - - // lance_lib_version should return None for non-semver - assert!(v.lance_lib_version().is_none()); - - // But the WriterVersion itself should still be valid and usable - assert_eq!(v.library, "lance"); - assert_eq!(v.version, "custom-build-v1"); - } - - #[test] - #[allow(deprecated)] - fn test_older_than_with_prerelease() { - // Test that older_than correctly handles prerelease - let v_rc = WriterVersion { - library: "lance".to_string(), - version: "2.0.0".to_string(), - prerelease: Some("rc.1".to_string()), - build_metadata: None, - }; - - // 2.0.0-rc.1 should be older than 2.0.0 - assert!(v_rc.older_than(2, 0, 0)); - - // 2.0.0-rc.1 should be older than 2.0.1 - assert!(v_rc.older_than(2, 0, 1)); - - // 2.0.0-rc.1 should not be older than 1.9.9 - assert!(!v_rc.older_than(1, 9, 9)); - - let v_release = WriterVersion { - library: "lance".to_string(), - version: "2.0.0".to_string(), - prerelease: None, - build_metadata: None, - }; - - // 2.0.0 should not be older than 2.0.0 - assert!(!v_release.older_than(2, 0, 0)); - - // 2.0.0 should be older than 2.0.1 - assert!(v_release.older_than(2, 0, 1)); - } - - #[test] - fn test_fragments_by_offset_range() { - let arrow_schema = ArrowSchema::new(vec![ArrowField::new( - "a", - arrow_schema::DataType::Int64, - false, - )]); - let schema = Schema::try_from(&arrow_schema).unwrap(); - let fragments = vec![ - Fragment::with_file_legacy(0, "path1", &schema, Some(10)), - Fragment::with_file_legacy(1, "path2", &schema, Some(15)), - Fragment::with_file_legacy(2, "path3", &schema, Some(20)), - ]; - let manifest = Manifest::new( - schema, - Arc::new(fragments), - DataStorageFormat::default(), - HashMap::new(), - ); - - let actual = manifest.fragments_by_offset_range(0..10); - assert_eq!(actual.len(), 1); - assert_eq!(actual[0].0, 0); - assert_eq!(actual[0].1.id, 0); - - let actual = manifest.fragments_by_offset_range(5..15); - assert_eq!(actual.len(), 2); - assert_eq!(actual[0].0, 0); - assert_eq!(actual[0].1.id, 0); - assert_eq!(actual[1].0, 10); - assert_eq!(actual[1].1.id, 1); - - let actual = manifest.fragments_by_offset_range(15..50); - assert_eq!(actual.len(), 2); - assert_eq!(actual[0].0, 10); - assert_eq!(actual[0].1.id, 1); - assert_eq!(actual[1].0, 25); - assert_eq!(actual[1].1.id, 2); - - // Out of range - let actual = manifest.fragments_by_offset_range(45..100); - assert!(actual.is_empty()); - - assert!(manifest.fragments_by_offset_range(200..400).is_empty()); - } - - #[test] - fn test_max_field_id() { - // Validate that max field id handles varying field ids by fragment. - let mut field0 = - Field::try_from(ArrowField::new("a", arrow_schema::DataType::Int64, false)).unwrap(); - field0.set_id(-1, &mut 0); - let mut field2 = - Field::try_from(ArrowField::new("b", arrow_schema::DataType::Int64, false)).unwrap(); - field2.set_id(-1, &mut 2); - - let schema = Schema { - fields: vec![field0, field2], - metadata: Default::default(), - }; - let fragments = vec![ - Fragment { - id: 0, - files: vec![DataFile::new_legacy_from_fields( - "path1", - vec![0, 1, 2], - None, - )], - deletion_file: None, - row_id_meta: None, - physical_rows: None, - created_at_version_meta: None, - last_updated_at_version_meta: None, - }, - Fragment { - id: 1, - files: vec![ - DataFile::new_legacy_from_fields("path2", vec![0, 1, 43], None), - DataFile::new_legacy_from_fields("path3", vec![2], None), - ], - deletion_file: None, - row_id_meta: None, - physical_rows: None, - created_at_version_meta: None, - last_updated_at_version_meta: None, - }, - ]; - - let manifest = Manifest::new( - schema, - Arc::new(fragments), - DataStorageFormat::default(), - HashMap::new(), - ); - - assert_eq!(manifest.max_field_id(), 43); - } - - #[test] - fn test_config() { - let arrow_schema = ArrowSchema::new(vec![ArrowField::new( - "a", - arrow_schema::DataType::Int64, - false, - )]); - let schema = Schema::try_from(&arrow_schema).unwrap(); - let fragments = vec![ - Fragment::with_file_legacy(0, "path1", &schema, Some(10)), - Fragment::with_file_legacy(1, "path2", &schema, Some(15)), - Fragment::with_file_legacy(2, "path3", &schema, Some(20)), - ]; - let mut manifest = Manifest::new( - schema, - Arc::new(fragments), - DataStorageFormat::default(), - HashMap::new(), - ); - - let mut config = manifest.config.clone(); - config.insert("lance.test".to_string(), "value".to_string()); - config.insert("other-key".to_string(), "other-value".to_string()); - - manifest.config_mut().extend(config.clone()); - assert_eq!(manifest.config, config.clone()); - - config.remove("other-key"); - manifest.config_mut().remove("other-key"); - assert_eq!(manifest.config, config); - } - - #[test] - fn test_manifest_summary() { - // Step 1: test empty manifest summary - let arrow_schema = ArrowSchema::new(vec![ - ArrowField::new("id", arrow_schema::DataType::Int64, false), - ArrowField::new("name", arrow_schema::DataType::Utf8, true), - ]); - let schema = Schema::try_from(&arrow_schema).unwrap(); - - let empty_manifest = Manifest::new( - schema.clone(), - Arc::new(vec![]), - DataStorageFormat::default(), - HashMap::new(), - ); - - let empty_summary = empty_manifest.summary(); - assert_eq!(empty_summary.total_rows, 0); - assert_eq!(empty_summary.total_files_size, 0); - assert_eq!(empty_summary.total_fragments, 0); - assert_eq!(empty_summary.total_data_files, 0); - assert_eq!(empty_summary.total_deletion_file_rows, 0); - assert_eq!(empty_summary.total_data_file_rows, 0); - assert_eq!(empty_summary.total_deletion_files, 0); - - // Step 2: write empty files and verify summary - let empty_fragments = vec![ - Fragment::with_file_legacy(0, "empty_file1.lance", &schema, Some(0)), - Fragment::with_file_legacy(1, "empty_file2.lance", &schema, Some(0)), - ]; - - let empty_files_manifest = Manifest::new( - schema.clone(), - Arc::new(empty_fragments), - DataStorageFormat::default(), - HashMap::new(), - ); - - let empty_files_summary = empty_files_manifest.summary(); - assert_eq!(empty_files_summary.total_rows, 0); - assert_eq!(empty_files_summary.total_files_size, 0); - assert_eq!(empty_files_summary.total_fragments, 2); - assert_eq!(empty_files_summary.total_data_files, 2); - assert_eq!(empty_files_summary.total_deletion_file_rows, 0); - assert_eq!(empty_files_summary.total_data_file_rows, 0); - assert_eq!(empty_files_summary.total_deletion_files, 0); - - // Step 3: write real data and verify summary - let real_fragments = vec![ - Fragment::with_file_legacy(0, "data_file1.lance", &schema, Some(100)), - Fragment::with_file_legacy(1, "data_file2.lance", &schema, Some(250)), - Fragment::with_file_legacy(2, "data_file3.lance", &schema, Some(75)), - ]; - - let real_data_manifest = Manifest::new( - schema.clone(), - Arc::new(real_fragments), - DataStorageFormat::default(), - HashMap::new(), - ); - - let real_data_summary = real_data_manifest.summary(); - assert_eq!(real_data_summary.total_rows, 425); // 100 + 250 + 75 - assert_eq!(real_data_summary.total_files_size, 0); // Zero for unknown - assert_eq!(real_data_summary.total_fragments, 3); - assert_eq!(real_data_summary.total_data_files, 3); - assert_eq!(real_data_summary.total_deletion_file_rows, 0); - assert_eq!(real_data_summary.total_data_file_rows, 425); - assert_eq!(real_data_summary.total_deletion_files, 0); - - let file_version = LanceFileVersion::default(); - // Step 4: write deletion files and verify summary - let mut fragment_with_deletion = Fragment::new(0) - .with_file( - "data_with_deletion.lance", - vec![0, 1], - vec![0, 1], - &file_version, - NonZero::new(1000), - ) - .with_physical_rows(50); - fragment_with_deletion.deletion_file = Some(DeletionFile { - read_version: 123, - id: 456, - file_type: DeletionFileType::Array, - num_deleted_rows: Some(10), - base_id: None, - }); - - let manifest_with_deletion = Manifest::new( - schema, - Arc::new(vec![fragment_with_deletion]), - DataStorageFormat::default(), - HashMap::new(), - ); - - let deletion_summary = manifest_with_deletion.summary(); - assert_eq!(deletion_summary.total_rows, 40); // 50 - 10 - assert_eq!(deletion_summary.total_files_size, 1000); - assert_eq!(deletion_summary.total_fragments, 1); - assert_eq!(deletion_summary.total_data_files, 1); - assert_eq!(deletion_summary.total_deletion_file_rows, 10); - assert_eq!(deletion_summary.total_data_file_rows, 50); - assert_eq!(deletion_summary.total_deletion_files, 1); - - //Just verify the transformation is OK - let stats_map: BTreeMap = deletion_summary.into(); - assert_eq!(stats_map.len(), 7) - } -} diff --git a/vendor/lance-table/src/format/transaction.rs b/vendor/lance-table/src/format/transaction.rs deleted file mode 100755 index e9d0bf42..00000000 --- a/vendor/lance-table/src/format/transaction.rs +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Transaction struct for lance-table format layer. -//! -//! This struct is introduced to provide a Struct-first API for passing transaction -//! information within the lance-table crate. It mirrors the protobuf Transaction -//! message at a semantic level while remaining crate-local, so lance-table does -//! not depend on higher layers (e.g., lance crate). -//! -//! Conversion to protobuf occurs at the write boundary. See the `From` -//! implementation below. - -use crate::format::pb; - -#[derive(Clone, Debug, PartialEq)] -pub struct Transaction { - /// Crate-local representation backing: protobuf Transaction. - /// Keeping this simple avoids ring dependencies while still enabling - /// Struct-first parameter passing in lance-table. - pub inner: pb::Transaction, -} - -impl Transaction { - /// Accessor for testing or internal inspection if needed. - pub fn as_pb(&self) -> &pb::Transaction { - &self.inner - } -} - -/// Write-boundary conversion: serialize using protobuf at the last step. -impl From for pb::Transaction { - fn from(tx: Transaction) -> Self { - tx.inner - } -} - -impl From for Transaction { - fn from(pb_tx: pb::Transaction) -> Self { - Self { inner: pb_tx } - } -} diff --git a/vendor/lance-table/src/io.rs b/vendor/lance-table/src/io.rs deleted file mode 100644 index 8a8151a2..00000000 --- a/vendor/lance-table/src/io.rs +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -pub mod commit; -pub mod deletion; -pub mod manifest; diff --git a/vendor/lance-table/src/io/commit.rs b/vendor/lance-table/src/io/commit.rs deleted file mode 100644 index 5dbf6200..00000000 --- a/vendor/lance-table/src/io/commit.rs +++ /dev/null @@ -1,1898 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Trait for commit implementations. -//! -//! In Lance, a transaction is committed by writing the next manifest file. -//! However, care should be taken to ensure that the manifest file is written -//! only once, even if there are concurrent writers. Different stores have -//! different abilities to handle concurrent writes, so a trait is provided -//! to allow for different implementations. -//! -//! The trait [CommitHandler] can be implemented to provide different commit -//! strategies. The default implementation for most object stores is -//! [RenameCommitHandler], which writes the manifest to a temporary path, then -//! renames the temporary path to the final path if no object already exists -//! at the final path. This is an atomic operation in most object stores, but -//! not in AWS S3. So for AWS S3, the default commit handler is -//! [UnsafeCommitHandler], which writes the manifest to the final path without -//! any checks. -//! -//! When providing your own commit handler, most often you are implementing in -//! terms of a lock. The trait [CommitLock] can be implemented as a simpler -//! alternative to [CommitHandler]. - -use std::io; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; -use std::{fmt::Debug, fs::DirEntry}; - -use super::manifest::write_manifest; -use futures::Stream; -use futures::future::Either; -use futures::{ - StreamExt, TryStreamExt, - future::{self, BoxFuture}, - stream::BoxStream, -}; -use lance_file::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION}; -use lance_io::object_writer::{ObjectWriter, WriteResult, get_etag}; -use log::warn; -use object_store::ObjectStoreExt as OSObjectStoreExt; -use object_store::PutOptions; -use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, path::Path}; -use tracing::info; -use url::Url; - -#[cfg(feature = "dynamodb")] -pub mod dynamodb; -pub mod external_manifest; - -use lance_core::{Error, Result}; -use lance_io::object_store::{ObjectStore, ObjectStoreExt, ObjectStoreParams}; -use lance_io::traits::{WriteExt, Writer}; - -use crate::format::{IndexMetadata, Manifest, Transaction, is_detached_version}; -use lance_core::utils::tracing::{AUDIT_MODE_CREATE, AUDIT_TYPE_MANIFEST, TRACE_FILE_AUDIT}; -#[cfg(feature = "dynamodb")] -use { - self::external_manifest::{ExternalManifestCommitHandler, ExternalManifestStore}, - aws_credential_types::provider::ProvideCredentials, - aws_credential_types::provider::error::CredentialsError, - lance_io::object_store::{StorageOptions, providers::aws::build_aws_credential}, - object_store::aws::AmazonS3ConfigKey, - object_store::aws::AwsCredentialProvider, - std::borrow::Cow, - std::time::{Duration, SystemTime}, -}; - -pub const VERSIONS_DIR: &str = "_versions"; -const MANIFEST_EXTENSION: &str = "manifest"; -const DETACHED_VERSION_PREFIX: &str = "d"; -/// File name for the JSON version hint file, stored under `_versions/`. -/// -/// The file contains `{"version":N}` where `N` is the latest committed version -/// at the time of writing. It enables O(1)/O(k) latest-version lookup via HEAD -/// requests on object stores where listing is not lexicographically ordered -/// (e.g. S3 Express, local filesystem) instead of an O(n) listing. -const VERSION_HINT_FILE: &str = "latest_version_hint.json"; - -/// How manifest files should be named. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ManifestNamingScheme { - /// `_versions/{version}.manifest` - V1, - /// `_manifests/{u64::MAX - version}.manifest` - /// - /// Zero-padded and reversed for O(1) lookup of latest version on object stores. - V2, -} - -impl ManifestNamingScheme { - pub fn manifest_path(&self, base: &Path, version: u64) -> Path { - if is_detached_version(version) { - // Detached versions should never show up first in a list operation which - // means it needs to come lexicographically after all attached manifest - // files and so we add the prefix `d`. There is no need to invert the - // version number since detached versions are not part of the version - base.clone().join(VERSIONS_DIR).join(format!( - "{DETACHED_VERSION_PREFIX}{version}.{MANIFEST_EXTENSION}" - )) - } else { - let directory = base.clone().join(VERSIONS_DIR); - match self { - Self::V1 => directory.join(format!("{version}.{MANIFEST_EXTENSION}")), - Self::V2 => { - let inverted_version = u64::MAX - version; - directory.join(format!("{inverted_version:020}.{MANIFEST_EXTENSION}")) - } - } - } - } - - pub fn parse_version(&self, filename: &str) -> Option { - let file_number = filename - .split_once('.') - // Detached versions will fail the `parse` step, which is ok. - .and_then(|(version_str, _)| version_str.parse::().ok()); - match self { - Self::V1 => file_number, - Self::V2 => file_number.map(|v| u64::MAX - v), - } - } - - /// Parse a detached version from a filename like `d123456.manifest`. - /// - /// Returns the full version number with the detached mask bit set. - pub fn parse_detached_version(filename: &str) -> Option { - if !filename.starts_with(DETACHED_VERSION_PREFIX) { - return None; - } - let without_prefix = &filename[DETACHED_VERSION_PREFIX.len()..]; - without_prefix - .split_once('.') - .and_then(|(version_str, _)| version_str.parse::().ok()) - } - - pub fn detect_scheme(filename: &str) -> Option { - if filename.starts_with(DETACHED_VERSION_PREFIX) { - // Currently, detached versions must imply V2 - return Some(Self::V2); - } - if filename.ends_with(MANIFEST_EXTENSION) { - const V2_LEN: usize = 20 + 1 + MANIFEST_EXTENSION.len(); - if filename.len() == V2_LEN { - Some(Self::V2) - } else { - Some(Self::V1) - } - } else { - None - } - } - - pub fn detect_scheme_staging(filename: &str) -> Self { - // We shouldn't have to worry about detached versions here since there is no - // such thing as "detached" and "staged" at the same time. - if filename.chars().nth(20) == Some('.') { - Self::V2 - } else { - Self::V1 - } - } -} - -/// Migrate all V1 manifests to V2 naming scheme. -/// -/// This function will rename all V1 manifests to V2 naming scheme. -/// -/// This function is idempotent, and can be run multiple times without -/// changing the state of the object store. -/// -/// However, it should not be run while other concurrent operations are happening. -/// And it should also run until completion before resuming other operations. -pub async fn migrate_scheme_to_v2(object_store: &ObjectStore, dataset_base: &Path) -> Result<()> { - object_store - .inner - .list(Some(&dataset_base.clone().join(VERSIONS_DIR))) - .try_filter(|res| { - let res = if let Some(filename) = res.location.filename() { - ManifestNamingScheme::detect_scheme(filename) == Some(ManifestNamingScheme::V1) - } else { - false - }; - future::ready(res) - }) - .try_for_each_concurrent(object_store.io_parallelism(), |meta| async move { - let filename = meta.location.filename().unwrap(); - let version = ManifestNamingScheme::V1.parse_version(filename).unwrap(); - let path = ManifestNamingScheme::V2.manifest_path(dataset_base, version); - object_store.inner.rename(&meta.location, &path).await?; - Ok(()) - }) - .await?; - - Ok(()) -} - -/// Function that writes the manifest to the object store. -/// -/// Returns the size of the written manifest. -pub type ManifestWriter = for<'a> fn( - object_store: &'a ObjectStore, - manifest: &'a mut Manifest, - indices: Option>, - path: &'a Path, - transaction: Option, -) -> BoxFuture<'a, Result>; - -/// Canonical manifest writer; its function item type exactly matches `ManifestWriter`. -/// Rationale: keep a crate-local writer implementation so call sites can pass this function -/// directly without non-primitive casts or lifetime coercions. -pub fn write_manifest_file_to_path<'a>( - object_store: &'a ObjectStore, - manifest: &'a mut Manifest, - indices: Option>, - path: &'a Path, - transaction: Option, -) -> BoxFuture<'a, Result> { - Box::pin(async move { - let mut object_writer = ObjectWriter::new(object_store, path).await?; - let pos = write_manifest(&mut object_writer, manifest, indices, transaction).await?; - object_writer - .write_magics(pos, MAJOR_VERSION, MINOR_VERSION, MAGIC) - .await?; - let res = Writer::shutdown(&mut object_writer).await?; - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = path.to_string()); - Ok(res) - }) -} - -#[derive(Debug, Clone)] -pub struct ManifestLocation { - /// The version the manifest corresponds to. - pub version: u64, - /// Path of the manifest file, relative to the table root. - pub path: Path, - /// Size, in bytes, of the manifest file. If it is not known, this field should be `None`. - pub size: Option, - /// Naming scheme of the manifest file. - pub naming_scheme: ManifestNamingScheme, - /// Optional e-tag, used for integrity checks. Manifests should be immutable, so - /// if we detect a change in the e-tag, it means the manifest was tampered with. - /// This might happen if the dataset was deleted and then re-created. - pub e_tag: Option, -} - -impl TryFrom for ManifestLocation { - type Error = Error; - - fn try_from(meta: object_store::ObjectMeta) -> Result { - let filename = meta.location.filename().ok_or_else(|| { - Error::internal("ObjectMeta location does not have a filename".to_string()) - })?; - let scheme = ManifestNamingScheme::detect_scheme(filename) - .ok_or_else(|| Error::internal(format!("Invalid manifest filename: '{}'", filename)))?; - let version = scheme - .parse_version(filename) - .ok_or_else(|| Error::internal(format!("Invalid manifest filename: '{}'", filename)))?; - Ok(Self { - version, - path: meta.location, - size: Some(meta.size), - naming_scheme: scheme, - e_tag: meta.e_tag, - }) - } -} - -/// Get the latest manifest path. -/// -/// - Local filesystem: a single directory read. -/// - Stores where listing is not lexicographically ordered (e.g. S3 Express): -/// the version hint (read the hint file, then probe higher versions with -/// HEADs), falling back to a listing if the hint is missing or stale. A full -/// listing on these stores is O(n) in the number of versions. -/// - Lexicographically ordered stores (e.g. S3 Standard, GCS): the listing -/// already resolves the latest version in roughly one request. -async fn current_manifest_path( - object_store: &ObjectStore, - base: &Path, -) -> Result { - if object_store.is_local() { - if let Ok(Some(location)) = current_manifest_local(base) { - return Ok(location); - } - } else if uses_version_hint(object_store) - && let Some(location) = read_version_hint_and_probe(object_store, base).await - { - return Ok(location); - } - - resolve_version_from_listing(object_store, base).await -} - -/// JSON body of the version hint file: `{"version":N}`. -#[derive(serde::Serialize, serde::Deserialize)] -struct VersionHint { - version: u64, -} - -/// Set `LANCE_USE_VERSION_HINT=0` (or `false`) to globally disable the version -/// hint — writers stop emitting the hint file and readers stop consulting it, -/// falling back to plain listing. Intended as a benchmark/escape-hatch knob; -/// the hint is on by default. -const VERSION_HINT_ENV: &str = "LANCE_USE_VERSION_HINT"; - -fn version_hint_globally_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var(VERSION_HINT_ENV) { - Ok(v) => !matches!( - v.trim().to_ascii_lowercase().as_str(), - "0" | "false" | "off" - ), - Err(_) => true, - }) -} - -/// Whether this object store benefits from a version hint. -/// -/// On stores where listing is lexicographically ordered (S3 Standard, GCS, -/// Azure, ...) the latest version is already resolved in roughly one request, -/// so the hint would only add a write per commit for nothing. We write (and -/// read) it only on stores where listing is not lexicographically ordered — -/// S3 Express and the local filesystem. Can be force-disabled with the -/// `LANCE_USE_VERSION_HINT=0` environment variable. -pub fn uses_version_hint(object_store: &ObjectStore) -> bool { - version_hint_globally_enabled() && !object_store.list_is_lexically_ordered -} - -/// Path to the JSON version hint file for a dataset. -fn version_hint_path(base: &Path) -> Path { - base.clone().join(VERSIONS_DIR).join(VERSION_HINT_FILE) -} - -/// Write the version hint file after a successful commit. -/// -/// The hint is stored as JSON: `{"version":N}`. This write is best-effort — -/// failures are logged and ignored, since the hint only accelerates reads and -/// never affects correctness (readers verify the hinted version and probe -/// upward from there). It is a no-op for detached versions and for stores that -/// do not benefit from a hint (see [`uses_version_hint`]). -pub async fn write_version_hint(object_store: &ObjectStore, base: &Path, version: u64) { - if is_detached_version(version) || !uses_version_hint(object_store) { - return; - } - let hint_path = version_hint_path(base); - let content = serde_json::to_vec(&VersionHint { version }).expect("serialize version hint"); - if let Err(e) = object_store.put(&hint_path, content.as_slice()).await { - warn!("Failed to write version hint file for version {version}: {e}"); - } -} - -/// Read the latest version from the hint file, or `None` if it does not exist -/// or cannot be parsed. -async fn read_version_from_hint(object_store: &ObjectStore, base: &Path) -> Option { - let bytes = object_store - .inner - .get(&version_hint_path(base)) - .await - .ok()? - .bytes() - .await - .ok()?; - Some(serde_json::from_slice::(&bytes).ok()?.version) -} - -/// Read the version hint and probe upward to find the true latest manifest. -/// -/// Returns `None` if the hint file is missing, the hinted version no longer -/// exists, or any error occurred — callers should fall back to listing. -async fn read_version_hint_and_probe( - object_store: &ObjectStore, - base: &Path, -) -> Option { - let hint_version = read_version_from_hint(object_store, base).await?; - let (version, scheme, mut probed) = probe_versions_upward(object_store, base, hint_version) - .await - .ok() - .flatten()?; - // `probed` is non-empty and its last entry is the highest version found. - let (_, meta) = probed.pop()?; - Some(ManifestLocation { - version, - path: scheme.manifest_path(base, version), - size: Some(meta.size), - naming_scheme: scheme, - e_tag: meta.e_tag, - }) -} - -/// Maximum version gap between the hint and the read version for which we use -/// the hint-based parallel-HEAD path; beyond this a single (paginated) listing -/// is cheaper, so callers fall back to it. -const MAX_HINT_PROBE_GAP: u64 = 1000; - -/// Probe `from_version`, then `from_version + 1`, `+ 2`, ... with HEAD requests -/// until one is not found. -/// -/// Assumes attached versions are contiguous above `from_version` (true in -/// practice: every commit increments by one, and cleanup only removes *old* -/// versions, never ones newer than the latest). A `NotFound` therefore marks -/// the end of the history. -/// -/// - `Ok(Some((true_latest_version, naming_scheme, [(version, meta), ...])))`: -/// the vec covers every version from `from_version` through the true latest -/// in ascending order. -/// - `Ok(None)`: `from_version` itself does not exist (a `NotFound` for both -/// naming schemes) — i.e. the hint pointed past the end. -/// - `Err(_)`: a transient object-store error was hit, so the probed range may -/// be incomplete; callers should fall back to a full listing rather than -/// trust a possibly-stale result. -async fn probe_versions_upward( - object_store: &ObjectStore, - base: &Path, - from_version: u64, -) -> Result< - Option<( - u64, - ManifestNamingScheme, - Vec<(u64, object_store::ObjectMeta)>, - )>, -> { - // Newer datasets use V2; fall back to V1 if the V2 path is not found. - let mut scheme = ManifestNamingScheme::V2; - let meta = match object_store - .inner - .head(&scheme.manifest_path(base, from_version)) - .await - { - Ok(meta) => meta, - Err(ObjectStoreError::NotFound { .. }) => { - scheme = ManifestNamingScheme::V1; - match object_store - .inner - .head(&scheme.manifest_path(base, from_version)) - .await - { - Ok(meta) => meta, - Err(ObjectStoreError::NotFound { .. }) => return Ok(None), - Err(e) => return Err(e.into()), - } - } - Err(e) => return Err(e.into()), - }; - - let mut probed = vec![(from_version, meta)]; - let mut version = from_version; - loop { - let next = version + 1; - match object_store - .inner - .head(&scheme.manifest_path(base, next)) - .await - { - Ok(meta) => { - probed.push((next, meta)); - version = next; - } - // NotFound means we found the latest version. - Err(ObjectStoreError::NotFound { .. }) => break, - // A transient error means a newer version might exist that we - // failed to observe — surface it so callers fall back to listing. - Err(e) => return Err(e.into()), - } - } - Ok(Some((version, scheme, probed))) -} - -/// List manifest locations with version `> since_version` using the version -/// hint, in descending order of version. -/// -/// Returns `None` if the hint is missing or stale enough that this is not -/// usable — callers should fall back to a full listing. `Some(vec![])` is the -/// fast path where the hint confirms there are no new versions. -async fn list_manifests_since_version_with_hint( - object_store: &ObjectStore, - base: &Path, - since_version: u64, -) -> Option> { - let hint_version = read_version_from_hint(object_store, base).await?; - - // A reader that is very far behind is cheaper to serve with one paginated - // listing than with thousands of HEADs. - if hint_version.saturating_sub(since_version) > MAX_HINT_PROBE_GAP { - return None; - } - - // If the hint is not newer than the read version, the only versions that - // could exist are right above it; otherwise start at the hint. - let probe_from = if hint_version > since_version { - hint_version - } else { - since_version + 1 - }; - - let (scheme, probed) = match probe_versions_upward(object_store, base, probe_from).await { - Ok(Some((_true_latest, scheme, probed))) => (scheme, probed), - // Nothing at `probe_from`. If we were probing from the hint, the hint - // is stale — bail to a full listing. If we were probing from - // `since_version + 1`, there are simply no new versions. - Ok(None) if hint_version > since_version => return None, - Ok(None) => return Some(Vec::new()), - // Transient error: don't trust the hint path, fall back to listing. - Err(_) => return None, - }; - - let mut locations: Vec = probed - .into_iter() - .filter(|(v, _)| *v > since_version) - .map(|(version, meta)| ManifestLocation { - version, - path: scheme.manifest_path(base, version), - size: Some(meta.size), - naming_scheme: scheme, - e_tag: meta.e_tag, - }) - .collect(); - - // Fill the gap between `since_version` and the hint with HEADs (the probe - // above already covered `hint_version` and up). The range is contiguous, so - // any error here (including a `NotFound`) means we can't trust the hint path - // — fall back to a full listing. - if hint_version > since_version + 1 { - let gap_locations: Vec = - futures::stream::iter((since_version + 1)..hint_version) - .map(|version| async move { - object_store - .inner - .head(&scheme.manifest_path(base, version)) - .await - .map(|meta| ManifestLocation { - version, - path: scheme.manifest_path(base, version), - size: Some(meta.size), - naming_scheme: scheme, - e_tag: meta.e_tag, - }) - }) - .buffer_unordered(object_store.io_parallelism()) - .try_collect() - .await - .ok()?; - locations.extend(gap_locations); - } - - locations.sort_by_key(|loc| std::cmp::Reverse(loc.version)); - Some(locations) -} - -/// Resolve the latest manifest by listing the versions directory. -async fn resolve_version_from_listing( - object_store: &ObjectStore, - base: &Path, -) -> Result { - let manifest_files = object_store.list(Some(base.clone().join(VERSIONS_DIR))); - - let mut valid_manifests = manifest_files.try_filter_map(|res| { - let filename = res.location.filename().unwrap(); - if let Some(scheme) = ManifestNamingScheme::detect_scheme(filename) { - // Only include if we can parse a version (skip detached versions) - if scheme.parse_version(filename).is_some() { - future::ready(Ok(Some((scheme, res)))) - } else { - future::ready(Ok(None)) - } - } else { - future::ready(Ok(None)) - } - }); - - let first = valid_manifests.next().await.transpose()?; - match (first, object_store.list_is_lexically_ordered) { - // If the first valid manifest we see is V2, we can assume that we are using - // V2 naming scheme for all manifests. - (Some((scheme @ ManifestNamingScheme::V2, meta)), true) => { - let version = scheme - .parse_version(meta.location.filename().unwrap()) - .unwrap(); - - // Sanity check: verify at least for the first 1k files that they are all V2 - // and that the version numbers are decreasing. We use the first 1k because - // this is the typical size of an object store list endpoint response page. - for (scheme, meta) in valid_manifests.take(999).try_collect::>().await? { - if scheme != ManifestNamingScheme::V2 { - warn!( - "Found V1 Manifest in a V2 directory. Use `migrate_manifest_paths_v2` \ - to migrate the directory." - ); - break; - } - let next_version = scheme - .parse_version(meta.location.filename().unwrap()) - .unwrap(); - if next_version >= version { - warn!( - "List operation was expected to be lexically ordered, but was not. This \ - could mean a corrupt read. Please make a bug report on the lance-format/lance \ - GitHub repository." - ); - break; - } - } - - Ok(ManifestLocation { - version, - path: meta.location, - size: Some(meta.size), - naming_scheme: scheme, - e_tag: meta.e_tag, - }) - } - // If the list is not lexically ordered, we need to iterate all manifests - // to find the latest version. This works for both V1 and V2 schemes. - (Some((first_scheme, meta)), _) => { - let mut current_version = first_scheme - .parse_version(meta.location.filename().unwrap()) - .unwrap(); - let mut current_meta = meta; - let scheme = first_scheme; - - while let Some((entry_scheme, meta)) = valid_manifests.next().await.transpose()? { - if entry_scheme != scheme { - return Err(Error::internal(format!( - "Found multiple manifest naming schemes in the same directory: {:?} and {:?}. \ - Use `migrate_manifest_paths_v2` to migrate the directory.", - scheme, entry_scheme - ))); - } - let version = entry_scheme - .parse_version(meta.location.filename().unwrap()) - .unwrap(); - if version > current_version { - current_version = version; - current_meta = meta; - } - } - Ok(ManifestLocation { - version: current_version, - path: current_meta.location, - size: Some(current_meta.size), - naming_scheme: scheme, - e_tag: current_meta.e_tag, - }) - } - (None, _) => Err(Error::not_found( - base.clone().join(VERSIONS_DIR).to_string(), - )), - } -} - -// This is an optimized function that searches for the latest manifest. In -// object_store, list operations lookup metadata for each file listed. This -// method only gets the metadata for the found latest manifest. -fn current_manifest_local(base: &Path) -> std::io::Result> { - let path = lance_io::local::to_local_path(&base.clone().join(VERSIONS_DIR)); - let entries = std::fs::read_dir(path)?; - - let mut latest_entry: Option<(u64, DirEntry)> = None; - - let mut scheme: Option = None; - - for entry in entries { - let entry = entry?; - let filename_raw = entry.file_name(); - let filename = filename_raw.to_string_lossy(); - - let Some(entry_scheme) = ManifestNamingScheme::detect_scheme(&filename) else { - // Need to ignore temporary files, such as - // .tmp_7.manifest_9c100374-3298-4537-afc6-f5ee7913666d - continue; - }; - - if let Some(scheme) = scheme { - if scheme != entry_scheme { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "Found multiple manifest naming schemes in the same directory: {:?} and {:?}", - scheme, entry_scheme - ), - )); - } - } else { - scheme = Some(entry_scheme); - } - - let Some(version) = entry_scheme.parse_version(&filename) else { - continue; - }; - - if let Some((latest_version, _)) = &latest_entry { - if version > *latest_version { - latest_entry = Some((version, entry)); - } - } else { - latest_entry = Some((version, entry)); - } - } - - if let Some((version, entry)) = latest_entry { - let path = Path::from_filesystem_path(entry.path()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; - let metadata = entry.metadata()?; - Ok(Some(ManifestLocation { - version, - path, - size: Some(metadata.len()), - naming_scheme: scheme.unwrap(), - e_tag: Some(get_etag(&metadata)), - })) - } else { - Ok(None) - } -} - -fn list_manifests<'a>( - base_path: &Path, - object_store: &'a dyn OSObjectStore, -) -> impl Stream> + 'a { - object_store - .read_dir_all(&base_path.clone().join(VERSIONS_DIR), None) - .filter_map(|obj_meta| { - futures::future::ready( - obj_meta - .map(|m| ManifestLocation::try_from(m).ok()) - .transpose(), - ) - }) - .boxed() -} - -/// Convert object metadata to ManifestLocation for detached manifests. -fn detached_manifest_location_from_meta( - meta: object_store::ObjectMeta, -) -> Option { - let filename = meta.location.filename()?; - let version = ManifestNamingScheme::parse_detached_version(filename)?; - Some(ManifestLocation { - version, - path: meta.location, - size: Some(meta.size), - naming_scheme: ManifestNamingScheme::V2, - e_tag: meta.e_tag, - }) -} - -/// List all detached manifest files in the versions directory. -pub fn list_detached_manifests<'a>( - base_path: &Path, - object_store: &'a dyn OSObjectStore, -) -> impl Stream> + 'a { - object_store - .read_dir_all(&base_path.clone().join(VERSIONS_DIR), None) - .filter_map(|obj_meta| { - futures::future::ready( - obj_meta - .map(detached_manifest_location_from_meta) - .transpose(), - ) - }) - .boxed() -} - -fn make_staging_manifest_path(base: &Path) -> Result { - let id = uuid::Uuid::new_v4().to_string(); - Path::parse(format!("{base}-{id}")).map_err(|e| Error::io_source(Box::new(e))) -} - -#[cfg(feature = "dynamodb")] -const DDB_URL_QUERY_KEY: &str = "ddbTableName"; - -/// Handle commits that prevent conflicting writes. -/// -/// Commit implementations ensure that if there are multiple concurrent writers -/// attempting to write the next version of a table, only one will win. In order -/// to work, all writers must use the same commit handler type. -/// This trait is also responsible for resolving where the manifests live. -/// -// TODO: pub(crate) -#[async_trait::async_trait] -#[allow(clippy::too_many_arguments)] -pub trait CommitHandler: Debug + Send + Sync { - async fn resolve_latest_location( - &self, - base_path: &Path, - object_store: &ObjectStore, - ) -> Result { - Ok(current_manifest_path(object_store, base_path).await?) - } - - async fn resolve_version_location( - &self, - base_path: &Path, - version: u64, - object_store: &dyn OSObjectStore, - ) -> Result { - default_resolve_version(base_path, version, object_store).await - } - - /// List detached manifest locations. - /// - /// Returns a stream of detached manifest locations in arbitrary order. - fn list_detached_manifest_locations<'a>( - &self, - base_path: &Path, - object_store: &'a ObjectStore, - ) -> BoxStream<'a, Result> { - list_detached_manifests(base_path, &object_store.inner).boxed() - } - - /// If `sorted_descending` is `true`, the stream will yield manifests in descending - /// order of version. When the object store has a lexicographically - /// ordered list and the naming scheme is V2, this will use an optimized - /// list operation. Otherwise, it will list all manifests and sort them - /// in memory. When `sorted_descending` is `false`, the stream will yield manifests - /// in arbitrary order. - fn list_manifest_locations<'a>( - &self, - base_path: &Path, - object_store: &'a ObjectStore, - sorted_descending: bool, - ) -> BoxStream<'a, Result> { - let underlying_stream = list_manifests(base_path, &object_store.inner); - - if !sorted_descending { - return underlying_stream.boxed(); - } - - async fn sort_stream( - input_stream: impl futures::Stream> + Unpin, - ) -> Result> + Unpin> { - let mut locations = input_stream.try_collect::>().await?; - locations.sort_by_key(|m| std::cmp::Reverse(m.version)); - Ok(futures::stream::iter(locations.into_iter().map(Ok))) - } - - // If the object store supports lexicographically ordered lists and - // the naming scheme is V2, we can use an optimized list operation. - if object_store.list_is_lexically_ordered { - // We don't know the naming scheme until we see the first manifest. - let mut peekable = underlying_stream.peekable(); - - futures::stream::once(async move { - let naming_scheme = match Pin::new(&mut peekable).peek().await { - Some(Ok(m)) => m.naming_scheme, - // If we get an error or no manifests are found, we default - // to V2 naming scheme, since it doesn't matter. - Some(Err(_)) => ManifestNamingScheme::V2, - None => ManifestNamingScheme::V2, - }; - - if naming_scheme == ManifestNamingScheme::V2 { - // If the first manifest is V2, we can use the optimized list operation. - Ok(Either::Left(peekable)) - } else { - sort_stream(peekable).await.map(Either::Right) - } - }) - .try_flatten() - .boxed() - } else { - // If the object store does not support lexicographically ordered lists, - // we need to sort the manifests in memory. Systems where this isn't - // supported (local fs, S3 express) are typically fast enough - // that this is not a problem. - futures::stream::once(sort_stream(underlying_stream)) - .try_flatten() - .boxed() - } - } - - /// List manifest locations with version `> since_version`, in descending - /// order of version. - /// - /// On lexically-ordered stores this is the standard listing with early - /// termination. On non-lexically-ordered stores (e.g. S3 Express) it uses - /// the version hint to avoid an O(n) listing, falling back to a full - /// listing if the hint is missing or stale. - fn list_manifest_locations_since<'a>( - &self, - base_path: &Path, - object_store: &'a ObjectStore, - since_version: u64, - ) -> BoxStream<'a, Result> { - if !uses_version_hint(object_store) { - return self - .list_manifest_locations(base_path, object_store, true) - .try_take_while(move |loc| future::ready(Ok(loc.version > since_version))) - .boxed(); - } - - let base_path = base_path.clone(); - futures::stream::once(async move { - let locations = match list_manifests_since_version_with_hint( - object_store, - &base_path, - since_version, - ) - .await - { - Some(locations) => locations, - None => { - let mut locations = list_manifests(&base_path, &object_store.inner) - .try_collect::>() - .await?; - locations.retain(|loc| loc.version > since_version); - locations.sort_by_key(|loc| std::cmp::Reverse(loc.version)); - locations - } - }; - Ok::<_, Error>(futures::stream::iter(locations.into_iter().map(Ok))) - }) - .try_flatten() - .boxed() - } - - /// Commit a manifest. - /// - /// This function should return an [CommitError::CommitConflict] if another - /// transaction has already been committed to the path. - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result; - - /// Delete the recorded manifest information for a dataset at the base_path - async fn delete(&self, _base_path: &Path) -> Result<()> { - Ok(()) - } -} - -async fn default_resolve_version( - base_path: &Path, - version: u64, - object_store: &dyn OSObjectStore, -) -> Result { - if is_detached_version(version) { - return Ok(ManifestLocation { - version, - // Detached versions are not supported with V1 naming scheme. If we need - // to support in the future we could use a different prefix (e.g. 'x' or something) - naming_scheme: ManifestNamingScheme::V2, - // Both V1 and V2 should give the same path for detached versions - path: ManifestNamingScheme::V2.manifest_path(base_path, version), - size: None, - e_tag: None, - }); - } - - // try V2, fallback to V1. - let scheme = ManifestNamingScheme::V2; - let path = scheme.manifest_path(base_path, version); - match object_store.head(&path).await { - Ok(meta) => Ok(ManifestLocation { - version, - path, - size: Some(meta.size), - naming_scheme: scheme, - e_tag: meta.e_tag, - }), - Err(ObjectStoreError::NotFound { .. }) => { - // fallback to V1 - let scheme = ManifestNamingScheme::V1; - Ok(ManifestLocation { - version, - path: scheme.manifest_path(base_path, version), - size: None, - naming_scheme: scheme, - e_tag: None, - }) - } - Err(e) => Err(e.into()), - } -} -/// Adapt an object_store credentials into AWS SDK creds -#[cfg(feature = "dynamodb")] -#[derive(Debug)] -struct OSObjectStoreToAwsCredAdaptor(AwsCredentialProvider); - -#[cfg(feature = "dynamodb")] -impl ProvideCredentials for OSObjectStoreToAwsCredAdaptor { - fn provide_credentials<'a>( - &'a self, - ) -> aws_credential_types::provider::future::ProvideCredentials<'a> - where - Self: 'a, - { - aws_credential_types::provider::future::ProvideCredentials::new(async { - let creds = self - .0 - .get_credential() - .await - .map_err(|e| CredentialsError::provider_error(Box::new(e)))?; - Ok(aws_credential_types::Credentials::new( - &creds.key_id, - &creds.secret_key, - creds.token.clone(), - Some( - SystemTime::now() - .checked_add(Duration::from_secs( - 60 * 10, // 10 min - )) - .expect("overflow"), - ), - "", - )) - }) - } -} - -#[cfg(feature = "dynamodb")] -async fn build_dynamodb_external_store( - table_name: &str, - creds: AwsCredentialProvider, - region: &str, - endpoint: Option, - app_name: &str, -) -> Result> { - use super::commit::dynamodb::DynamoDBExternalManifestStore; - use aws_sdk_dynamodb::{ - Client, - config::{IdentityCache, Region, retry::RetryConfig}, - }; - - let mut dynamodb_config = aws_sdk_dynamodb::config::Builder::new() - .behavior_version_latest() - .region(Some(Region::new(region.to_string()))) - .credentials_provider(OSObjectStoreToAwsCredAdaptor(creds)) - // caching should be handled by passed AwsCredentialProvider - .identity_cache(IdentityCache::no_cache()) - // Be more resilient to transient network issues. - // 5 attempts = 1 initial + 4 retries with exponential backoff. - .retry_config(RetryConfig::standard().with_max_attempts(5)); - - if let Some(endpoint) = endpoint { - dynamodb_config = dynamodb_config.endpoint_url(endpoint); - } - let client = Client::from_conf(dynamodb_config.build()); - - DynamoDBExternalManifestStore::new_external_store(client.into(), table_name, app_name).await -} - -pub async fn commit_handler_from_url( - url_or_path: &str, - // This looks unused if dynamodb feature disabled - #[allow(unused_variables)] options: &Option, -) -> Result> { - let local_handler: Arc = if cfg!(windows) { - Arc::new(RenameCommitHandler) - } else { - Arc::new(ConditionalPutCommitHandler) - }; - - let url = match Url::parse(url_or_path) { - Ok(url) if url.scheme().len() == 1 && cfg!(windows) => { - // On Windows, the drive is parsed as a scheme - return Ok(local_handler); - } - Ok(url) => url, - Err(_) => { - return Ok(local_handler); - } - }; - - match url.scheme() { - "file" | "file-object-store" => Ok(local_handler), - "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "shared-memory" => { - Ok(Arc::new(ConditionalPutCommitHandler)) - } - #[cfg(not(feature = "dynamodb"))] - "s3+ddb" => Err(Error::invalid_input_source( - "`s3+ddb://` scheme requires `dynamodb` feature to be enabled".into(), - )), - #[cfg(feature = "dynamodb")] - "s3+ddb" => { - if url.query_pairs().count() != 1 { - return Err(Error::invalid_input_source( - "`s3+ddb://` scheme and expects exactly one query `ddbTableName`".into(), - )); - } - let table_name = match url.query_pairs().next() { - Some((Cow::Borrowed(key), Cow::Borrowed(table_name))) - if key == DDB_URL_QUERY_KEY => - { - if table_name.is_empty() { - return Err(Error::invalid_input_source( - "`s3+ddb://` scheme requires non empty dynamodb table name".into(), - )); - } - table_name - } - _ => { - return Err(Error::invalid_input_source( - "`s3+ddb://` scheme and expects exactly one query `ddbTableName`".into(), - )); - } - }; - let options = options.clone().unwrap_or_default(); - let storage_options_raw = - StorageOptions(options.storage_options().cloned().unwrap_or_default()); - let dynamo_endpoint = get_dynamodb_endpoint(&storage_options_raw); - let storage_options = storage_options_raw.as_s3_options(); - - let region = storage_options.get(&AmazonS3ConfigKey::Region).cloned(); - - // Get accessor from the options - let accessor = options.get_accessor(); - - let (aws_creds, region) = build_aws_credential( - options.s3_credentials_refresh_offset, - options.aws_credentials.clone(), - Some(&storage_options), - region, - accessor, - ) - .await?; - - Ok(Arc::new(ExternalManifestCommitHandler { - external_manifest_store: build_dynamodb_external_store( - table_name, - aws_creds.clone(), - ®ion, - dynamo_endpoint, - "lancedb", - ) - .await?, - })) - } - _ => Ok(Arc::new(UnsafeCommitHandler)), - } -} - -#[cfg(feature = "dynamodb")] -fn get_dynamodb_endpoint(storage_options: &StorageOptions) -> Option { - if let Some(endpoint) = storage_options.0.get("dynamodb_endpoint") { - Some(endpoint.clone()) - } else { - std::env::var("DYNAMODB_ENDPOINT").ok() - } -} - -/// Errors that can occur when committing a manifest. -#[derive(Debug)] -pub enum CommitError { - /// Another transaction has already been written to the path - CommitConflict, - /// Something else went wrong - OtherError(Error), -} - -impl From for CommitError { - fn from(e: Error) -> Self { - Self::OtherError(e) - } -} - -impl From for Error { - fn from(e: CommitError) -> Self { - match e { - CommitError::CommitConflict => Self::internal("Commit conflict".to_string()), - CommitError::OtherError(e) => e, - } - } -} - -/// Whether we have issued a warning about using the unsafe commit handler. -static WARNED_ON_UNSAFE_COMMIT: AtomicBool = AtomicBool::new(false); - -/// A naive commit implementation that does not prevent conflicting writes. -/// -/// This will log a warning the first time it is used. -pub struct UnsafeCommitHandler; - -#[async_trait::async_trait] -#[allow(clippy::too_many_arguments)] -impl CommitHandler for UnsafeCommitHandler { - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result { - // Log a one-time warning - if !WARNED_ON_UNSAFE_COMMIT.load(std::sync::atomic::Ordering::Relaxed) { - WARNED_ON_UNSAFE_COMMIT.store(true, std::sync::atomic::Ordering::Relaxed); - log::warn!( - "Using unsafe commit handler. Concurrent writes may result in data loss. \ - Consider providing a commit handler that prevents conflicting writes." - ); - } - - let version_path = naming_scheme.manifest_path(base_path, manifest.version); - let res = - manifest_writer(object_store, manifest, indices, &version_path, transaction).await?; - - write_version_hint(object_store, base_path, manifest.version).await; - - Ok(ManifestLocation { - version: manifest.version, - size: Some(res.size as u64), - naming_scheme, - path: version_path, - e_tag: res.e_tag, - }) - } -} - -impl Debug for UnsafeCommitHandler { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("UnsafeCommitHandler").finish() - } -} - -/// A commit implementation that uses a lock to prevent conflicting writes. -#[async_trait::async_trait] -pub trait CommitLock: Debug { - type Lease: CommitLease; - - /// Attempt to lock the table for the given version. - /// - /// If it is already locked by another transaction, wait until it is unlocked. - /// Once it is unlocked, return [CommitError::CommitConflict] if the version - /// has already been committed. Otherwise, return the lock. - /// - /// To prevent poisoned locks, it's recommended to set a timeout on the lock - /// of at least 30 seconds. - /// - /// It is not required that the lock tracks the version. It is provided in - /// case the locking is handled by a catalog service that needs to know the - /// current version of the table. - async fn lock(&self, version: u64) -> std::result::Result; -} - -#[async_trait::async_trait] -pub trait CommitLease: Send + Sync { - /// Return the lease, indicating whether the commit was successful. - async fn release(&self, success: bool) -> std::result::Result<(), CommitError>; -} - -#[async_trait::async_trait] -impl CommitHandler for T { - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result { - let path = naming_scheme.manifest_path(base_path, manifest.version); - // NOTE: once we have the lease we cannot use ? to return errors, since - // we must release the lease before returning. - let lease = self.lock(manifest.version).await?; - - // Head the location and make sure it's not already committed - match object_store.inner.head(&path).await { - Ok(_) => { - // The path already exists, so it's already committed - // Release the lock - lease.release(false).await?; - - return Err(CommitError::CommitConflict); - } - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => { - // Something else went wrong - // Release the lock - lease.release(false).await?; - - return Err(CommitError::OtherError(e.into())); - } - } - let res = manifest_writer(object_store, manifest, indices, &path, transaction).await; - - // Release the lock - lease.release(res.is_ok()).await?; - - let res = res?; - - write_version_hint(object_store, base_path, manifest.version).await; - - Ok(ManifestLocation { - version: manifest.version, - size: Some(res.size as u64), - naming_scheme, - path, - e_tag: res.e_tag, - }) - } -} - -#[async_trait::async_trait] -impl CommitHandler for Arc { - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result { - self.as_ref() - .commit( - manifest, - indices, - base_path, - object_store, - manifest_writer, - naming_scheme, - transaction, - ) - .await - } -} - -/// A commit implementation that uses a temporary path and renames the object. -/// -/// This only works for object stores that support atomic rename if not exist. -pub struct RenameCommitHandler; - -#[async_trait::async_trait] -impl CommitHandler for RenameCommitHandler { - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result { - // Create a temporary object, then use `rename_if_not_exists` to commit. - // If failed, clean up the temporary object. - - let path = naming_scheme.manifest_path(base_path, manifest.version); - let tmp_path = make_staging_manifest_path(&path)?; - - let res = manifest_writer(object_store, manifest, indices, &tmp_path, transaction).await?; - - match object_store - .inner - .rename_if_not_exists(&tmp_path, &path) - .await - { - Ok(_) => { - // Successfully committed - write_version_hint(object_store, base_path, manifest.version).await; - Ok(ManifestLocation { - version: manifest.version, - path, - size: Some(res.size as u64), - naming_scheme, - e_tag: None, // Re-name can change e-tag. - }) - } - Err(ObjectStoreError::AlreadyExists { .. }) => { - // Another transaction has already been committed - // Attempt to clean up temporary object, but ignore errors if we can't - let _ = object_store.delete(&tmp_path).await; - - return Err(CommitError::CommitConflict); - } - Err(e) => { - // Something else went wrong - return Err(CommitError::OtherError(e.into())); - } - } - } -} - -impl Debug for RenameCommitHandler { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RenameCommitHandler").finish() - } -} - -pub struct ConditionalPutCommitHandler; - -#[async_trait::async_trait] -impl CommitHandler for ConditionalPutCommitHandler { - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result { - let path = naming_scheme.manifest_path(base_path, manifest.version); - - let memory_store = ObjectStore::memory(); - let dummy_path = "dummy"; - manifest_writer( - &memory_store, - manifest, - indices, - &dummy_path.into(), - transaction, - ) - .await?; - let dummy_data = memory_store.read_one_all(&dummy_path.into()).await?; - let size = dummy_data.len() as u64; - let res = object_store - .inner - .put_opts( - &path, - dummy_data.into(), - PutOptions { - mode: object_store::PutMode::Create, - ..Default::default() - }, - ) - .await - .map_err(|err| match err { - ObjectStoreError::AlreadyExists { .. } | ObjectStoreError::Precondition { .. } => { - CommitError::CommitConflict - } - _ => CommitError::OtherError(err.into()), - })?; - - write_version_hint(object_store, base_path, manifest.version).await; - - Ok(ManifestLocation { - version: manifest.version, - path, - size: Some(size), - naming_scheme, - e_tag: res.e_tag, - }) - } -} - -impl Debug for ConditionalPutCommitHandler { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ConditionalPutCommitHandler").finish() - } -} - -#[derive(Debug, Clone)] -pub struct CommitConfig { - pub num_retries: u32, - pub skip_auto_cleanup: bool, - // TODO: add isolation_level -} - -impl Default for CommitConfig { - fn default() -> Self { - Self { - num_retries: 20, - skip_auto_cleanup: false, - } - } -} - -#[cfg(test)] -mod tests { - use lance_core::utils::tempfile::TempObjDir; - - use super::*; - - #[test] - fn test_manifest_naming_scheme() { - let v1 = ManifestNamingScheme::V1; - let v2 = ManifestNamingScheme::V2; - - assert_eq!( - v1.manifest_path(&Path::from("base"), 0), - Path::from("base/_versions/0.manifest") - ); - assert_eq!( - v1.manifest_path(&Path::from("base"), 42), - Path::from("base/_versions/42.manifest") - ); - - assert_eq!( - v2.manifest_path(&Path::from("base"), 0), - Path::from("base/_versions/18446744073709551615.manifest") - ); - assert_eq!( - v2.manifest_path(&Path::from("base"), 42), - Path::from("base/_versions/18446744073709551573.manifest") - ); - - assert_eq!(v1.parse_version("0.manifest"), Some(0)); - assert_eq!(v1.parse_version("42.manifest"), Some(42)); - assert_eq!( - v1.parse_version("42.manifest-cee4fbbb-eb19-4ea3-8ca7-54f5ec33dedc"), - Some(42) - ); - - assert_eq!(v2.parse_version("18446744073709551615.manifest"), Some(0)); - assert_eq!(v2.parse_version("18446744073709551573.manifest"), Some(42)); - assert_eq!( - v2.parse_version("18446744073709551573.manifest-cee4fbbb-eb19-4ea3-8ca7-54f5ec33dedc"), - Some(42) - ); - - assert_eq!(ManifestNamingScheme::detect_scheme("0.manifest"), Some(v1)); - assert_eq!( - ManifestNamingScheme::detect_scheme("18446744073709551615.manifest"), - Some(v2) - ); - assert_eq!(ManifestNamingScheme::detect_scheme("something else"), None); - } - - #[tokio::test] - async fn test_manifest_naming_migration() { - let object_store = ObjectStore::memory(); - let base = Path::from("base"); - let versions_dir = base.clone().join(VERSIONS_DIR); - - // Write two v1 files and one v1 - let original_files = vec![ - versions_dir.clone().join("irrelevant"), - ManifestNamingScheme::V1.manifest_path(&base, 0), - ManifestNamingScheme::V2.manifest_path(&base, 1), - ]; - for path in original_files { - object_store.put(&path, b"".as_slice()).await.unwrap(); - } - - migrate_scheme_to_v2(&object_store, &base).await.unwrap(); - - let expected_files = vec![ - ManifestNamingScheme::V2.manifest_path(&base, 1), - ManifestNamingScheme::V2.manifest_path(&base, 0), - versions_dir.clone().join("irrelevant"), - ]; - let actual_files = object_store - .inner - .list(Some(&versions_dir)) - .map_ok(|res| res.location) - .try_collect::>() - .await - .unwrap(); - assert_eq!(actual_files, expected_files); - } - - #[tokio::test] - #[rstest::rstest] - async fn test_list_manifests_sorted( - #[values(true, false)] lexical_list_store: bool, - #[values(ManifestNamingScheme::V1, ManifestNamingScheme::V2)] - naming_scheme: ManifestNamingScheme, - ) { - let tempdir; - let (object_store, base) = if lexical_list_store { - (Box::new(ObjectStore::memory()), Path::from("base")) - } else { - tempdir = TempObjDir::default(); - let path = tempdir.clone().join("base"); - let store = Box::new(ObjectStore::local()); - assert!(!store.list_is_lexically_ordered); - (store, path) - }; - - // Write 12 manifest files, latest first - let mut expected_paths = Vec::new(); - for i in (0..12).rev() { - let path = naming_scheme.manifest_path(&base, i); - object_store.put(&path, b"".as_slice()).await.unwrap(); - expected_paths.push(path); - } - - let actual_versions = ConditionalPutCommitHandler - .list_manifest_locations(&base, &object_store, true) - .map_ok(|location| location.path) - .try_collect::>() - .await - .unwrap(); - - assert_eq!(actual_versions, expected_paths); - } - - #[tokio::test] - #[rstest::rstest] - async fn test_current_manifest_path( - #[values(true, false)] lexical_list_store: bool, - #[values(ManifestNamingScheme::V1, ManifestNamingScheme::V2)] - naming_scheme: ManifestNamingScheme, - ) { - // Use memory store for both cases to avoid local FS special codepath. - // Modify list_is_lexically_ordered to simulate different object stores. - let mut object_store = ObjectStore::memory(); - object_store.list_is_lexically_ordered = lexical_list_store; - let object_store = Box::new(object_store); - let base = Path::from("base"); - - // Write 12 manifest files in non-sequential order - for version in [5, 2, 11, 0, 8, 3, 10, 1, 7, 4, 9, 6] { - let path = naming_scheme.manifest_path(&base, version); - object_store.put(&path, b"".as_slice()).await.unwrap(); - } - - let location = current_manifest_path(&object_store, &base).await.unwrap(); - - assert_eq!(location.version, 11); - assert_eq!(location.naming_scheme, naming_scheme); - assert_eq!(location.path, naming_scheme.manifest_path(&base, 11)); - } - - /// A memory store that reports `list_is_lexically_ordered == false`, like - /// S3 Express, so the version-hint paths are exercised. - fn non_lexical_memory_store() -> Box { - let mut object_store = ObjectStore::memory(); - object_store.list_is_lexically_ordered = false; - Box::new(object_store) - } - - #[tokio::test] - async fn test_write_version_hint() { - let base = Path::from("base"); - - // No hint is written on lexically-ordered stores (it would not be read). - let lexical = ObjectStore::memory(); - write_version_hint(&lexical, &base, 42).await; - assert_eq!(read_version_from_hint(&lexical, &base).await, None); - - let object_store = non_lexical_memory_store(); - write_version_hint(&object_store, &base, 42).await; - assert_eq!(read_version_from_hint(&object_store, &base).await, Some(42)); - - // A later commit overwrites the hint. - write_version_hint(&object_store, &base, 100).await; - assert_eq!( - read_version_from_hint(&object_store, &base).await, - Some(100) - ); - - // Detached versions are never written to the hint. - write_version_hint( - &object_store, - &base, - crate::format::DETACHED_VERSION_MASK | 7, - ) - .await; - assert_eq!( - read_version_from_hint(&object_store, &base).await, - Some(100) - ); - - // A corrupt / non-JSON hint file is treated as missing. - let hint_path = version_hint_path(&base); - object_store - .put(&hint_path, b"not json".as_slice()) - .await - .unwrap(); - assert_eq!(read_version_from_hint(&object_store, &base).await, None); - } - - #[tokio::test] - #[rstest::rstest] - async fn test_read_version_hint_and_probe( - #[values(ManifestNamingScheme::V1, ManifestNamingScheme::V2)] - naming_scheme: ManifestNamingScheme, - ) { - let object_store = non_lexical_memory_store(); - let base = Path::from("base"); - - // No hint file yet. - assert!( - read_version_hint_and_probe(&object_store, &base) - .await - .is_none() - ); - - for version in 1..=5 { - object_store - .put(&naming_scheme.manifest_path(&base, version), b"".as_slice()) - .await - .unwrap(); - } - - // Stale hint: should probe forward and find version 5. - write_version_hint(&object_store, &base, 3).await; - let location = read_version_hint_and_probe(&object_store, &base) - .await - .unwrap(); - assert_eq!(location.version, 5); - assert_eq!(location.naming_scheme, naming_scheme); - - // Up-to-date hint: returns version 5 directly. - write_version_hint(&object_store, &base, 5).await; - let location = read_version_hint_and_probe(&object_store, &base) - .await - .unwrap(); - assert_eq!(location.version, 5); - - // Hint points past the latest version: not usable. - write_version_hint(&object_store, &base, 10).await; - assert!( - read_version_hint_and_probe(&object_store, &base) - .await - .is_none() - ); - } - - #[tokio::test] - async fn test_list_manifests_since_version_with_hint() { - let object_store = non_lexical_memory_store(); - let base = Path::from("base"); - let scheme = ManifestNamingScheme::V2; - - for version in 1..=10 { - object_store - .put(&scheme.manifest_path(&base, version), b"".as_slice()) - .await - .unwrap(); - } - - // No hint yet -> not usable, caller must fall back. - assert!( - list_manifests_since_version_with_hint(&object_store, &base, 7) - .await - .is_none() - ); - - // Hint exactly at the read version -> fast path, nothing new. - write_version_hint(&object_store, &base, 10).await; - assert!(matches!( - list_manifests_since_version_with_hint(&object_store, &base, 10).await, - Some(v) if v.is_empty() - )); - - // Hint ahead of the read version, with a gap to fill (8, 9) plus probing - // from the hint (10). Results are descending by version. - let locations = list_manifests_since_version_with_hint(&object_store, &base, 7) - .await - .unwrap(); - assert_eq!( - locations.iter().map(|l| l.version).collect::>(), - vec![10, 9, 8] - ); - - // Slightly stale hint (points at 8) still probes up to the true latest. - write_version_hint(&object_store, &base, 8).await; - let locations = list_manifests_since_version_with_hint(&object_store, &base, 7) - .await - .unwrap(); - assert_eq!( - locations.iter().map(|l| l.version).collect::>(), - vec![10, 9, 8] - ); - - // Hint points past the latest -> not usable, caller falls back. - write_version_hint(&object_store, &base, 20).await; - assert!( - list_manifests_since_version_with_hint(&object_store, &base, 7) - .await - .is_none() - ); - } - - #[tokio::test] - async fn test_current_manifest_path_with_hint_non_lexical() { - // Simulate S3 Express (non-lexically ordered list) with many versions. - let object_store = non_lexical_memory_store(); - let base = Path::from("base"); - let naming_scheme = ManifestNamingScheme::V2; - - for version in 1..=100 { - object_store - .put(&naming_scheme.manifest_path(&base, version), b"".as_slice()) - .await - .unwrap(); - } - - // Slightly stale hint: probing from 98 still resolves the true latest. - write_version_hint(&object_store, &base, 98).await; - let location = current_manifest_path(&object_store, &base).await.unwrap(); - assert_eq!(location.version, 100); - } - - #[tokio::test] - async fn test_current_manifest_path_with_stale_hint_falls_back_to_listing() { - let object_store = non_lexical_memory_store(); - let base = Path::from("base"); - let naming_scheme = ManifestNamingScheme::V2; - - // Only version 5 exists, but the hint claims version 10. - object_store - .put(&naming_scheme.manifest_path(&base, 5), b"".as_slice()) - .await - .unwrap(); - write_version_hint(&object_store, &base, 10).await; - - // The stale hint is ignored; listing finds version 5. - let location = current_manifest_path(&object_store, &base).await.unwrap(); - assert_eq!(location.version, 5); - } - - #[test] - fn test_parse_detached_version() { - // Valid detached version filenames - assert_eq!( - ManifestNamingScheme::parse_detached_version("d12345.manifest"), - Some(12345) - ); - assert_eq!( - ManifestNamingScheme::parse_detached_version("d9223372036854775808.manifest"), - Some(9223372036854775808) - ); - - // Invalid: not starting with 'd' prefix - assert_eq!( - ManifestNamingScheme::parse_detached_version("12345.manifest"), - None - ); - - // Invalid: regular V2 manifest - assert_eq!( - ManifestNamingScheme::parse_detached_version("18446744073709551615.manifest"), - None - ); - - // Invalid: no extension - assert_eq!(ManifestNamingScheme::parse_detached_version("d12345"), None); - } - - #[tokio::test] - async fn test_list_detached_manifests() { - use crate::format::DETACHED_VERSION_MASK; - use futures::TryStreamExt; - - let object_store = ObjectStore::memory(); - let base = Path::from("base"); - let versions_dir = base.clone().join(VERSIONS_DIR); - - // Create some regular manifests - for version in [1, 2, 3] { - let path = ManifestNamingScheme::V2.manifest_path(&base, version); - object_store.put(&path, b"".as_slice()).await.unwrap(); - } - - // Create some detached manifests - let detached_versions: Vec = vec![ - 100 | DETACHED_VERSION_MASK, - 200 | DETACHED_VERSION_MASK, - 300 | DETACHED_VERSION_MASK, - ]; - for version in &detached_versions { - let path = versions_dir.clone().join(format!("d{}.manifest", version)); - object_store.put(&path, b"".as_slice()).await.unwrap(); - } - - // List detached manifests - let detached_locations: Vec = - list_detached_manifests(&base, &object_store.inner) - .try_collect() - .await - .unwrap(); - - assert_eq!(detached_locations.len(), 3); - for loc in &detached_locations { - assert_eq!(loc.naming_scheme, ManifestNamingScheme::V2); - } - - let mut found_versions: Vec = detached_locations.iter().map(|l| l.version).collect(); - found_versions.sort(); - let mut expected_versions = detached_versions.clone(); - expected_versions.sort(); - assert_eq!(found_versions, expected_versions); - } - - #[tokio::test] - async fn test_commit_handler_from_url_memory_schemes() { - // Both `memory://` and `shared-memory://` must route to - // ConditionalPutCommitHandler — otherwise concurrent writers fall - // through to UnsafeCommitHandler and silently clobber each other's - // manifests. - for url in ["memory://bucket-a/ds", "shared-memory://bucket-a/ds"] { - let handler = commit_handler_from_url(url, &None).await.unwrap(); - assert_eq!( - format!("{:?}", handler), - "ConditionalPutCommitHandler", - "{url} should route to ConditionalPutCommitHandler", - ); - } - } -} diff --git a/vendor/lance-table/src/io/commit/dynamodb.rs b/vendor/lance-table/src/io/commit/dynamodb.rs deleted file mode 100644 index d4dab02f..00000000 --- a/vendor/lance-table/src/io/commit/dynamodb.rs +++ /dev/null @@ -1,495 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! DynamoDB based external manifest store -//! - -use std::collections::HashSet; -use std::sync::{Arc, LazyLock}; - -use async_trait::async_trait; -use aws_sdk_dynamodb::Client; -use aws_sdk_dynamodb::error::SdkError; -use aws_sdk_dynamodb::operation::RequestId; -use aws_sdk_dynamodb::operation::delete_item::builders::DeleteItemFluentBuilder; -use aws_sdk_dynamodb::operation::{ - get_item::builders::GetItemFluentBuilder, put_item::builders::PutItemFluentBuilder, - query::builders::QueryFluentBuilder, -}; -use aws_sdk_dynamodb::types::{AttributeValue, KeyType}; -use object_store::path::Path; -use snafu::OptionExt; -use tokio::sync::RwLock; -use tracing::warn; - -use crate::io::commit::external_manifest::ExternalManifestStore; -use lance_core::error::NotFoundSnafu; -use lance_core::error::box_error; -use lance_core::{Error, Result}; - -use super::ManifestLocation; -use super::external_manifest::detect_naming_scheme_from_path; - -#[derive(Debug)] -struct WrappedSdkError(SdkError); - -impl From> for Error -where - E: std::error::Error + Send + Sync + 'static, -{ - fn from(e: WrappedSdkError) -> Self { - Self::io_source(box_error(e)) - } -} - -impl std::fmt::Display for WrappedSdkError -where - E: std::error::Error + Send + Sync + 'static, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let request_id = self.0.request_id().unwrap_or("unknown"); - let service_err = &self.0.raw_response(); - write!(f, "WrappedSdkError: request_id: {}", request_id)?; - if let Some(err) = service_err { - write!(f, ", service_error: {:?}", err) - } else { - write!(f, ", no service error") - } - } -} - -impl std::error::Error for WrappedSdkError -where - E: std::error::Error + Send + Sync + 'static, -{ - // Implement the necessary methods for the Error trait here. - // For example, you can delegate to the inner SdkError: - - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.0) - } -} - -trait SdkResultExt { - fn wrap_err(self) -> Result; -} - -impl SdkResultExt for std::result::Result> -where - E: std::error::Error + Send + Sync + 'static, -{ - fn wrap_err(self) -> Result { - self.map_err(|err| { - warn!( - target: "lance::dynamodb", - request_id = err.request_id().unwrap_or("unknown"), - "DynamoDB SDK error: {err:?}", - ); - Error::from(WrappedSdkError(err)) - }) - } -} - -/// An external manifest store backed by DynamoDB -/// -/// When calling DynamoDBExternalManifestStore::new_external_store() -/// the key schema, (PK, SK), is checked. If the table does not exist, -/// or the key schema is not as expected, an error is returned. -/// -/// The table schema is expected as follows: -/// PK: base_uri -- string -/// SK: version -- number -/// path -- string -/// committer -- string -/// -/// Consistency: This store is expected to have read-after-write consistency -/// consistent_read should always be set to true -/// -/// Transaction Safety: This store uses DynamoDB conditional write to ensure -/// only one writer can win per version. -#[derive(Debug)] -pub struct DynamoDBExternalManifestStore { - client: Arc, - table_name: String, - committer_name: String, -} - -// these are in macro because I want to use them in a match statement -macro_rules! base_uri { - () => { - "base_uri" - }; -} -macro_rules! version { - () => { - "version" - }; -} -macro_rules! path { - () => { - "path" - }; -} -macro_rules! committer { - () => { - "committer" - }; -} - -impl DynamoDBExternalManifestStore { - pub async fn new_external_store( - client: Arc, - table_name: &str, - committer_name: &str, - ) -> Result> { - static SANITY_CHECK_CACHE: LazyLock>> = - LazyLock::new(|| RwLock::new(HashSet::new())); - - let store = Arc::new(Self { - client: client.clone(), - table_name: table_name.to_string(), - committer_name: committer_name.to_string(), - }); - - // already checked this table before, skip - // this is to avoid checking the table schema every time - // because it's expensive to call DescribeTable - if SANITY_CHECK_CACHE.read().await.contains(table_name) { - return Ok(store); - } - - // Check if the table schema is correct - let describe_result = client - .describe_table() - .table_name(table_name) - .send() - .await - .wrap_err()?; - let table = describe_result - .table - .ok_or_else(|| Error::io(format!("dynamodb table: {table_name} does not exist")))?; - let mut schema = table.key_schema.ok_or_else(|| { - Error::io(format!( - "dynamodb table: {table_name} does not have a key schema" - )) - })?; - - let mut has_hash_key = false; - let mut has_range_key = false; - - // there should be two keys, HASH(base_uri) and RANGE(version) - for _ in 0..2 { - let key = schema.pop().ok_or_else(|| { - Error::io(format!( - "dynamodb table: {table_name} must have HASH and RANGE keys" - )) - })?; - match (key.key_type, key.attribute_name.as_str()) { - (KeyType::Hash, base_uri!()) => { - has_hash_key = true; - } - (KeyType::Range, version!()) => { - has_range_key = true; - } - _ => { - return Err(Error::io(format!( - "dynamodb table: {} unknown key type encountered name:{}", - table_name, key.attribute_name - ))); - } - } - } - - // Both keys must be present - if !(has_hash_key && has_range_key) { - return Err(Error::io(format!( - "dynamodb table: {} must have HASH and RANGE keys, named `{}` and `{}` respectively", - table_name, - base_uri!(), - version!() - ))); - } - - SANITY_CHECK_CACHE - .write() - .await - .insert(table_name.to_string()); - - Ok(store) - } - - fn ddb_put(&self) -> PutItemFluentBuilder { - self.client.put_item().table_name(&self.table_name) - } - - fn ddb_get(&self) -> GetItemFluentBuilder { - self.client - .get_item() - .table_name(&self.table_name) - .consistent_read(true) - } - - fn ddb_query(&self) -> QueryFluentBuilder { - self.client - .query() - .table_name(&self.table_name) - .consistent_read(true) - } - - fn ddb_delete(&self) -> DeleteItemFluentBuilder { - self.client.delete_item().table_name(&self.table_name) - } -} - -#[async_trait] -impl ExternalManifestStore for DynamoDBExternalManifestStore { - /// Get the manifest path for a given base_uri and version - async fn get(&self, base_uri: &str, version: u64) -> Result { - let get_item_result = self - .ddb_get() - .key(base_uri!(), AttributeValue::S(base_uri.into())) - .key(version!(), AttributeValue::N(version.to_string())) - .send() - .await - .wrap_err()?; - - let item = get_item_result.item.context(NotFoundSnafu { - uri: format!( - "dynamodb not found: base_uri: {}; version: {}", - base_uri, version - ), - })?; - - let path = item - .get(path!()) - .ok_or_else(|| Error::not_found(format!("key {} is not present", path!())))?; - - match path { - AttributeValue::S(path) => Ok(path.clone()), - _ => Err(Error::invalid_input(format!( - "key {} is not a string", - path!() - ))), - } - } - - async fn get_manifest_location( - &self, - base_uri: &str, - version: u64, - ) -> Result { - let get_item_result = self - .ddb_get() - .key(base_uri!(), AttributeValue::S(base_uri.into())) - .key(version!(), AttributeValue::N(version.to_string())) - .send() - .await - .wrap_err()?; - - let item = get_item_result.item.context(NotFoundSnafu { - uri: format!( - "dynamodb not found: base_uri: {}; version: {}", - base_uri, version - ), - })?; - - let path = item - .get(path!()) - .ok_or_else(|| Error::not_found(format!("key {} is not present", path!())))? - .as_s() - .map_err(|_| Error::invalid_input(format!("key {} is not a string", path!())))? - .as_str(); - let path = Path::from(path); - - let size = item - .get("size") - .and_then(|attr| attr.as_n().ok().and_then(|v| v.parse().ok())); - - let e_tag = item.get("e_tag").and_then(|attr| attr.as_s().ok().cloned()); - - let naming_scheme = detect_naming_scheme_from_path(&path)?; - - Ok(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }) - } - - /// Get the latest version of a dataset at the base_uri - async fn get_latest_version(&self, base_uri: &str) -> Result> { - self.get_latest_manifest_location(base_uri) - .await - .map(|location| location.map(|loc| (loc.version, loc.path.to_string()))) - } - - async fn get_latest_manifest_location( - &self, - base_uri: &str, - ) -> Result> { - let query_result = self - .ddb_query() - .key_condition_expression(format!("{} = :{}", base_uri!(), base_uri!())) - .expression_attribute_values( - format!(":{}", base_uri!()), - AttributeValue::S(base_uri.into()), - ) - .scan_index_forward(false) - .limit(1) - .send() - .await - .wrap_err()?; - - match query_result.items { - Some(mut items) => { - if items.is_empty() { - return Ok(None); - } - if items.len() > 1 { - return Err(Error::invalid_input(format!( - "dynamodb table: {} returned unexpected number of items", - self.table_name - ))); - } - - let item = items.pop().expect("length checked"); - let version_attribute = item - .get(version!()) - .ok_or_else(|| Error::not_found( - format!("dynamodb error: found entries for {} but the returned data does not contain {} column", base_uri, version!()) - ))?; - - let path_attribute = item - .get(path!()) - .ok_or_else(|| Error::not_found( - format!("dynamodb error: found entries for {} but the returned data does not contain {} column", base_uri, path!()) - ))?; - - let size = item.get("size").and_then(|attr| match attr { - AttributeValue::N(size) => size.parse().ok(), - _ => None, - }); - - let e_tag = item.get("e_tag").and_then(|attr| attr.as_s().ok().cloned()); - - match (version_attribute, path_attribute) { - (AttributeValue::N(version), AttributeValue::S(path)) => { - let version = version.parse().map_err(|e| Error::invalid_input(format!("dynamodb error: could not parse the version number returned {}, error: {}", version, e)))?; - let path = Path::from(path.as_str()); - let naming_scheme = detect_naming_scheme_from_path(&path)?; - let location = ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }; - Ok(Some(location)) - } - _ => Err(Error::invalid_input(format!( - "dynamodb error: found entries for {base_uri} but the returned data is not number type" - ))), - } - } - _ => Ok(None), - } - } - - /// Put the manifest path for a given base_uri and version, should fail if the version already exists - async fn put_if_not_exists( - &self, - base_uri: &str, - version: u64, - path: &str, - size: u64, - e_tag: Option, - ) -> Result<()> { - let mut put_item = self - .ddb_put() - .item(base_uri!(), AttributeValue::S(base_uri.into())) - .item(version!(), AttributeValue::N(version.to_string())) - .item(path!(), AttributeValue::S(path.to_string())) - .item(committer!(), AttributeValue::S(self.committer_name.clone())) - .item("size", AttributeValue::N(size.to_string())); - - if let Some(e_tag) = e_tag { - put_item = put_item.item("e_tag", AttributeValue::S(e_tag)); - } - - put_item - .condition_expression(format!( - "attribute_not_exists({}) AND attribute_not_exists({})", - base_uri!(), - version!(), - )) - .send() - .await - .wrap_err()?; - - Ok(()) - } - - /// Put the manifest path for a given base_uri and version, should fail if the version **does not** already exist - async fn put_if_exists( - &self, - base_uri: &str, - version: u64, - path: &str, - size: u64, - e_tag: Option, - ) -> Result<()> { - let mut put_item = self - .ddb_put() - .item(base_uri!(), AttributeValue::S(base_uri.into())) - .item(version!(), AttributeValue::N(version.to_string())) - .item(path!(), AttributeValue::S(path.to_string())) - .item(committer!(), AttributeValue::S(self.committer_name.clone())) - .item("size", AttributeValue::N(size.to_string())); - - if let Some(e_tag) = e_tag { - put_item = put_item.item("e_tag", AttributeValue::S(e_tag)); - } - - put_item - .condition_expression(format!( - "attribute_exists({}) AND attribute_exists({})", - base_uri!(), - version!(), - )) - .send() - .await - .wrap_err()?; - - Ok(()) - } - - /// Delete the manifest information for the given base_uri in dynamodb - async fn delete(&self, base_uri: &str) -> Result<()> { - let query_result = self - .ddb_query() - .key_condition_expression(format!("{} = :{}", base_uri!(), base_uri!())) - .expression_attribute_values( - format!(":{}", base_uri!()), - AttributeValue::S(base_uri.into()), - ) - .send() - .await - .wrap_err()?; - - if let Some(items) = query_result.items { - for item in items { - if let Some(AttributeValue::N(version)) = item.get("version") { - self.ddb_delete() - .key(base_uri!(), AttributeValue::S(base_uri.to_string())) - .key(version!(), AttributeValue::N(version.clone())) - .send() - .await - .wrap_err()?; - } - } - } - Ok(()) - } -} diff --git a/vendor/lance-table/src/io/commit/external_manifest.rs b/vendor/lance-table/src/io/commit/external_manifest.rs deleted file mode 100644 index 75993ca8..00000000 --- a/vendor/lance-table/src/io/commit/external_manifest.rs +++ /dev/null @@ -1,515 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Trait for external manifest handler. -//! -//! This trait abstracts an external storage with put_if_not_exists semantics. - -use std::sync::Arc; - -use async_trait::async_trait; -use lance_core::utils::tracing::{ - AUDIT_MODE_CREATE, AUDIT_MODE_DELETE, AUDIT_TYPE_MANIFEST, TRACE_FILE_AUDIT, -}; -use lance_core::{Error, Result}; -use lance_io::object_store::ObjectStore; -use log::warn; -use object_store::ObjectMeta; -use object_store::ObjectStoreExt; -use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, path::Path}; -use tracing::info; - -use super::{ - MANIFEST_EXTENSION, ManifestLocation, ManifestNamingScheme, current_manifest_path, - default_resolve_version, make_staging_manifest_path, write_version_hint, -}; -use crate::format::{IndexMetadata, Manifest, Transaction}; -use crate::io::commit::{CommitError, CommitHandler}; - -/// External manifest store -/// -/// This trait abstracts an external storage for source of truth for manifests. -/// The storage is expected to remember (uri, version) -> manifest_path -/// and able to run transactions on the manifest_path. -/// -/// This trait is called an **External** manifest store because the store is -/// expected to work in tandem with the object store. We are only leveraging -/// the external store for concurrent commit. Any manifest committed thru this -/// trait should ultimately be materialized in the object store. -/// For a visual explanation of the commit loop see -/// -#[async_trait] -pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { - /// Get the manifest path for a given base_uri and version - async fn get(&self, base_uri: &str, version: u64) -> Result; - - async fn get_manifest_location( - &self, - base_uri: &str, - version: u64, - ) -> Result { - let path = self.get(base_uri, version).await?; - let path = Path::parse(&path).map_err(|e| Error::invalid_input(e.to_string()))?; - let naming_scheme = detect_naming_scheme_from_path(&path)?; - Ok(ManifestLocation { - version, - path, - size: None, - naming_scheme, - e_tag: None, - }) - } - - /// Get the latest version of a dataset at the base_uri, and the path to the manifest. - /// The path is provided as an optimization. The path is deterministic based on - /// the version and the store should not customize it. - async fn get_latest_version(&self, base_uri: &str) -> Result>; - - /// Get the latest manifest location for a given base_uri. - /// - /// By default, this calls get_latest_version. Impls should - /// override this method if they store both the location and size - /// of the latest manifest. - async fn get_latest_manifest_location( - &self, - base_uri: &str, - ) -> Result> { - self.get_latest_version(base_uri).await.and_then(|res| { - res.map(|(version, uri)| { - let path = Path::parse(&uri).map_err(|e| Error::invalid_input(e.to_string()))?; - let naming_scheme = detect_naming_scheme_from_path(&path)?; - Ok(ManifestLocation { - version, - path, - size: None, - naming_scheme, - e_tag: None, - }) - }) - .transpose() - }) - } - - /// Put the manifest to the external store. - /// - /// The staging manifest has been written to `staging_path` on the object store. - /// This method should atomically claim the version and return the final manifest location. - /// - /// The default implementation uses put_if_not_exists and put_if_exists to - /// implement a staging-based workflow. Implementations that can write directly - /// (e.g., namespace-backed stores) should override this method. - #[allow(clippy::too_many_arguments)] - async fn put( - &self, - base_path: &Path, - version: u64, - staging_path: &Path, - size: u64, - e_tag: Option, - object_store: &dyn OSObjectStore, - naming_scheme: ManifestNamingScheme, - ) -> Result { - // Default implementation: staging-based workflow - - // Step 1: Record staging path atomically - self.put_if_not_exists( - base_path.as_ref(), - version, - staging_path.as_ref(), - size, - e_tag.clone(), - ) - .await?; - - // Step 2: Copy staging to final path - let final_path = naming_scheme.manifest_path(base_path, version); - let copied = match object_store.copy(staging_path, &final_path).await { - Ok(_) => true, - Err(ObjectStoreError::NotFound { .. }) => false, - Err(e) => return Err(e.into()), - }; - if copied { - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_path.as_ref()); - } - - // Get final e_tag (may change after copy for large files) - let e_tag = if copied && size < 5 * 1024 * 1024 { - e_tag - } else { - let meta = object_store.head(&final_path).await?; - meta.e_tag - }; - - let location = ManifestLocation { - version, - path: final_path.clone(), - size: Some(size), - naming_scheme, - e_tag: e_tag.clone(), - }; - - if !copied { - return Ok(location); - } - - // Step 3: Update external store to final path - self.put_if_exists( - base_path.as_ref(), - version, - final_path.as_ref(), - size, - e_tag, - ) - .await?; - - // Step 4: Delete staging manifest - match object_store.delete(staging_path).await { - Ok(_) => {} - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(e.into()), - } - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); - - Ok(location) - } - - /// Put the manifest path for a given base_uri and version, should fail if the version already exists - async fn put_if_not_exists( - &self, - base_uri: &str, - version: u64, - path: &str, - size: u64, - e_tag: Option, - ) -> Result<()>; - - /// Put the manifest path for a given base_uri and version, should fail if the version **does not** already exist - async fn put_if_exists( - &self, - base_uri: &str, - version: u64, - path: &str, - size: u64, - e_tag: Option, - ) -> Result<()>; - - /// Delete the manifest information for given base_uri from the store - async fn delete(&self, _base_uri: &str) -> Result<()> { - Ok(()) - } -} - -pub(crate) fn detect_naming_scheme_from_path(path: &Path) -> Result { - path.filename() - .and_then(|name| { - ManifestNamingScheme::detect_scheme(name) - .or_else(|| Some(ManifestNamingScheme::detect_scheme_staging(name))) - }) - .ok_or_else(|| { - Error::corrupt_file( - path.clone(), - "Path does not follow known manifest naming convention.", - ) - }) -} - -/// External manifest commit handler -/// This handler is used to commit a manifest to an external store -/// for detailed design, see -#[derive(Debug)] -pub struct ExternalManifestCommitHandler { - pub external_manifest_store: Arc, -} - -impl ExternalManifestCommitHandler { - /// The manifest is considered committed once the staging manifest is written - /// to object store and that path is committed to the external store. - /// - /// However, to fully complete this, the staging manifest should be materialized - /// into the final path, the final path should be committed to the external store - /// and the staging manifest should be deleted. These steps may be completed - /// by any number of readers or writers, so care should be taken to ensure - /// that the manifest is not lost nor any errors occur due to duplicate - /// operations. - #[allow(clippy::too_many_arguments)] - async fn finalize_manifest( - &self, - base_path: &Path, - staging_manifest_path: &Path, - version: u64, - size: u64, - e_tag: Option, - store: &dyn OSObjectStore, - naming_scheme: ManifestNamingScheme, - ) -> std::result::Result { - // step 1: copy the manifest to the final location - let final_manifest_path = naming_scheme.manifest_path(base_path, version); - - let copied = match store - .copy(staging_manifest_path, &final_manifest_path) - .await - { - Ok(_) => true, - Err(ObjectStoreError::NotFound { .. }) => false, // Another writer beat us to it. - Err(e) => return Err(e.into()), - }; - if copied { - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_manifest_path.as_ref()); - } - - // On S3, the etag can change if originally was MultipartUpload and later was Copy - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html#AmazonS3-Type-Object-ETag - // We only do MultipartUpload for > 5MB files, so we can skip this check - // if size < 5MB. However, we need to double check the final_manifest_path - // exists before we change the external store, otherwise we may point to a - // non-existing manifest. - let e_tag = if copied && size < 5 * 1024 * 1024 { - e_tag - } else { - let meta = store.head(&final_manifest_path).await?; - meta.e_tag - }; - - let location = ManifestLocation { - version, - path: final_manifest_path, - size: Some(size), - naming_scheme, - e_tag, - }; - - if !copied { - return Ok(location); - } - - // step 2: flip the external store to point to the final location - self.external_manifest_store - .put_if_exists( - base_path.as_ref(), - version, - location.path.as_ref(), - size, - location.e_tag.clone(), - ) - .await?; - - // step 3: delete the staging manifest - match store.delete(staging_manifest_path).await { - Ok(_) => {} - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(e.into()), - } - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_manifest_path.as_ref()); - - Ok(location) - } -} - -#[async_trait] -impl CommitHandler for ExternalManifestCommitHandler { - async fn resolve_latest_location( - &self, - base_path: &Path, - object_store: &ObjectStore, - ) -> std::result::Result { - let location = self - .external_manifest_store - .get_latest_manifest_location(base_path.as_ref()) - .await?; - - match location { - Some(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }) => { - // The path is finalized, no need to check object store - if path.extension() == Some(MANIFEST_EXTENSION) { - return Ok(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }); - } - - let (size, e_tag) = if let Some(size) = size { - (size, e_tag) - } else { - match object_store.inner.head(&path).await { - Ok(meta) => (meta.size, meta.e_tag), - Err(ObjectStoreError::NotFound { .. }) => { - // there may be other threads that have finished executing finalize_manifest. - let new_location = self - .external_manifest_store - .get_manifest_location(base_path.as_ref(), version) - .await?; - return Ok(new_location); - } - Err(e) => return Err(e.into()), - } - }; - - let final_location = self - .finalize_manifest( - base_path, - &path, - version, - size, - e_tag.clone(), - &object_store.inner, - naming_scheme, - ) - .await?; - - Ok(final_location) - } - // Dataset not found in the external store, this could be because the dataset did not - // use external store for commit before. In this case, we search for the latest manifest - None => current_manifest_path(object_store, base_path).await, - } - } - - async fn resolve_version_location( - &self, - base_path: &Path, - version: u64, - object_store: &dyn OSObjectStore, - ) -> std::result::Result { - let location_res = self - .external_manifest_store - .get_manifest_location(base_path.as_ref(), version) - .await; - - let location = match location_res { - Ok(p) => p, - // not board external manifest yet, direct to object store - Err(Error::NotFound { .. }) => { - let path = default_resolve_version(base_path, version, object_store) - .await - .map_err(|_| Error::not_found(format!("{}@{}", base_path, version)))? - .path; - match object_store.head(&path).await { - Ok(ObjectMeta { size, e_tag, .. }) => { - let res = self - .external_manifest_store - .put_if_not_exists( - base_path.as_ref(), - version, - path.as_ref(), - size, - e_tag.clone(), - ) - .await; - if let Err(e) = res { - warn!( - "could not update external manifest store during load, with error: {}", - e - ); - } - let naming_scheme = - ManifestNamingScheme::detect_scheme_staging(path.filename().unwrap()); - return Ok(ManifestLocation { - version, - path, - size: Some(size), - naming_scheme, - e_tag, - }); - } - Err(ObjectStoreError::NotFound { .. }) => { - return Err(Error::not_found(path.to_string())); - } - Err(e) => return Err(e.into()), - } - } - Err(e) => return Err(e), - }; - - // finalized path, just return - if location.path.extension() == Some(MANIFEST_EXTENSION) { - return Ok(location); - } - - let naming_scheme = - ManifestNamingScheme::detect_scheme_staging(location.path.filename().unwrap()); - - let (size, e_tag) = if let Some(size) = location.size { - (size, location.e_tag.clone()) - } else { - let meta = object_store.head(&location.path).await?; - (meta.size as u64, meta.e_tag) - }; - - self.finalize_manifest( - base_path, - &location.path, - version, - size, - e_tag, - object_store, - naming_scheme, - ) - .await - } - - async fn commit( - &self, - manifest: &mut Manifest, - indices: Option>, - base_path: &Path, - object_store: &ObjectStore, - manifest_writer: super::ManifestWriter, - naming_scheme: ManifestNamingScheme, - transaction: Option, - ) -> std::result::Result { - // path we get here is the path to the manifest we want to write - // use object_store.base_path.as_ref() for getting the root of the dataset - - // step 1: Write the manifest we want to commit to object store with a temporary name - let path = naming_scheme.manifest_path(base_path, manifest.version); - let staging_path = make_staging_manifest_path(&path)?; - let write_res = - manifest_writer(object_store, manifest, indices, &staging_path, transaction).await?; - - // step 2 & 3: Put the manifest to external store - let result = self - .external_manifest_store - .put( - base_path, - manifest.version, - &staging_path, - write_res.size as u64, - write_res.e_tag, - &object_store.inner, - naming_scheme, - ) - .await; - - match result { - Ok(location) => { - write_version_hint(object_store, base_path, manifest.version).await; - Ok(location) - } - Err(_) => { - // delete the staging manifest - match object_store.inner.delete(&staging_path).await { - Ok(_) => {} - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(CommitError::OtherError(e.into())), - } - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); - Err(CommitError::CommitConflict {}) - } - } - } - - async fn delete(&self, base_path: &Path) -> Result<()> { - self.external_manifest_store - .delete(base_path.as_ref()) - .await - } -} diff --git a/vendor/lance-table/src/io/deletion.rs b/vendor/lance-table/src/io/deletion.rs deleted file mode 100644 index 01bc6d3b..00000000 --- a/vendor/lance-table/src/io/deletion.rs +++ /dev/null @@ -1,370 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::{collections::HashSet, sync::Arc}; - -use arrow_array::{RecordBatch, UInt32Array}; -use arrow_ipc::CompressionType; -use arrow_ipc::reader::FileReader as ArrowFileReader; -use arrow_ipc::writer::{FileWriter as ArrowFileWriter, IpcWriteOptions}; -use arrow_schema::{ArrowError, DataType, Field, Schema}; -use bytes::Buf; -use lance_core::error::{CorruptFileSnafu, box_error}; -use lance_core::utils::deletion::DeletionVector; -use lance_core::utils::tracing::{AUDIT_MODE_CREATE, AUDIT_TYPE_DELETION, TRACE_FILE_AUDIT}; -use lance_core::{Error, Result}; -use lance_io::object_store::ObjectStore; -use object_store::path::Path; -use rand::Rng; -use roaring::bitmap::RoaringBitmap; -use snafu::ResultExt; -use tracing::{info, instrument}; - -use crate::format::{DeletionFile, DeletionFileType}; - -pub const DELETIONS_DIR: &str = "_deletions"; - -/// Get the Arrow schema for an Arrow deletion file. -fn deletion_arrow_schema() -> Arc { - Arc::new(Schema::new(vec![Field::new( - "row_id", - DataType::UInt32, - false, - )])) -} - -/// Get the file path for a deletion file. This is relative to the dataset root. -pub fn deletion_file_path(base: &Path, fragment_id: u64, deletion_file: &DeletionFile) -> Path { - let DeletionFile { - read_version, - id, - file_type, - .. - } = deletion_file; - let suffix = file_type.suffix(); - base.clone() - .join(DELETIONS_DIR) - .join(format!("{fragment_id}-{read_version}-{id}.{suffix}")) -} - -pub fn relative_deletion_file_path(fragment_id: u64, deletion_file: &DeletionFile) -> String { - let DeletionFile { - read_version, - id, - file_type, - .. - } = deletion_file; - let suffix = file_type.suffix(); - format!("{DELETIONS_DIR}/{fragment_id}-{read_version}-{id}.{suffix}") -} - -/// Write a deletion file for a fragment for a given deletion vector. -/// -/// Returns the deletion file if one was written. If no deletions were present, -/// returns `Ok(None)`. -pub async fn write_deletion_file( - base: &Path, - fragment_id: u64, - read_version: u64, - removed_rows: &DeletionVector, - object_store: &ObjectStore, -) -> Result> { - let deletion_file = match removed_rows { - DeletionVector::NoDeletions => None, - DeletionVector::Set(set) => { - let id = rand::rng().random::(); - let deletion_file = DeletionFile { - read_version, - id, - file_type: DeletionFileType::Array, - num_deleted_rows: Some(set.len()), - base_id: None, - }; - let path = deletion_file_path(base, fragment_id, &deletion_file); - - let array = UInt32Array::from_iter(set.iter().copied()); - let array = Arc::new(array); - - let schema = deletion_arrow_schema(); - let batch = RecordBatch::try_new(schema.clone(), vec![array])?; - - let mut out: Vec = Vec::new(); - let write_options = - IpcWriteOptions::default().try_with_compression(Some(CompressionType::ZSTD))?; - { - let mut writer = ArrowFileWriter::try_new_with_options( - &mut out, - schema.as_ref(), - write_options, - )?; - writer.write(&batch)?; - writer.finish()?; - // Drop writer so out is no longer borrowed. - } - - object_store.put(&path, &out).await?; - - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DELETION, path = path.to_string()); - - Some(deletion_file) - } - DeletionVector::Bitmap(bitmap) => { - let id = rand::rng().random::(); - let deletion_file = DeletionFile { - read_version, - id, - file_type: DeletionFileType::Bitmap, - num_deleted_rows: Some(bitmap.len() as usize), - base_id: None, - }; - let path = deletion_file_path(base, fragment_id, &deletion_file); - - let mut out: Vec = Vec::new(); - bitmap.serialize_into(&mut out)?; - - object_store.put(&path, &out).await?; - - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DELETION, path = path.to_string()); - - Some(deletion_file) - } - }; - Ok(deletion_file) -} - -#[instrument( - level = "debug", - skip(base, object_store), - fields( - base = base.as_ref(), - bytes_read = tracing::field::Empty - ) -)] -pub async fn read_deletion_file( - fragment_id: u64, - deletion_file: &DeletionFile, - base: &Path, - object_store: &ObjectStore, -) -> Result { - let span = tracing::Span::current(); - match deletion_file.file_type { - DeletionFileType::Array => { - let path = deletion_file_path(base, fragment_id, deletion_file); - - let data = object_store.read_one_all(&path).await?; - span.record("bytes_read", data.len()); - let data = std::io::Cursor::new(data); - let mut batches: Vec = ArrowFileReader::try_new(data, None)? - .collect::>() - .map_err(box_error) - .context(CorruptFileSnafu { path: path.clone() })?; - - if batches.len() != 1 { - return Err(Error::corrupt_file( - path, - format!( - "Expected exactly one batch in deletion file, got {}", - batches.len() - ), - )); - } - - let batch = batches.pop().unwrap(); - if batch.schema() != deletion_arrow_schema() { - return Err(Error::corrupt_file( - path, - format!( - "Expected schema {:?} in deletion file, got {:?}", - deletion_arrow_schema(), - batch.schema() - ), - )); - } - - let array = batch.columns()[0] - .as_any() - .downcast_ref::() - .unwrap(); - - let mut set = HashSet::with_capacity(array.len()); - for val in array.iter() { - if let Some(val) = val { - set.insert(val); - } else { - return Err(Error::corrupt_file( - path, - "Null values are not allowed in deletion files", - )); - } - } - - Ok(DeletionVector::Set(set)) - } - DeletionFileType::Bitmap => { - let path = deletion_file_path(base, fragment_id, deletion_file); - - let data = object_store.read_one_all(&path).await?; - span.record("bytes_read", data.len()); - let reader = data.reader(); - let bitmap = RoaringBitmap::deserialize_from(reader) - .map_err(box_error) - .context(CorruptFileSnafu { path })?; - - Ok(DeletionVector::Bitmap(bitmap)) - } - } -} - -#[cfg(test)] -mod test { - - use super::*; - use object_store::ObjectStoreExt; - - #[tokio::test] - async fn test_write_no_deletions() { - let dv = DeletionVector::NoDeletions; - - let (object_store, path) = ObjectStore::from_uri("memory:///no_deletion") - .await - .unwrap(); - let file = write_deletion_file(&path, 0, 0, &dv, &object_store) - .await - .unwrap(); - assert!(file.is_none()); - } - - #[tokio::test] - async fn test_write_array() { - let dv = DeletionVector::Set(HashSet::from_iter(0..100)); - - let fragment_id = 21; - let read_version = 12; - - let object_store = ObjectStore::memory(); - let path = Path::from("/write"); - let file = write_deletion_file(&path, fragment_id, read_version, &dv, &object_store) - .await - .unwrap(); - - assert!(matches!( - file, - Some(DeletionFile { - file_type: DeletionFileType::Array, - .. - }) - )); - - let file = file.unwrap(); - assert_eq!(file.read_version, read_version); - let path = deletion_file_path(&path, fragment_id, &file); - assert_eq!( - path, - Path::from(format!("/write/_deletions/21-12-{}.arrow", file.id)) - ); - - let data = object_store - .inner - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(); - let data = std::io::Cursor::new(data); - let mut batches: Vec = ArrowFileReader::try_new(data, None) - .unwrap() - .collect::>() - .unwrap(); - - assert_eq!(batches.len(), 1); - let batch = batches.pop().unwrap(); - assert_eq!(batch.schema(), deletion_arrow_schema()); - let array = batch["row_id"] - .as_any() - .downcast_ref::() - .unwrap(); - let read_dv = DeletionVector::from_iter(array.iter().map(|v| v.unwrap())); - assert_eq!(read_dv, dv); - } - - #[tokio::test] - async fn test_write_bitmap() { - let dv = DeletionVector::Bitmap(RoaringBitmap::from_iter(0..100)); - - let fragment_id = 21; - let read_version = 12; - - let object_store = ObjectStore::memory(); - let path = Path::from("/bitmap"); - let file = write_deletion_file(&path, fragment_id, read_version, &dv, &object_store) - .await - .unwrap(); - - assert!(matches!( - file, - Some(DeletionFile { - file_type: DeletionFileType::Bitmap, - .. - }) - )); - - let file = file.unwrap(); - assert_eq!(file.read_version, read_version); - let path = deletion_file_path(&path, fragment_id, &file); - assert_eq!( - path, - Path::from(format!("/bitmap/_deletions/21-12-{}.bin", file.id)) - ); - - let data = object_store - .inner - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(); - let reader = data.reader(); - let read_bitmap = RoaringBitmap::deserialize_from(reader).unwrap(); - assert_eq!(read_bitmap, dv.into_iter().collect::()); - } - - #[tokio::test] - async fn test_roundtrip_array() { - let dv = DeletionVector::Set(HashSet::from_iter(0..100)); - - let fragment_id = 21; - let read_version = 12; - - let object_store = ObjectStore::memory(); - let path = Path::from("/roundtrip"); - let file = write_deletion_file(&path, fragment_id, read_version, &dv, &object_store) - .await - .unwrap(); - - let read_dv = read_deletion_file(fragment_id, &file.unwrap(), &path, &object_store) - .await - .unwrap(); - assert_eq!(read_dv, dv); - } - - #[tokio::test] - async fn test_roundtrip_bitmap() { - let dv = DeletionVector::Bitmap(RoaringBitmap::from_iter(0..100)); - - let fragment_id = 21; - let read_version = 12; - - let object_store = ObjectStore::memory(); - let path = Path::from("/bitmap"); - let file = write_deletion_file(&path, fragment_id, read_version, &dv, &object_store) - .await - .unwrap(); - - let read_dv = read_deletion_file(fragment_id, &file.unwrap(), &path, &object_store) - .await - .unwrap(); - assert_eq!(read_dv, dv); - } -} diff --git a/vendor/lance-table/src/io/manifest.rs b/vendor/lance-table/src/io/manifest.rs deleted file mode 100644 index 4b918177..00000000 --- a/vendor/lance-table/src/io/manifest.rs +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use async_trait::async_trait; -use byteorder::{ByteOrder, LittleEndian}; -use bytes::{Bytes, BytesMut}; -use lance_arrow::DataTypeExt; -use lance_file::{ - previous::writer::ManifestProvider as PreviousManifestProvider, version::LanceFileVersion, -}; -use object_store::ObjectStoreExt; -use object_store::path::Path; -use prost::Message; -use std::collections::HashMap; -use std::{ops::Range, sync::Arc}; -use tracing::instrument; - -use lance_core::{Error, Result, datatypes::Schema}; -use lance_io::{ - encodings::{Encoder, binary::BinaryEncoder, plain::PlainEncoder}, - object_store::ObjectStore, - traits::{WriteExt, Writer}, - utils::read_message, -}; - -use crate::format::{DataStorageFormat, IndexMetadata, MAGIC, Manifest, Transaction, pb}; - -use super::commit::ManifestLocation; - -/// Read Manifest on URI. -/// -/// This only reads manifest files. It does not read data files. -#[instrument(level = "debug", skip(object_store))] -pub async fn read_manifest( - object_store: &ObjectStore, - path: &Path, - known_size: Option, -) -> Result { - let file_size = if let Some(known_size) = known_size { - known_size - } else { - object_store.inner.head(path).await?.size - }; - const PREFETCH_SIZE: u64 = 64 * 1024; - let initial_start = file_size.saturating_sub(PREFETCH_SIZE); - let range = Range { - start: initial_start, - end: file_size, - }; - let buf = object_store.inner.get_range(path, range).await?; - - // In case of corruption, the known_size might be wrong. We can retry without - // the size to be more robust. - if (buf.len() < 16 || !buf.ends_with(MAGIC)) && known_size.is_some() { - return Box::pin(read_manifest(object_store, path, None)).await; - } - - if buf.len() < 16 { - return Err(Error::corrupt_file( - path.clone(), - "Invalid format: file size is smaller than 16 bytes".to_string(), - )); - } - if !buf.ends_with(MAGIC) { - return Err(Error::corrupt_file( - path.clone(), - "Invalid format: magic number does not match".to_string(), - )); - } - let manifest_pos = LittleEndian::read_i64(&buf[buf.len() - 16..buf.len() - 8]) as usize; - let manifest_len = file_size as usize - manifest_pos; - - let buf: Bytes = if manifest_len <= buf.len() { - // The prefetch captured the entire manifest. We just need to trim the buffer. - buf.slice(buf.len() - manifest_len..buf.len()) - } else { - // The prefetch only captured part of the manifest. We need to make an - // additional range request to read the remainder. - let mut buf2: BytesMut = object_store - .inner - .get_range( - path, - Range { - start: manifest_pos as u64, - end: file_size - PREFETCH_SIZE, - }, - ) - .await? - .into_iter() - .collect(); - buf2.extend_from_slice(&buf); - buf2.freeze() - }; - - let recorded_length = LittleEndian::read_u32(&buf[0..4]) as usize; - // Need to trim the magic number at end and message length at beginning - let buf = buf.slice(4..buf.len() - 16); - - if buf.len() != recorded_length { - return Err(Error::invalid_input(format!( - "Invalid format: manifest length does not match. Expected {}, got {}", - recorded_length, - buf.len() - ))); - } - - let proto = pb::Manifest::decode(buf)?; - Manifest::try_from(proto) -} - -#[instrument(level = "debug", skip(object_store, manifest))] -pub async fn read_manifest_indexes( - object_store: &ObjectStore, - location: &ManifestLocation, - manifest: &Manifest, -) -> Result> { - if let Some(pos) = manifest.index_section.as_ref() { - let reader = if let Some(size) = location.size { - object_store - .open_with_size(&location.path, size as usize) - .await? - } else { - object_store.open(&location.path).await? - }; - let section: pb::IndexSection = read_message(reader.as_ref(), *pos).await?; - - let indices = section - .indices - .into_iter() - .map(IndexMetadata::try_from) - .collect::>>()?; - Ok(indices) - } else { - Ok(vec![]) - } -} - -async fn do_write_manifest( - writer: &mut dyn Writer, - manifest: &mut Manifest, - indices: Option>, - mut transaction: Option, -) -> Result { - // Write indices if presented. - if let Some(indices) = indices.as_ref() { - let section = pb::IndexSection { - indices: indices.iter().map(|i| i.into()).collect(), - }; - let pos = writer.write_protobuf(§ion).await?; - manifest.index_section = Some(pos); - } - - // Write inline transaction if presented. - if let Some(tx) = transaction.take() { - // Convert to protobuf at the write boundary to persist inline - let pb_tx: pb::Transaction = tx.into(); - let pos = writer.write_protobuf(&pb_tx).await?; - manifest.transaction_section = Some(pos); - } - - writer.write_struct(manifest).await -} - -/// Write manifest to an open file. -pub async fn write_manifest( - writer: &mut dyn Writer, - manifest: &mut Manifest, - indices: Option>, - transaction: Option, -) -> Result { - // Write dictionary values. - let max_field_id = manifest.schema.max_field_id().unwrap_or(-1); - let is_legacy_storage = manifest.should_use_legacy_format(); - for field_id in 0..max_field_id + 1 { - if let Some(field) = manifest.schema.mut_field_by_id(field_id) - && field.data_type().is_dictionary() - && is_legacy_storage - { - let dict_info = field.dictionary.as_mut().ok_or_else(|| { - Error::io(format!("Lance field {} misses dictionary info", field.name)) - })?; - - let value_arr = dict_info.values.as_ref().ok_or_else(|| { - Error::io(format!( - "Lance field {} is dictionary type, but misses the dictionary value array", - field.name - )) - })?; - - let data_type = value_arr.data_type(); - let pos = match data_type { - dt if dt.is_numeric() => { - let mut encoder = PlainEncoder::new(writer, dt); - encoder.encode(&[value_arr]).await? - } - dt if dt.is_binary_like() => { - let mut encoder = BinaryEncoder::new(writer); - encoder.encode(&[value_arr]).await? - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_arr.data_type() - ))); - } - }; - dict_info.offset = pos; - dict_info.length = value_arr.len(); - } - } - - do_write_manifest(writer, manifest, indices, transaction).await -} - -/// Implementation of ManifestProvider that describes a Lance file by writing -/// a manifest that contains nothing but default fields and the schema -pub struct ManifestDescribing {} - -#[async_trait] -impl PreviousManifestProvider for ManifestDescribing { - async fn store_schema( - object_writer: &mut dyn Writer, - schema: &Schema, - ) -> Result> { - let mut manifest = Manifest::new( - schema.clone(), - Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Legacy), - HashMap::new(), - ); - let pos = do_write_manifest(object_writer, &mut manifest, None, None).await?; - Ok(Some(pos)) - } -} - -#[cfg(test)] -mod test { - use arrow_array::{Int32Array, RecordBatch}; - use std::collections::HashMap; - - use crate::format::SelfDescribingFileReader; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_file::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION}; - use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, - }; - use rand::{Rng, distr::Alphanumeric}; - use tokio::io::AsyncWriteExt; - - use super::*; - - async fn test_roundtrip_manifest(prefix_size: usize, manifest_min_size: usize) { - let store = ObjectStore::memory(); - let path = Path::from("/read_large_manifest"); - - let mut writer = store.create(&path).await.unwrap(); - - // Write prefix we should ignore - let prefix: Vec = rand::rng() - .sample_iter(&Alphanumeric) - .take(prefix_size) - .collect(); - writer.write_all(&prefix).await.unwrap(); - - let long_name: String = rand::rng() - .sample_iter(&Alphanumeric) - .take(manifest_min_size) - .map(char::from) - .collect(); - - let arrow_schema = - ArrowSchema::new(vec![ArrowField::new(long_name, DataType::Int64, false)]); - let schema = Schema::try_from(&arrow_schema).unwrap(); - - let mut config = HashMap::new(); - config.insert("key".to_string(), "value".to_string()); - - let mut manifest = Manifest::new( - schema, - Arc::new(vec![]), - DataStorageFormat::default(), - HashMap::new(), - ); - let pos = write_manifest(writer.as_mut(), &mut manifest, None, None) - .await - .unwrap(); - writer - .write_magics(pos, MAJOR_VERSION, MINOR_VERSION, MAGIC) - .await - .unwrap(); - Writer::shutdown(writer.as_mut()).await.unwrap(); - - let roundtripped_manifest = read_manifest(&store, &path, None).await.unwrap(); - - assert_eq!(manifest, roundtripped_manifest); - - store.inner.delete(&path).await.unwrap(); - } - - #[tokio::test] - async fn test_read_large_manifest() { - test_roundtrip_manifest(0, 100_000).await; - test_roundtrip_manifest(1000, 100_000).await; - test_roundtrip_manifest(1000, 1000).await; - } - - #[tokio::test] - async fn test_update_schema_metadata() { - let store = ObjectStore::memory(); - let path = Path::from("/update_schema_metadata"); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( - "i", - DataType::Int32, - false, - )])); - let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( - &store, - &path, - schema.clone(), - &Default::default(), - ) - .await - .unwrap(); - - let array = Int32Array::from_iter_values(0..10); - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(array)]).unwrap(); - file_writer - .write(std::slice::from_ref(&batch)) - .await - .unwrap(); - let mut metadata = HashMap::new(); - metadata.insert(String::from("lance:extra"), String::from("for_test")); - file_writer.finish_with_metadata(&metadata).await.unwrap(); - - let reader = store.open(&path).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described_from_reader(reader.into(), None) - .await - .unwrap(); - let schema = ArrowSchema::from(reader.schema()); - assert_eq!(schema.metadata().get("lance:extra").unwrap(), "for_test"); - } -} diff --git a/vendor/lance-table/src/lib.rs b/vendor/lance-table/src/lib.rs deleted file mode 100644 index ebe892ba..00000000 --- a/vendor/lance-table/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -pub mod feature_flags; -pub mod format; -pub mod io; -pub mod rowids; -pub mod utils; diff --git a/vendor/lance-table/src/rowids.rs b/vendor/lance-table/src/rowids.rs deleted file mode 100644 index 000c584c..00000000 --- a/vendor/lance-table/src/rowids.rs +++ /dev/null @@ -1,1364 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors -//! Indices for mapping row ids to their corresponding addresses. -//! -//! Each fragment in a table has a [RowIdSequence] that contains the row ids -//! in the order they appear in the fragment. The [RowIdIndex] aggregates these -//! sequences and maps row ids to their corresponding addresses across the -//! whole dataset. -//! -//! [RowIdSequence]s are serialized individually and stored in the fragment -//! metadata. Use [read_row_ids] and [write_row_ids] to read and write these -//! sequences. The on-disk format is designed to align well with the in-memory -//! representation, to avoid unnecessary deserialization. -use std::ops::{Range, RangeInclusive}; -// TODO: replace this with Arrow BooleanBuffer. - -// These are all internal data structures, and are private. -mod bitmap; -mod encoded_array; -mod index; -pub mod segment; -mod serde; -pub mod version; - -use deepsize::DeepSizeOf; -// These are the public API. -pub use index::FragmentRowIdIndex; -pub use index::RowIdIndex; -use lance_core::{ - Error, Result, - utils::mask::{RowAddrMask, RowAddrTreeMap}, -}; -use lance_io::ReadBatchParams; -pub use serde::{read_row_ids, write_row_ids}; - -use crate::utils::LanceIteratorExtension; -use lance_core::utils::mask::RowSetOps; -use segment::U64Segment; -use tracing::instrument; - -/// A sequence of row ids. -/// -/// Row ids are u64s that: -/// -/// 1. Are **unique** within a table (except for tombstones) -/// 2. Are *often* but not always sorted and/or contiguous. -/// -/// This sequence of row ids is optimized to be compact when the row ids are -/// contiguous and sorted. However, it does not require that the row ids are -/// contiguous or sorted. -/// -/// We can make optimizations that assume uniqueness. -#[derive(Debug, Clone, DeepSizeOf, PartialEq, Eq, Default)] -pub struct RowIdSequence(Vec); - -impl std::fmt::Display for RowIdSequence { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut iter = self.iter(); - let mut first_10 = Vec::new(); - let mut last_10 = Vec::new(); - for row_id in iter.by_ref() { - first_10.push(row_id); - if first_10.len() > 10 { - break; - } - } - - while let Some(row_id) = iter.next_back() { - last_10.push(row_id); - if last_10.len() > 10 { - break; - } - } - last_10.reverse(); - - let theres_more = iter.next().is_some(); - - write!(f, "[")?; - for row_id in first_10 { - write!(f, "{}", row_id)?; - } - if theres_more { - write!(f, ", ...")?; - } - for row_id in last_10 { - write!(f, ", {}", row_id)?; - } - write!(f, "]") - } -} - -impl From> for RowIdSequence { - fn from(range: Range) -> Self { - Self(vec![U64Segment::Range(range)]) - } -} - -impl From<&[u64]> for RowIdSequence { - fn from(row_ids: &[u64]) -> Self { - Self(vec![U64Segment::from_slice(row_ids)]) - } -} - -impl RowIdSequence { - pub fn new() -> Self { - Self::default() - } - - pub fn iter(&self) -> impl DoubleEndedIterator + '_ { - self.0.iter().flat_map(|segment| segment.iter()) - } - - pub fn len(&self) -> u64 { - self.0.iter().map(|segment| segment.len() as u64).sum() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Returns the bounding range `[min, max]` across all row IDs in this sequence, - /// or `None` if the sequence contains no values. - /// - /// This is a conservative bounding box: a value falling within the returned range - /// is not guaranteed to exist in the sequence (segments may be sparse), but any - /// value that *does* exist is guaranteed to fall within the range. This makes - /// the result suitable as a cheap pre-filter before a full scan. - pub fn row_id_range(&self) -> Option> { - let min = self - .0 - .iter() - .filter_map(|s| s.range()) - .map(|r| *r.start()) - .min()?; - let max = self - .0 - .iter() - .filter_map(|s| s.range()) - .map(|r| *r.end()) - .max()?; - Some(min..=max) - } - - /// Combines this row id sequence with another row id sequence. - pub fn extend(&mut self, other: Self) { - // If the last element of this sequence and the first element of next - // sequence are ranges, we might be able to combine them into a single - // range. - if let (Some(U64Segment::Range(range1)), Some(U64Segment::Range(range2))) = - (self.0.last(), other.0.first()) - && range1.end == range2.start - { - let new_range = U64Segment::Range(range1.start..range2.end); - self.0.pop(); - self.0.push(new_range); - self.0.extend(other.0.into_iter().skip(1)); - return; - } - // TODO: add other optimizations, such as combining two RangeWithHoles. - self.0.extend(other.0); - } - - /// Remove a set of row ids from the sequence. - pub fn delete(&mut self, row_ids: impl IntoIterator) { - // Order the row ids by position in which they appear in the sequence. - let (row_ids, offsets) = self.find_ids(row_ids); - - let capacity = self.0.capacity(); - let old_segments = std::mem::replace(&mut self.0, Vec::with_capacity(capacity)); - let mut remaining_segments = old_segments.as_slice(); - - for (segment_idx, range) in offsets { - let segments_handled = old_segments.len() - remaining_segments.len(); - let segments_to_add = segment_idx - segments_handled; - self.0 - .extend_from_slice(&remaining_segments[..segments_to_add]); - remaining_segments = &remaining_segments[segments_to_add..]; - - let segment; - (segment, remaining_segments) = remaining_segments.split_first().unwrap(); - - let segment_ids = &row_ids[range]; - self.0.push(segment.delete(segment_ids)); - } - - // Add the remaining segments. - self.0.extend_from_slice(remaining_segments); - } - - /// Delete row ids by position. - pub fn mask(&mut self, positions: impl IntoIterator) -> Result<()> { - let mut local_positions = Vec::new(); - let mut positions_iter = positions.into_iter(); - let mut curr_position = positions_iter.next(); - let mut offset = 0; - let mut cutoff = 0; - - for segment in &mut self.0 { - // Make vector of local positions - cutoff += segment.len() as u32; - while let Some(position) = curr_position { - if position >= cutoff { - break; - } - local_positions.push(position - offset); - curr_position = positions_iter.next(); - } - - if !local_positions.is_empty() { - segment.mask(&local_positions); - local_positions.clear(); - } - offset = cutoff; - } - - self.0.retain(|segment| !segment.is_empty()); - - Ok(()) - } - - /// Find the row ids in the sequence. - /// - /// Returns the row ids sorted by their appearance in the sequence. - /// Also returns the segment index and the range where that segment's - /// row id matches are found in the returned row id vector. - fn find_ids( - &self, - row_ids: impl IntoIterator, - ) -> (Vec, Vec<(usize, Range)>) { - // Often, the row ids will already be provided in the order they appear. - // So the optimal way to search will be to cycle through rather than - // restarting the search from the beginning each time. - let mut segment_iter = self.0.iter().enumerate().cycle(); - - let mut segment_matches = vec![Vec::new(); self.0.len()]; - - row_ids.into_iter().for_each(|row_id| { - let mut i = 0; - // If we've cycled through all segments, we know the row id is not in the sequence. - while i < self.0.len() { - let (segment_idx, segment) = segment_iter.next().unwrap(); - if segment.range().is_some_and(|range| range.contains(&row_id)) - && let Some(offset) = segment.position(row_id) - { - segment_matches.get_mut(segment_idx).unwrap().push(offset); - // The row id was not found it the segment. It might be in a later segment. - } - i += 1; - } - }); - for matches in &mut segment_matches { - matches.sort_unstable(); - } - - let mut offset = 0; - let segment_ranges = segment_matches - .iter() - .enumerate() - .filter(|(_, matches)| !matches.is_empty()) - .map(|(segment_idx, matches)| { - let range = offset..offset + matches.len(); - offset += matches.len(); - (segment_idx, range) - }) - .collect(); - let row_ids = segment_matches - .into_iter() - .enumerate() - .flat_map(|(segment_idx, offset)| { - offset - .into_iter() - .map(move |offset| self.0[segment_idx].get(offset).unwrap()) - }) - .collect(); - - (row_ids, segment_ranges) - } - - pub fn slice(&self, offset: usize, len: usize) -> RowIdSeqSlice<'_> { - if len == 0 { - return RowIdSeqSlice { - segments: &[], - offset_start: 0, - offset_last: 0, - }; - } - - // Find the starting position - let mut offset_start = offset; - let mut segment_offset = 0; - for segment in &self.0 { - let segment_len = segment.len(); - if offset_start < segment_len { - break; - } - offset_start -= segment_len; - segment_offset += 1; - } - - // Find the ending position - let mut offset_last = offset_start + len; - let mut segment_offset_last = segment_offset; - for segment in &self.0[segment_offset..] { - let segment_len = segment.len(); - if offset_last <= segment_len { - break; - } - offset_last -= segment_len; - segment_offset_last += 1; - } - - RowIdSeqSlice { - segments: &self.0[segment_offset..=segment_offset_last], - offset_start, - offset_last, - } - } - - /// Get the row id at the given index. - /// - /// If the index is out of bounds, this will return None. - pub fn get(&self, index: usize) -> Option { - let mut offset = 0; - for segment in &self.0 { - let segment_len = segment.len(); - if index < offset + segment_len { - return segment.get(index - offset); - } - offset += segment_len; - } - None - } - - /// Get row ids from the sequence based on the provided _sorted_ offsets - /// - /// Any out of bounds offsets will be ignored - /// - /// # Panics - /// - /// If the input selection is not sorted, this function will panic - pub fn select<'a>( - &'a self, - selection: impl Iterator + 'a, - ) -> impl Iterator + 'a { - let mut seg_iter = self.0.iter(); - let mut cur_seg = seg_iter.next(); - let mut rows_passed = 0; - let mut cur_seg_len = cur_seg.map(|seg| seg.len()).unwrap_or(0); - let mut last_index = 0; - selection.filter_map(move |index| { - if index < last_index { - panic!("Selection is not sorted"); - } - last_index = index; - - cur_seg?; - - while (index - rows_passed) >= cur_seg_len { - rows_passed += cur_seg_len; - cur_seg = seg_iter.next(); - if let Some(cur_seg) = cur_seg { - cur_seg_len = cur_seg.len(); - } else { - return None; - } - } - - Some(cur_seg.unwrap().get(index - rows_passed).unwrap()) - }) - } - - /// Given a mask of row ids, calculate the offset ranges of the row ids that are present - /// in the sequence. - /// - /// For example, given a mask that selects all even ids and a sequence that is - /// [80..85, 86..90, 14] - /// - /// this will return [0, 2, 4, 5, 7, 9] - /// because the range expands to - /// - /// [80, 81, 82, 83, 84, 86, 87, 88, 89, 14] with offsets - /// [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - /// - /// This function is useful when determining which row offsets to read from a fragment given - /// a mask. - #[instrument(level = "debug", skip_all)] - pub fn mask_to_offset_ranges(&self, mask: &RowAddrMask) -> Vec> { - let mut offset = 0; - let mut ranges = Vec::new(); - for segment in &self.0 { - match segment { - U64Segment::Range(range) => { - let mut ids = RowAddrTreeMap::from(range.clone()); - ids.mask(mask); - ranges.extend(GroupingIterator::new( - unsafe { ids.into_addr_iter() }.map(|addr| addr - range.start + offset), - )); - offset += range.end - range.start; - } - U64Segment::RangeWithHoles { range, holes } => { - let offset_start = offset; - let mut ids = RowAddrTreeMap::from(range.clone()); - offset += range.end - range.start; - for hole in holes.iter() { - if ids.remove(hole) { - offset -= 1; - } - } - ids.mask(mask); - - // Sadly we can't just subtract the offset because of the holes - let mut sorted_holes = holes.clone().into_iter().collect::>(); - sorted_holes.sort_unstable(); - let mut next_holes_iter = sorted_holes.into_iter().peekable(); - let mut holes_passed = 0; - ranges.extend(GroupingIterator::new(unsafe { ids.into_addr_iter() }.map( - |addr| { - while let Some(next_hole) = next_holes_iter.peek() { - if *next_hole < addr { - next_holes_iter.next(); - holes_passed += 1; - } else { - break; - } - } - addr - range.start + offset_start - holes_passed - }, - ))); - } - U64Segment::RangeWithBitmap { range, bitmap } => { - let mut ids = RowAddrTreeMap::from(range.clone()); - let offset_start = offset; - offset += range.end - range.start; - for (i, val) in range.clone().enumerate() { - if !bitmap.get(i) && ids.remove(val) { - offset -= 1; - } - } - ids.mask(mask); - let mut bitmap_iter = bitmap.iter(); - let mut bitmap_iter_pos = 0; - let mut holes_passed = 0; - ranges.extend(GroupingIterator::new(unsafe { ids.into_addr_iter() }.map( - |addr| { - let position_in_range = addr - range.start; - while bitmap_iter_pos < position_in_range { - if !bitmap_iter.next().unwrap() { - holes_passed += 1; - } - bitmap_iter_pos += 1; - } - offset_start + position_in_range - holes_passed - }, - ))); - } - U64Segment::SortedArray(array) | U64Segment::Array(array) => { - // TODO: Could probably optimize the sorted array case to be O(N) instead of O(N log N) - ranges.extend(GroupingIterator::new(array.iter().enumerate().filter_map( - |(off, id)| { - if mask.selected(id) { - Some(off as u64 + offset) - } else { - None - } - }, - ))); - offset += array.len() as u64; - } - } - } - ranges - } -} - -/// An iterator that groups row ids into ranges -/// -/// For example, given an input iterator of [1, 2, 3, 5, 6, 7, 10, 11, 12] -/// this will return an iterator of [(1..4), (5..8), (10..13)] -struct GroupingIterator> { - iter: I, - cur_range: Option>, -} - -impl> GroupingIterator { - fn new(iter: I) -> Self { - Self { - iter, - cur_range: None, - } - } -} - -impl> Iterator for GroupingIterator { - type Item = Range; - - fn next(&mut self) -> Option { - for id in self.iter.by_ref() { - if let Some(range) = self.cur_range.as_mut() { - if range.end == id { - range.end = id + 1; - } else { - let ret = Some(range.clone()); - self.cur_range = Some(id..id + 1); - return ret; - } - } else { - self.cur_range = Some(id..id + 1); - } - } - self.cur_range.take() - } -} - -impl From<&RowIdSequence> for RowAddrTreeMap { - fn from(row_ids: &RowIdSequence) -> Self { - let mut tree_map = Self::new(); - for segment in &row_ids.0 { - match segment { - U64Segment::Range(range) => { - tree_map.insert_range(range.clone()); - } - U64Segment::RangeWithBitmap { range, bitmap } => { - tree_map.insert_range(range.clone()); - for (i, val) in range.clone().enumerate() { - if !bitmap.get(i) { - tree_map.remove(val); - } - } - } - U64Segment::RangeWithHoles { range, holes } => { - tree_map.insert_range(range.clone()); - for hole in holes.iter() { - tree_map.remove(hole); - } - } - U64Segment::SortedArray(array) | U64Segment::Array(array) => { - for val in array.iter() { - tree_map.insert(val); - } - } - } - } - tree_map - } -} - -#[derive(Debug)] -pub struct RowIdSeqSlice<'a> { - /// Current slice of the segments we cover - segments: &'a [U64Segment], - /// Offset into the first segment to start iterating from - offset_start: usize, - /// Offset into the last segment to stop iterating at - offset_last: usize, -} - -impl RowIdSeqSlice<'_> { - pub fn iter(&self) -> impl Iterator + '_ { - let mut known_size = self.segments.iter().map(|segment| segment.len()).sum(); - known_size -= self.offset_start; - known_size -= self.segments.last().map(|s| s.len()).unwrap_or_default() - self.offset_last; - - let end = self.segments.len().saturating_sub(1); - self.segments - .iter() - .enumerate() - .flat_map(move |(i, segment)| { - match i { - 0 if self.segments.len() == 1 => { - let len = self.offset_last - self.offset_start; - // TODO: Optimize this so we don't have to use skip - // (take is probably fine though.) - Box::new(segment.iter().skip(self.offset_start).take(len)) - as Box> - } - 0 => Box::new(segment.iter().skip(self.offset_start)), - i if i == end => Box::new(segment.iter().take(self.offset_last)), - _ => Box::new(segment.iter()), - } - }) - .exact_size(known_size) - } -} - -/// Re-chunk a sequences of row ids into chunks of a given size. -/// -/// The sequences may less than chunk sizes, because the sequences only -/// contains the row ids that we want to keep, they come from the updates records. -/// But the chunk sizes are based on the fragment physical rows(may contain inserted records). -/// So if the sequences are smaller than the chunk sizes, we need to -/// assign the incremental row ids in the further step. This behavior is controlled by the -/// `allow_incomplete` parameter. -/// -/// # Errors -/// -/// If `allow_incomplete` is false, will return an error if the sum of the chunk sizes -/// is not equal to the total number of row ids in the sequences. -pub fn rechunk_sequences( - sequences: impl IntoIterator, - chunk_sizes: impl IntoIterator, - allow_incomplete: bool, -) -> Result> { - // TODO: return an iterator. (with a good size hint?) - let chunk_sizes_vec: Vec = chunk_sizes.into_iter().collect(); - let total_chunks = chunk_sizes_vec.len(); - let mut chunked_sequences = Vec::with_capacity(total_chunks); - let mut segment_iter = sequences - .into_iter() - .flat_map(|sequence| sequence.0.into_iter()) - .peekable(); - - let too_few_segments_error = |chunk_index: usize, expected_chunk_size: u64, remaining: u64| { - Error::invalid_input(format!( - "Got too few segments for chunk {}. Expected chunk size: {}, remaining needed: {}", - chunk_index, expected_chunk_size, remaining - )) - }; - - let too_many_segments_error = |processed_chunks: usize, total_chunk_sizes: usize| { - Error::invalid_input(format!( - "Got too many segments for the provided chunk lengths. Processed {} chunks out of {} expected", - processed_chunks, total_chunk_sizes - )) - }; - - let mut segment_offset = 0_u64; - - for (chunk_index, chunk_size) in chunk_sizes_vec.iter().enumerate() { - let chunk_size = *chunk_size; - let mut sequence = RowIdSequence(Vec::new()); - let mut remaining = chunk_size; - - while remaining > 0 { - let remaining_in_segment = segment_iter - .peek() - .map_or(0, |segment| segment.len() as u64 - segment_offset); - - // Step 1: Handle segment remaining to be empty(also empty seg) - skip and continue - if remaining_in_segment == 0 { - if segment_iter.next().is_some() { - segment_offset = 0; - continue; - } else { - // No more segments available - if allow_incomplete { - break; - } else { - return Err(too_few_segments_error(chunk_index, chunk_size, remaining)); - } - } - } - - // Step 2: Handle still remaining segment based on size comparison - match remaining_in_segment.cmp(&remaining) { - std::cmp::Ordering::Greater => { - // Segment is larger than remaining space - slice it - let segment = segment_iter - .peek() - .ok_or_else(|| too_few_segments_error(chunk_index, chunk_size, remaining))? - .slice(segment_offset as usize, remaining as usize); - sequence.extend(RowIdSequence(vec![segment])); - segment_offset += remaining; - remaining = 0; - } - std::cmp::Ordering::Equal | std::cmp::Ordering::Less => { - // UNIFIED HANDLING: Both equal and less cases subtract from remaining - // Equal case: remaining -= remaining_in_segment (remaining becomes 0) - // Less case: remaining -= remaining_in_segment (remaining becomes positive) - let segment = segment_iter - .next() - .ok_or_else(|| too_few_segments_error(chunk_index, chunk_size, remaining))? - .slice(segment_offset as usize, remaining_in_segment as usize); - sequence.extend(RowIdSequence(vec![segment])); - segment_offset = 0; - remaining -= remaining_in_segment; - } - } - } - - chunked_sequences.push(sequence); - } - - if segment_iter.peek().is_some() { - return Err(too_many_segments_error( - chunked_sequences.len(), - total_chunks, - )); - } - - Ok(chunked_sequences) -} - -/// Selects the row ids from a sequence based on the provided offsets. -pub fn select_row_ids<'a>( - sequence: &'a RowIdSequence, - offsets: &'a ReadBatchParams, -) -> Result> { - let out_of_bounds_err = |offset: u32| { - Error::invalid_input(format!( - "Index out of bounds: {} for sequence of length {}", - offset, - sequence.len() - )) - }; - - match offsets { - // TODO: Optimize this if indices are sorted, which is a common case. - ReadBatchParams::Indices(indices) => indices - .values() - .iter() - .map(|index| { - sequence - .get(*index as usize) - .ok_or_else(|| out_of_bounds_err(*index)) - }) - .collect(), - ReadBatchParams::Range(range) => { - if range.end > sequence.len() as usize { - return Err(out_of_bounds_err(range.end as u32)); - } - let sequence = sequence.slice(range.start, range.end - range.start); - Ok(sequence.iter().collect()) - } - ReadBatchParams::Ranges(ranges) => { - let num_rows = ranges - .iter() - .map(|r| (r.end - r.start) as usize) - .sum::(); - let mut result = Vec::with_capacity(num_rows); - for range in ranges.as_ref() { - if range.end > sequence.len() { - return Err(out_of_bounds_err(range.end as u32)); - } - let sequence = - sequence.slice(range.start as usize, (range.end - range.start) as usize); - result.extend(sequence.iter()); - } - Ok(result) - } - - ReadBatchParams::RangeFull => Ok(sequence.iter().collect()), - ReadBatchParams::RangeTo(to) => { - if to.end > sequence.len() as usize { - return Err(out_of_bounds_err(to.end as u32)); - } - let len = to.end; - let sequence = sequence.slice(0, len); - Ok(sequence.iter().collect()) - } - ReadBatchParams::RangeFrom(from) => { - let sequence = sequence.slice(from.start, sequence.len() as usize - from.start); - Ok(sequence.iter().collect()) - } - } -} - -#[cfg(test)] -mod test { - use super::*; - - use pretty_assertions::assert_eq; - use test::bitmap::Bitmap; - - #[test] - fn test_row_id_sequence_from_range() { - let sequence = RowIdSequence::from(0..10); - assert_eq!(sequence.len(), 10); - assert_eq!(sequence.is_empty(), false); - - let iter = sequence.iter(); - assert_eq!(iter.collect::>(), (0..10).collect::>()); - } - - #[test] - fn test_row_id_sequence_extend() { - let mut sequence = RowIdSequence::from(0..10); - sequence.extend(RowIdSequence::from(10..20)); - assert_eq!(sequence.0, vec![U64Segment::Range(0..20)]); - - let mut sequence = RowIdSequence::from(0..10); - sequence.extend(RowIdSequence::from(20..30)); - assert_eq!( - sequence.0, - vec![U64Segment::Range(0..10), U64Segment::Range(20..30)] - ); - } - - #[test] - fn test_row_id_sequence_delete() { - let mut sequence = RowIdSequence::from(0..10); - sequence.delete(vec![1, 3, 5, 7, 9]); - let mut expected_bitmap = Bitmap::new_empty(9); - for i in [0, 2, 4, 6, 8] { - expected_bitmap.set(i as usize); - } - assert_eq!( - sequence.0, - vec![U64Segment::RangeWithBitmap { - range: 0..9, - bitmap: expected_bitmap - },] - ); - - let mut sequence = RowIdSequence::from(0..10); - sequence.extend(RowIdSequence::from(12..20)); - sequence.delete(vec![0, 9, 10, 11, 12, 13]); - assert_eq!( - sequence.0, - vec![U64Segment::Range(1..9), U64Segment::Range(14..20),] - ); - - let mut sequence = RowIdSequence::from(0..10); - sequence.delete(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - assert_eq!(sequence.0, vec![U64Segment::Range(0..0)]); - } - - #[test] - fn test_row_id_slice() { - // The type of sequence isn't that relevant to the implementation, so - // we can just have a single one with all the segment types. - let sequence = RowIdSequence(vec![ - U64Segment::Range(30..35), // 5 - U64Segment::RangeWithHoles { - // 8 - range: 50..60, - holes: vec![53, 54].into(), - }, - U64Segment::SortedArray(vec![7, 9].into()), // 2 - U64Segment::RangeWithBitmap { - range: 0..5, - bitmap: [true, false, true, false, true].as_slice().into(), - }, - U64Segment::Array(vec![35, 39].into()), - U64Segment::Range(40..50), - ]); - - // All possible offsets and lengths - for offset in 0..sequence.len() as usize { - for len in 0..sequence.len() as usize { - if offset + len > sequence.len() as usize { - continue; - } - let slice = sequence.slice(offset, len); - - let actual = slice.iter().collect::>(); - let expected = sequence.iter().skip(offset).take(len).collect::>(); - assert_eq!( - actual, expected, - "Failed for offset {} and len {}", - offset, len - ); - - let (claimed_size, claimed_max) = slice.iter().size_hint(); - assert_eq!(claimed_max, Some(claimed_size)); // Exact size hint - assert_eq!(claimed_size, actual.len()); // Correct size hint - } - } - } - - #[test] - fn test_row_id_slice_empty() { - let sequence = RowIdSequence::from(0..10); - let slice = sequence.slice(10, 0); - assert_eq!(slice.iter().collect::>(), Vec::::new()); - } - - #[test] - fn test_row_id_sequence_rechunk() { - fn assert_rechunked( - input: Vec, - chunk_sizes: Vec, - expected: Vec, - ) { - let chunked = rechunk_sequences(input, chunk_sizes, false).unwrap(); - assert_eq!(chunked, expected); - } - - // Small pieces to larger ones - let many_segments = vec![ - RowIdSequence(vec![U64Segment::Range(0..5), U64Segment::Range(35..40)]), - RowIdSequence::from(10..18), - RowIdSequence::from(18..28), - RowIdSequence::from(28..30), - ]; - let fewer_segments = vec![ - RowIdSequence(vec![U64Segment::Range(0..5), U64Segment::Range(35..40)]), - RowIdSequence::from(10..30), - ]; - assert_rechunked( - many_segments.clone(), - fewer_segments.iter().map(|seq| seq.len()).collect(), - fewer_segments.clone(), - ); - - // Large pieces to smaller ones - assert_rechunked( - fewer_segments, - many_segments.iter().map(|seq| seq.len()).collect(), - many_segments.clone(), - ); - - // Equal pieces - assert_rechunked( - many_segments.clone(), - many_segments.iter().map(|seq| seq.len()).collect(), - many_segments.clone(), - ); - - // Too few segments -> error - let result = rechunk_sequences(many_segments.clone(), vec![100], false); - assert!(result.is_err()); - - // Too many segments -> error - let result = rechunk_sequences(many_segments, vec![5], false); - assert!(result.is_err()); - } - - #[test] - fn test_select_row_ids() { - // All forms of offsets - let offsets = [ - ReadBatchParams::Indices(vec![1, 3, 9, 5, 7, 6].into()), - ReadBatchParams::Range(2..8), - ReadBatchParams::RangeFull, - ReadBatchParams::RangeTo(..5), - ReadBatchParams::RangeFrom(5..), - ReadBatchParams::Ranges(vec![2..3, 5..10].into()), - ]; - - // Sequences with all segment types. These have at least 10 elements, - // so they are valid for all the above offsets. - let sequences = [ - RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::RangeWithHoles { - range: 50..60, - holes: vec![53, 54].into(), - }, - U64Segment::SortedArray(vec![7, 9].into()), - ]), - RowIdSequence(vec![ - U64Segment::RangeWithBitmap { - range: 0..5, - bitmap: [true, false, true, false, true].as_slice().into(), - }, - U64Segment::Array(vec![30, 20, 10].into()), - U64Segment::Range(40..50), - ]), - ]; - - for params in offsets { - for sequence in &sequences { - let row_ids = select_row_ids(sequence, ¶ms).unwrap(); - let flat_sequence = sequence.iter().collect::>(); - - // Transform params into bounded ones - let selection: Vec = match ¶ms { - ReadBatchParams::RangeFull => (0..flat_sequence.len()).collect(), - ReadBatchParams::RangeTo(to) => (0..to.end).collect(), - ReadBatchParams::RangeFrom(from) => (from.start..flat_sequence.len()).collect(), - ReadBatchParams::Range(range) => range.clone().collect(), - ReadBatchParams::Ranges(ranges) => ranges - .iter() - .flat_map(|r| r.start as usize..r.end as usize) - .collect(), - ReadBatchParams::Indices(indices) => { - indices.values().iter().map(|i| *i as usize).collect() - } - }; - - let expected = selection - .into_iter() - .map(|i| flat_sequence[i]) - .collect::>(); - assert_eq!( - row_ids, expected, - "Failed for params {:?} on the sequence {:?}", - ¶ms, sequence - ); - } - } - } - - #[test] - fn test_select_row_ids_out_of_bounds() { - let offsets = [ - ReadBatchParams::Indices(vec![1, 1000, 4].into()), - ReadBatchParams::Range(2..1000), - ReadBatchParams::RangeTo(..1000), - ]; - - let sequence = RowIdSequence::from(0..10); - - for params in offsets { - let result = select_row_ids(&sequence, ¶ms); - assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), Error::InvalidInput { .. })); - } - } - - #[test] - fn test_row_id_sequence_to_treemap() { - let sequence = RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::RangeWithHoles { - range: 50..60, - holes: vec![53, 54].into(), - }, - U64Segment::SortedArray(vec![7, 9].into()), - U64Segment::RangeWithBitmap { - range: 10..15, - bitmap: [true, false, true, false, true].as_slice().into(), - }, - U64Segment::Array(vec![35, 39].into()), - U64Segment::Range(40..50), - ]); - - let tree_map = RowAddrTreeMap::from(&sequence); - let expected = vec![ - 0, 1, 2, 3, 4, 7, 9, 10, 12, 14, 35, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 55, 56, 57, 58, 59, - ] - .into_iter() - .collect::(); - assert_eq!(tree_map, expected); - } - - #[test] - fn test_row_addr_mask() { - // 0, 1, 2, 3, 4 - // 50, 51, 52, 55, 56, 57, 58, 59 - // 7, 9 - // 10, 12, 14 - // 35, 39 - let sequence = RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::RangeWithHoles { - range: 50..60, - holes: vec![53, 54].into(), - }, - U64Segment::SortedArray(vec![7, 9].into()), - U64Segment::RangeWithBitmap { - range: 10..15, - bitmap: [true, false, true, false, true].as_slice().into(), - }, - U64Segment::Array(vec![35, 39].into()), - ]); - - // Masking one in each segment - let values_to_remove = [4, 55, 7, 12, 39]; - let positions_to_remove = sequence - .iter() - .enumerate() - .filter_map(|(i, val)| { - if values_to_remove.contains(&val) { - Some(i as u32) - } else { - None - } - }) - .collect::>(); - let mut sequence = sequence; - sequence.mask(positions_to_remove).unwrap(); - let expected = RowIdSequence(vec![ - U64Segment::Range(0..4), - U64Segment::RangeWithBitmap { - range: 50..60, - bitmap: [ - true, true, true, false, false, false, true, true, true, true, - ] - .as_slice() - .into(), - }, - U64Segment::Range(9..10), - U64Segment::RangeWithBitmap { - range: 10..15, - bitmap: [true, false, false, false, true].as_slice().into(), - }, - U64Segment::Array(vec![35].into()), - ]); - assert_eq!(sequence, expected); - } - - #[test] - fn test_row_addr_mask_everything() { - let mut sequence = RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::SortedArray(vec![7, 9].into()), - ]); - sequence.mask(0..sequence.len() as u32).unwrap(); - let expected = RowIdSequence(vec![]); - assert_eq!(sequence, expected); - } - - #[test] - fn test_selection() { - let sequence = RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::Range(10..15), - U64Segment::Range(20..25), - ]); - let selection = sequence.select(vec![2, 4, 13, 14, 57].into_iter()); - assert_eq!(selection.collect::>(), vec![2, 4, 23, 24]); - } - - #[test] - #[should_panic] - fn test_selection_unsorted() { - let sequence = RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::Range(10..15), - U64Segment::Range(20..25), - ]); - let _ = sequence - .select(vec![2, 4, 3].into_iter()) - .collect::>(); - } - - #[test] - fn test_mask_to_offset_ranges() { - // Tests with a simple range segment - let sequence = RowIdSequence(vec![U64Segment::Range(0..10)]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[0, 2, 4, 6, 8])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..1, 2..3, 4..5, 6..7, 8..9]); - - let sequence = RowIdSequence(vec![U64Segment::Range(40..60)]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[54])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![14..15]); - - let sequence = RowIdSequence(vec![U64Segment::Range(40..60)]); - let mask = RowAddrMask::from_block(RowAddrTreeMap::from_iter(&[54])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..14, 15..20]); - - // Test with a range segment with holes - // 0, 1, 3, 4, 5, 7, 8, 9 - let sequence = RowIdSequence(vec![U64Segment::RangeWithHoles { - range: 0..10, - holes: vec![2, 6].into(), - }]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[0, 2, 4, 6, 8])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..1, 3..4, 6..7]); - - let sequence = RowIdSequence(vec![U64Segment::RangeWithHoles { - range: 40..60, - holes: vec![47, 43].into(), - }]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[44])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![3..4]); - - let sequence = RowIdSequence(vec![U64Segment::RangeWithHoles { - range: 40..60, - holes: vec![47, 43].into(), - }]); - let mask = RowAddrMask::from_block(RowAddrTreeMap::from_iter(&[44])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..3, 4..18]); - - // Test with a range segment with bitmap - // 0, 1, 4, 5, 6, 7 - let sequence = RowIdSequence(vec![U64Segment::RangeWithBitmap { - range: 0..10, - bitmap: [ - true, true, false, false, true, true, true, true, false, false, - ] - .as_slice() - .into(), - }]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[0, 2, 4, 6, 8])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..1, 2..3, 4..5]); - - let sequence = RowIdSequence(vec![U64Segment::RangeWithBitmap { - range: 40..45, - bitmap: [true, true, false, false, true].as_slice().into(), - }]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[44])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![2..3]); - - let sequence = RowIdSequence(vec![U64Segment::RangeWithBitmap { - range: 40..45, - bitmap: [true, true, false, false, true].as_slice().into(), - }]); - let mask = RowAddrMask::from_block(RowAddrTreeMap::from_iter(&[44])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..2]); - - // Test with a sorted array segment - let sequence = RowIdSequence(vec![U64Segment::SortedArray(vec![0, 2, 4, 6, 8].into())]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[0, 6, 8])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..1, 3..5]); - - let sequence = RowIdSequence(vec![U64Segment::Array(vec![8, 2, 6, 0, 4].into())]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[0, 6, 8])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..1, 2..4]); - - // Test with multiple segments - // 0, 1, 2, 3, 4, 100, 101, 102, 104, 44, 46, 78 - // *, -, *, -, -, ***, ---, ---, ***, --, **, -- - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 - let sequence = RowIdSequence(vec![ - U64Segment::Range(0..5), - U64Segment::RangeWithHoles { - range: 100..105, - holes: vec![103].into(), - }, - U64Segment::SortedArray(vec![44, 46, 78].into()), - ]); - let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(&[0, 2, 46, 100, 104])); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..1, 2..3, 5..6, 8..9, 10..11]); - - // Test with empty mask (should select everything) - let sequence = RowIdSequence(vec![U64Segment::Range(0..10)]); - let mask = RowAddrMask::default(); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![0..10]); - - // Test with allow nothing mask - let sequence = RowIdSequence(vec![U64Segment::Range(0..10)]); - let mask = RowAddrMask::allow_nothing(); - let ranges = sequence.mask_to_offset_ranges(&mask); - assert_eq!(ranges, vec![]); - } - - #[test] - fn test_row_id_sequence_rechunk_with_empty_segments() { - // equal case (segment exactly fills remaining space) - let input_sequences = vec![ - RowIdSequence::from(0..2), // [0, 1] - 2 elements - RowIdSequence::from(20..23), // [20, 21, 22] - 3 elements - ]; - let chunk_sizes = vec![2, 3]; // First chunk wants 2, second wants 3 - - let result = rechunk_sequences(input_sequences, chunk_sizes, false).unwrap(); - assert_eq!(result.len(), 2); - assert_eq!(result[0].len(), 2); - assert_eq!(result[1].len(), 3); - - let first_chunk: Vec = result[0].iter().collect(); - let second_chunk: Vec = result[1].iter().collect(); - assert_eq!(first_chunk, vec![0, 1]); - assert_eq!(second_chunk, vec![20, 21, 22]); - - // less case (segment smaller than remaining space) - let input_sequences = vec![ - RowIdSequence::from(0..2), // [0, 1] - 2 elements (less than remaining) - RowIdSequence::from(20..21), // [20] - 1 element (less than remaining) - RowIdSequence::from(30..32), // [30, 31] - 2 elements (exactly fills remaining) - ]; - let chunk_sizes = vec![5]; // Request 5 elements, have exactly 5 - - let result = rechunk_sequences(input_sequences, chunk_sizes, false).unwrap(); - assert_eq!(result.len(), 1); - assert_eq!(result[0].len(), 5); - - let elements: Vec = result[0].iter().collect(); - assert_eq!(elements, vec![0, 1, 20, 30, 31]); - - // empty segment in the middle - let input_sequences = vec![ - RowIdSequence::from(0..2), // [0, 1] - 2 elements - RowIdSequence::from(10..10), // [] - 0 elements (empty) - RowIdSequence::from(20..22), // [20, 21] - 2 elements - ]; - let chunk_sizes = vec![3, 1]; - let result = rechunk_sequences(input_sequences, chunk_sizes, false).unwrap(); - - assert_eq!(result.len(), 2); - assert_eq!(result[0].len(), 3); - assert_eq!(result[1].len(), 1); - - let first_chunk_elements: Vec = result[0].iter().collect(); - let second_chunk_elements: Vec = result[1].iter().collect(); - assert_eq!(first_chunk_elements, vec![0, 1, 20]); - assert_eq!(second_chunk_elements, vec![21]); - - // multiple empty segments - let input_sequences = vec![ - RowIdSequence::from(0..1), // [0] - 1 element - RowIdSequence::from(10..10), // [] - 0 elements (empty) - RowIdSequence::from(20..20), // [] - 0 elements (empty) - RowIdSequence::from(30..32), // [30, 31] - 2 elements - ]; - let chunk_sizes = vec![3]; - let result = rechunk_sequences(input_sequences, chunk_sizes, false).unwrap(); - - assert_eq!(result.len(), 1); - assert_eq!(result[0].len(), 3); - - let elements: Vec = result[0].iter().collect(); - assert_eq!(elements, vec![0, 30, 31]); - - // empty segment at chunk boundary - let input_sequences = vec![ - RowIdSequence::from(0..3), // [0, 1, 2] - 3 elements (exactly fills first chunk) - RowIdSequence::from(10..10), // [] - 0 elements (empty, at boundary) - RowIdSequence::from(20..22), // [20, 21] - 2 elements (for second chunk) - ]; - let chunk_sizes = vec![3, 2]; - let result = rechunk_sequences(input_sequences, chunk_sizes, false).unwrap(); - - assert_eq!(result.len(), 2); - assert_eq!(result[0].len(), 3); - assert_eq!(result[1].len(), 2); - - let first_chunk_elements: Vec = result[0].iter().collect(); - let second_chunk_elements: Vec = result[1].iter().collect(); - assert_eq!(first_chunk_elements, vec![0, 1, 2]); - assert_eq!(second_chunk_elements, vec![20, 21]); - - // empty segments with allow_incomplete = true - let input_sequences = vec![ - RowIdSequence::from(0..2), // [0, 1] - 2 elements - RowIdSequence::from(10..10), // [] - 0 elements (empty) - ]; - let chunk_sizes = vec![5]; // Request more than available - let result = rechunk_sequences(input_sequences, chunk_sizes, true).unwrap(); - - assert_eq!(result.len(), 1); - assert_eq!(result[0].len(), 2); - - let elements: Vec = result[0].iter().collect(); - assert_eq!(elements, vec![0, 1]); - } - - #[test] - fn test_row_id_range_empty() { - let seq = RowIdSequence::from(0u64..0); - assert_eq!(seq.row_id_range(), None); - } - - #[test] - fn test_row_id_range_single_contiguous() { - let seq = RowIdSequence::from(10u64..20); - assert_eq!(seq.row_id_range(), Some(10..=19)); - } - - #[test] - fn test_row_id_range_unsorted_array() { - // Array variant: range() returns min..=max as bounding box - let seq = RowIdSequence::from([50u64, 10, 30].as_slice()); - let r = seq.row_id_range().unwrap(); - assert!(*r.start() <= 10); - assert!(*r.end() >= 50); - } - - #[test] - fn test_row_id_range_multi_segment() { - // Two disjoint ranges; bounding box should span both - let mut seq = RowIdSequence::from(0u64..5); - seq.extend(RowIdSequence::from(100u64..105)); - let r = seq.row_id_range().unwrap(); - assert_eq!(*r.start(), 0); - assert_eq!(*r.end(), 104); - } -} diff --git a/vendor/lance-table/src/rowids/bitmap.rs b/vendor/lance-table/src/rowids/bitmap.rs deleted file mode 100644 index 9001c04c..00000000 --- a/vendor/lance-table/src/rowids/bitmap.rs +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use deepsize::DeepSizeOf; - -#[derive(PartialEq, Eq, Clone, DeepSizeOf)] -pub struct Bitmap { - pub data: Vec, - pub len: usize, -} - -impl std::fmt::Debug for Bitmap { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "Bitmap {{ data: ")?; - for i in 0..self.len { - write!(f, "{}", if self.get(i) { "1" } else { "0" })?; - } - write!(f, ", len: {} }}", self.len) - } -} - -impl Bitmap { - pub fn new_empty(len: usize) -> Self { - let data = vec![0; len.div_ceil(8)]; - Self { data, len } - } - - pub fn new_full(len: usize) -> Self { - let mut data = vec![0xff; len.div_ceil(8)]; - // Zero past the end of len - let remainder = len % 8; - if remainder != 0 { - let last_byte = data.last_mut().unwrap(); - let bits_to_clear = 8 - remainder; - for offset_from_end in 0..bits_to_clear { - let i = 7 - offset_from_end; - *last_byte &= !(1 << i); - } - } - Self { data, len } - } - - pub fn set(&mut self, i: usize) { - self.data[i / 8] |= 1 << (i % 8); - } - - pub fn clear(&mut self, i: usize) { - self.data[i / 8] &= !(1 << (i % 8)); - } - - pub fn get(&self, i: usize) -> bool { - self.data[i / 8] & (1 << (i % 8)) != 0 - } - - pub fn len(&self) -> usize { - self.len - } - - pub fn slice(&self, start: usize, len: usize) -> BitmapSlice<'_> { - BitmapSlice { - bitmap: self, - start, - len, - } - } - - pub fn count_ones(&self) -> usize { - self.data.iter().map(|&x| x.count_ones() as usize).sum() - } - - pub fn count_zeros(&self) -> usize { - self.len - self.count_ones() - } - - pub fn iter(&self) -> impl Iterator + '_ { - self.data - .iter() - .flat_map(|&x| (0..8).map(move |i| x & (1 << i) != 0)) - .take(self.len) - } -} - -impl From<&[bool]> for Bitmap { - fn from(slice: &[bool]) -> Self { - let mut bitmap = Self::new_empty(slice.len()); - for (i, &b) in slice.iter().enumerate() { - if b { - bitmap.set(i); - } - } - bitmap - } -} - -// Make a slice of bitmap -pub struct BitmapSlice<'a> { - bitmap: &'a Bitmap, - start: usize, - len: usize, -} - -impl BitmapSlice<'_> { - pub fn count_ones(&self) -> usize { - if self.len == 0 { - return 0; - } - let first_byte = self.start / 8; - let last_byte = (self.start + self.len - 1) / 8; - if first_byte == last_byte { - let byte = self.bitmap.data[first_byte]; - let mut count = 0; - for i in self.start % 8..((self.start + self.len - 1) % 8 + 1) { - if byte & (1 << i) != 0 { - count += 1; - } - } - count - } else { - let mut count = 0; - // Handle first byte - for i in self.start % 8..8 { - if self.bitmap.data[first_byte] & (1 << i) != 0 { - count += 1; - } - } - - // Handle last bytes - for i in 0..((self.start + self.len - 1) % 8 + 1) { - if self.bitmap.data[last_byte] & (1 << i) != 0 { - count += 1; - } - } - - // Middle bytes can just use count_ones - count += self.bitmap.data[first_byte + 1..last_byte] - .iter() - .map(|&x| x.count_ones() as usize) - .sum::(); - count - } - } - - pub fn count_zeros(&self) -> usize { - self.len - self.count_ones() - } -} - -impl From> for Bitmap { - fn from(slice: BitmapSlice) -> Self { - let mut bitmap = Self::new_empty(slice.len); - for i in 0..slice.len { - if slice.bitmap.get(slice.start + i) { - bitmap.set(i); - } - } - bitmap - } -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prop_assert_eq; - - #[test] - fn test_bitmap() { - let mut bitmap = Bitmap::new_empty(10); - assert_eq!(bitmap.len(), 10); - assert_eq!(bitmap.count_ones(), 0); - - bitmap.set(0); - bitmap.set(1); - bitmap.set(4); - bitmap.set(5); - bitmap.set(9); - assert_eq!(bitmap.count_ones(), 5); - assert_eq!( - format!("{:?}", bitmap), - "Bitmap { data: 1100110001, len: 10 }" - ); - - bitmap.clear(1); - bitmap.clear(4); - assert_eq!(bitmap.count_ones(), 3); - assert_eq!( - format!("{:?}", bitmap), - "Bitmap { data: 1000010001, len: 10 }" - ); - - let bitmap_slice = bitmap.slice(5, 5); - assert_eq!(bitmap_slice.count_ones(), 2); - } - - #[test] - fn test_equality() { - for len in 48..56 { - let mut bitmap1 = Bitmap::new_empty(len); - for i in 0..len { - if i % 2 == 0 { - bitmap1.set(i); - } - } - - let mut bitmap2 = Bitmap::new_full(len); - for i in 0..len { - if i % 2 == 1 { - bitmap2.clear(i); - } - } - - assert_eq!(bitmap1, bitmap2); - } - } - - proptest::proptest! { - #[test] - fn test_bitmap_slice( - values in proptest::collection::vec(proptest::bool::ANY, 0..100), - mut start in 0..100usize, - mut len in 0..100usize, - ) { - if start > values.len() { - start = values.len(); - } - if len > values.len() - start { - len = values.len() - start; - } - - let bitmap = Bitmap::from(values.as_slice()); - let slice = bitmap.slice(start, len); - let values_slice = values[start..(start + len)].to_vec(); - - prop_assert_eq!(slice.count_ones(), values_slice.iter().filter(|&&x| x).count()); - } - } - - #[test] - fn test_bitmap_iter_empty() { - let bitmap = Bitmap::new_empty(10); - let values: Vec = bitmap.iter().collect(); - assert_eq!(values, vec![false; 10]); - } - - #[test] - fn test_bitmap_iter_full() { - let bitmap = Bitmap::new_full(10); - let values: Vec = bitmap.iter().collect(); - assert_eq!(values, vec![true; 10]); - } - - #[test] - fn test_bitmap_iter_partial() { - let mut bitmap = Bitmap::new_empty(10); - bitmap.set(0); - bitmap.set(3); - bitmap.set(7); - bitmap.set(9); - - let values: Vec = bitmap.iter().collect(); - let expected = vec![ - true, // 0 - false, // 1 - false, // 2 - true, // 3 - false, // 4 - false, // 5 - false, // 6 - true, // 7 - false, // 8 - true, // 9 - ]; - assert_eq!(values, expected); - } - - #[test] - fn test_bitmap_iter_edge_cases() { - // Test with length that's not a multiple of 8 - let mut bitmap = Bitmap::new_empty(15); - bitmap.set(0); - bitmap.set(7); - bitmap.set(14); - - let values: Vec = bitmap.iter().collect(); - let expected = vec![ - true, // 0 - false, // 1 - false, // 2 - false, // 3 - false, // 4 - false, // 5 - false, // 6 - true, // 7 - false, // 8 - false, // 9 - false, // 10 - false, // 11 - false, // 12 - false, // 13 - true, // 14 - ]; - assert_eq!(values, expected); - } - - proptest::proptest! { - #[test] - fn test_bitmap_iter_property( - values in proptest::collection::vec(proptest::bool::ANY, 0..100) - ) { - let bitmap = Bitmap::from(values.as_slice()); - let iter_values: Vec = bitmap.iter().collect(); - assert_eq!(iter_values, values); - } - } -} diff --git a/vendor/lance-table/src/rowids/encoded_array.rs b/vendor/lance-table/src/rowids/encoded_array.rs deleted file mode 100644 index 06614765..00000000 --- a/vendor/lance-table/src/rowids/encoded_array.rs +++ /dev/null @@ -1,400 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::ops::Range; - -use deepsize::DeepSizeOf; - -/// Encoded array of u64 values. -/// -/// This is a internal data type used as part of row id indices. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] -pub enum EncodedU64Array { - /// u64 values represented as u16 offset from a base value. - /// - /// Useful when the min and max value are within u16 range (0..65535). - /// Only space saving when there are more than 2 values. - U16 { base: u64, offsets: Vec }, - /// u64 values represented as u32 offset from a base value. - /// - /// Useful when the min and max value are within u32 range (0..~4 billion). - U32 { base: u64, offsets: Vec }, - /// Just a plain vector of u64 values. - /// - /// For when the values cover a wide range. - U64(Vec), -} - -impl EncodedU64Array { - pub fn len(&self) -> usize { - match self { - Self::U16 { offsets, .. } => offsets.len(), - Self::U32 { offsets, .. } => offsets.len(), - Self::U64(values) => values.len(), - } - } - - pub fn iter(&self) -> Box + '_> { - match self { - Self::U16 { base, offsets } => { - Box::new(offsets.iter().cloned().map(move |o| base + o as u64)) - } - Self::U32 { base, offsets } => { - Box::new(offsets.iter().cloned().map(move |o| base + o as u64)) - } - Self::U64(values) => Box::new(values.iter().cloned()), - } - } - - pub fn get(&self, i: usize) -> Option { - match self { - Self::U16 { base, offsets } => { - if i < offsets.len() { - Some(*base + offsets[i] as u64) - } else { - None - } - } - Self::U32 { base, offsets } => { - if i < offsets.len() { - Some(*base + offsets[i] as u64) - } else { - None - } - } - Self::U64(values) => values.get(i).copied(), - } - } - - pub fn min(&self) -> Option { - match self { - Self::U16 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base) - } - } - Self::U32 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base) - } - } - Self::U64(values) => values.iter().copied().min(), - } - } - - pub fn max(&self) -> Option { - match self { - Self::U16 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base + offsets.iter().copied().max().unwrap() as u64) - } - } - Self::U32 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base + offsets.iter().copied().max().unwrap() as u64) - } - } - Self::U64(values) => values.iter().copied().max(), - } - } - - pub fn first(&self) -> Option { - match self { - Self::U16 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base + *offsets.first().unwrap() as u64) - } - } - Self::U32 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base + *offsets.first().unwrap() as u64) - } - } - Self::U64(values) => values.first().copied(), - } - } - - pub fn last(&self) -> Option { - match self { - Self::U16 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base + *offsets.last().unwrap() as u64) - } - } - Self::U32 { base, offsets } => { - if offsets.is_empty() { - None - } else { - Some(*base + *offsets.last().unwrap() as u64) - } - } - Self::U64(values) => values.last().copied(), - } - } - - pub fn binary_search(&self, val: u64) -> std::result::Result { - match self { - Self::U16 { base, offsets } => match val.checked_sub(*base) { - None => Err(0), - Some(val) => { - if val > u16::MAX as u64 { - return Err(offsets.len()); - } - let u16 = val as u16; - offsets.binary_search(&u16) - } - }, - Self::U32 { base, offsets } => match val.checked_sub(*base) { - None => Err(0), - Some(val) => { - if val > u32::MAX as u64 { - return Err(offsets.len()); - } - let u32 = val as u32; - offsets.binary_search(&u32) - } - }, - Self::U64(values) => values.binary_search(&val), - } - } - - pub fn slice(&self, offset: usize, len: usize) -> Self { - match self { - Self::U16 { base, offsets } => offsets[offset..(offset + len)] - .iter() - .map(|o| *base + *o as u64) - .collect(), - Self::U32 { base, offsets } => offsets[offset..(offset + len)] - .iter() - .map(|o| *base + *o as u64) - .collect(), - Self::U64(values) => { - let values = values[offset..(offset + len)].to_vec(); - Self::U64(values) - } - } - } -} - -impl From> for EncodedU64Array { - fn from(values: Vec) -> Self { - let min = values.iter().copied().min().unwrap_or(0); - let max = values.iter().copied().max().unwrap_or(0); - let range = max - min; - if values.is_empty() { - Self::U64(Vec::new()) - } else if range <= u16::MAX as u64 { - let base = min; - let offsets = values.iter().map(|v| (*v - base) as u16).collect(); - Self::U16 { base, offsets } - } else if range <= u32::MAX as u64 { - let base = min; - let offsets = values.iter().map(|v| (*v - base) as u32).collect(); - Self::U32 { base, offsets } - } else { - Self::U64(values) - } - } -} - -impl From> for EncodedU64Array { - fn from(range: Range) -> Self { - let min = range.start; - let max = range.end; - let range = max - min; - if range < u16::MAX as u64 { - let base = min; - let offsets = (0..range as u16).collect(); - Self::U16 { base, offsets } - } else if range < u32::MAX as u64 { - let base = min; - let offsets = (0..range as u32).collect(); - Self::U32 { base, offsets } - } else { - Self::U64((min..max).collect()) - } - } -} - -impl FromIterator for EncodedU64Array { - fn from_iter>(iter: I) -> Self { - let values: Vec = iter.into_iter().collect(); - Self::from(values) - } -} - -impl IntoIterator for EncodedU64Array { - type Item = u64; - type IntoIter = Box>; - fn into_iter(self) -> Self::IntoIter { - match self { - Self::U16 { base, offsets } => { - Box::new(offsets.into_iter().map(move |o| base + o as u64)) - } - Self::U32 { base, offsets } => { - Box::new(offsets.into_iter().map(move |o| base + o as u64)) - } - Self::U64(values) => Box::new(values.into_iter()), - } - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_encoded_array_from_vec() { - fn roundtrip_array(values: Vec, expected: &EncodedU64Array) { - let encoded = EncodedU64Array::from(values.clone()); - assert_eq!(&encoded, expected); - - assert_eq!(values.len(), encoded.len()); - assert_eq!(values.first(), encoded.first().as_ref()); - assert_eq!(values.last(), encoded.last().as_ref()); - assert_eq!(values.iter().min(), encoded.min().as_ref()); - assert_eq!(values.iter().max(), encoded.max().as_ref()); - - let roundtripped = encoded.iter().collect::>(); - assert_eq!(values, roundtripped); - - for (i, v) in values.iter().enumerate() { - assert_eq!(Some(*v), encoded.get(i)); - } - - let encoded2 = values.into_iter().collect::(); - assert_eq!(&encoded2, expected); - } - - // Empty - roundtrip_array(vec![], &EncodedU64Array::U64(vec![])); - - // Single value - roundtrip_array( - vec![42], - &EncodedU64Array::U16 { - base: 42, - offsets: vec![0], - }, - ); - - // u16 version, it can start beyond the u16 range, but the - // relative values must be within u16 range. - let relative_values = [42, 0, 43, u16::MAX as u64, 99]; - let values = relative_values.map(|v| v + 2 * u16::MAX as u64).to_vec(); - let expected = EncodedU64Array::U16 { - base: 2 * u16::MAX as u64, - offsets: relative_values.iter().map(|v| *v as u16).collect(), - }; - roundtrip_array(values, &expected); - - // u32 version - let relative_values = [42, 0, 43, u32::MAX as u64, 99]; - let values = relative_values.map(|v| v + 2 * u32::MAX as u64).to_vec(); - let expected = EncodedU64Array::U32 { - base: 2 * u32::MAX as u64, - offsets: relative_values.iter().map(|v| *v as u32).collect(), - }; - roundtrip_array(values, &expected); - - // u64 version - let values = [42, 0, 43, u64::MAX, 99].to_vec(); - let expected = EncodedU64Array::U64(values.clone()); - roundtrip_array(values, &expected); - } - - #[test] - fn test_double_ended_iter() { - let arrays = vec![ - EncodedU64Array::U16 { - base: 42, - offsets: vec![0, 1, 2, 3, 4], - }, - EncodedU64Array::U32 { - base: 42, - offsets: vec![0, 1, 2, 3, 4], - }, - EncodedU64Array::U64(vec![42, 43, 44, 45, 46]), - ]; - for array in arrays { - // Should be able to iterate forwards and backwards, and get the same thing. - let forwards = array.iter().collect::>(); - let mut backwards = array.iter().rev().collect::>(); - backwards.reverse(); - assert_eq!(forwards, backwards); - - // Should be able to pull from both sides in lockstep. - let mut expected = Vec::with_capacity(array.len()); - let mut actual = Vec::with_capacity(array.len()); - let mut iter = array.iter(); - // Alternating forwards and backwards - for i in 0..array.len() { - if i % 2 == 0 { - actual.push(iter.next().unwrap()); - expected.push(array.get(i / 2).unwrap()); - } else { - let i = array.len() - 1 - i / 2; - actual.push(iter.next_back().unwrap()); - expected.push(array.get(i).unwrap()); - }; - } - assert_eq!(expected, actual); - } - } - - #[test] - fn test_encoded_array_from_range() { - // u16 version - let range = (2 * u16::MAX as u64)..(40 + 2 * u16::MAX as u64); - let encoded = EncodedU64Array::from(range.clone()); - let expected_base = 2 * u16::MAX as u64; - assert!( - matches!( - encoded, - EncodedU64Array::U16 { - base, - .. - } if base == expected_base - ), - "{:?}", - encoded - ); - let roundtripped = encoded.into_iter().collect::>(); - assert_eq!(range.collect::>(), roundtripped); - - // u32 version - let range = (2 * u32::MAX as u64)..(u16::MAX as u64 + 10 + 2 * u32::MAX as u64); - let encoded = EncodedU64Array::from(range.clone()); - let expected_base = 2 * u32::MAX as u64; - assert!(matches!( - encoded, - EncodedU64Array::U32 { - base, - .. - } if base == expected_base - )); - let roundtripped = encoded.into_iter().collect::>(); - assert_eq!(range.collect::>(), roundtripped); - - // We'll skip u64 since it would take a lot of memory. - - // Empty one - let range = 42..42; - let encoded = EncodedU64Array::from(range); - assert_eq!(encoded.len(), 0); - } -} diff --git a/vendor/lance-table/src/rowids/index.rs b/vendor/lance-table/src/rowids/index.rs deleted file mode 100644 index c7c34702..00000000 --- a/vendor/lance-table/src/rowids/index.rs +++ /dev/null @@ -1,822 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::ops::RangeInclusive; -use std::sync::Arc; - -use super::{RowIdSequence, U64Segment}; -use deepsize::DeepSizeOf; -use lance_core::{Error, Result}; -use lance_core::utils::address::RowAddress; -use lance_core::utils::deletion::DeletionVector; -use rangemap::RangeInclusiveMap; - -/// An index of row ids -/// -/// This index is used to map row ids to their corresponding addresses. These -/// addresses correspond to physical positions in the dataset. See [RowAddress]. -/// -/// This structure only contains rows that physically exist. However, it may -/// map to addresses that have been tombstoned. A separate tombstone index is -/// used to track tombstoned rows. -// (Implementation) -// Disjoint ranges of row ids are stored as the keys of the map. The values are -// a pair of segments. The first segment is the row ids, and the second segment -// is the addresses. -#[derive(Debug)] -pub struct RowIdIndex(RangeInclusiveMap); - -pub struct FragmentRowIdIndex { - pub fragment_id: u32, - pub row_id_sequence: Arc, - pub deletion_vector: Arc, -} - -impl RowIdIndex { - /// Create a new index from a list of fragment ids and their corresponding row id sequences. - pub fn new(fragment_indices: &[FragmentRowIdIndex]) -> Result { - let chunks = fragment_indices - .iter() - .flat_map(decompose_sequence) - .collect::>(); - - let mut final_chunks = Vec::new(); - for processed_chunk in prep_index_chunks(chunks) { - match processed_chunk { - RawIndexChunk::NonOverlapping(chunk) => { - final_chunks.push(chunk); - } - RawIndexChunk::Overlapping(_range, overlapping_chunks) => { - // Intersecting row-id ranges don't imply intersecting id sets; - // sparse ids and deletion holes leave the union short of the span. - // The real invariant (no id in two fragments) is checked in the merge. - let merged_chunk = merge_overlapping_chunks(overlapping_chunks)?; - final_chunks.push(merged_chunk); - } - } - } - - Ok(Self(RangeInclusiveMap::from_iter(final_chunks))) - } - - /// Get the address for a given row id. - /// - /// Will return None if the row id does not exist in the index. - pub fn get(&self, row_id: u64) -> Option { - let (row_id_segment, address_segment) = self.0.get(&row_id)?; - let pos = row_id_segment.position(row_id)?; - let address = address_segment.get(pos)?; - Some(RowAddress::from(address)) - } - - /// Get addresses for many row ids in one pass over the index. - /// - /// Returns one entry per input id, in input order (`None` for missing). - /// Sorts a working copy of the input internally so the chunk iterator - /// is advanced at most once per chunk, amortizing the per-id tree walk - /// from O(N · log F) to O(F + N). - pub fn get_many(&self, row_ids: &[u64]) -> Vec> { - let n = row_ids.len(); - let mut out = vec![None; n]; - if n == 0 { - return out; - } - - let mut sorted: Vec<(u64, usize)> = row_ids.iter().copied().zip(0..n).collect(); - sorted.sort_unstable_by_key(|&(id, _)| id); - - let mut chunks = self.0.iter().peekable(); - for (id, orig_idx) in sorted { - // Advance past chunks that end before this id. - while let Some((range, _)) = chunks.peek() { - if *range.end() < id { - chunks.next(); - } else { - break; - } - } - let Some((range, (row_id_seg, addr_seg))) = chunks.peek() else { - break; - }; - if id < *range.start() { - continue; // falls in a gap between chunks - } - if let Some(pos) = row_id_seg.position(id) - && let Some(addr) = addr_seg.get(pos) - { - out[orig_idx] = Some(RowAddress::from(addr)); - } - } - out - } -} - -impl DeepSizeOf for RowIdIndex { - fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize { - self.0 - .iter() - .map(|(_, (row_id_segment, address_segment))| { - (2 * std::mem::size_of::()) - + std::mem::size_of::<(U64Segment, U64Segment)>() - + row_id_segment.deep_size_of_children(context) - + address_segment.deep_size_of_children(context) - }) - .sum() - } -} - -fn decompose_sequence( - frag_index: &FragmentRowIdIndex, -) -> Vec<(RangeInclusive, (U64Segment, U64Segment))> { - let mut start_address: u64 = RowAddress::first_row(frag_index.fragment_id).into(); - let mut current_offset = 0u32; - let no_deletions = frag_index.deletion_vector.is_empty(); - - frag_index - .row_id_sequence - .0 - .iter() - .filter_map(|segment| { - let segment_len = segment.len(); - - let result = if no_deletions { - decompose_segment_no_deletions(segment, start_address) - } else { - decompose_segment_with_deletions( - segment, - start_address, - current_offset, - &frag_index.deletion_vector, - ) - }; - - current_offset += segment_len as u32; - start_address += segment_len as u64; - - result - }) - .collect() -} - -/// Build an IndexChunk from a list of (row_id, address) pairs. -fn build_chunk_from_pairs(pairs: Vec<(u64, u64)>) -> Option { - if pairs.is_empty() { - return None; - } - let (row_ids, addresses): (Vec, Vec) = pairs.into_iter().unzip(); - let row_id_segment = U64Segment::from_iter(row_ids); - let address_segment = U64Segment::from_iter(addresses); - let coverage = row_id_segment.range()?; - Some((coverage, (row_id_segment, address_segment))) -} - -/// Fast path: no deletions. O(1) for Range segments. -fn decompose_segment_no_deletions(segment: &U64Segment, start_address: u64) -> Option { - match segment { - U64Segment::Range(range) if !range.is_empty() => { - let len = range.end - range.start; - let row_id_segment = U64Segment::Range(range.clone()); - let address_segment = U64Segment::Range(start_address..start_address + len); - let coverage = range.start..=range.end - 1; - Some((coverage, (row_id_segment, address_segment))) - } - _ if segment.is_empty() => None, - _ => { - // Non-Range segments: must iterate to build address mapping. - let pairs: Vec<(u64, u64)> = segment - .iter() - .enumerate() - .map(|(i, row_id)| (row_id, start_address + i as u64)) - .collect(); - build_chunk_from_pairs(pairs) - } - } -} - -/// Slow path: has deletions, must check each row. -fn decompose_segment_with_deletions( - segment: &U64Segment, - start_address: u64, - current_offset: u32, - deletion_vector: &DeletionVector, -) -> Option { - let pairs: Vec<(u64, u64)> = segment - .iter() - .enumerate() - .filter_map(|(i, row_id)| { - let row_offset = current_offset + i as u32; - if !deletion_vector.contains(row_offset) { - Some((row_id, start_address + i as u64)) - } else { - None - } - }) - .collect(); - build_chunk_from_pairs(pairs) -} - -type IndexChunk = (RangeInclusive, (U64Segment, U64Segment)); - -#[derive(Debug)] -enum RawIndexChunk { - NonOverlapping(IndexChunk), - Overlapping(RangeInclusive, Vec), -} - -impl RawIndexChunk { - fn range_end(&self) -> u64 { - match self { - Self::NonOverlapping((range, _)) => *range.end(), - Self::Overlapping(range, _) => *range.end(), - } - } -} - -/// Given a vector of index chunks, sort them and return an iterator of index chunks. -/// -/// The iterator will yield chunks that are non-overlapping or a set of chunks -/// that are overlapping. -fn prep_index_chunks(mut chunks: Vec) -> impl Iterator { - chunks.sort_by_key(|(range, _)| u64::MAX - *range.start()); - - let mut output = Vec::new(); - - // Start assuming non-overlapping in first chunk. - if let Some(first_chunk) = chunks.pop() { - output.push(RawIndexChunk::NonOverlapping(first_chunk)); - } else { - // Early return for empty. - return output.into_iter(); - } - - let mut current_range = 0..=0; - let mut current_overlap = Vec::new(); - while let Some(chunk) = chunks.pop() { - debug_assert_eq!( - current_overlap - .iter() - .map(|(range, _): &IndexChunk| *range.start()) - .min() - .unwrap_or_default(), - *current_range.start(), - ); - debug_assert_eq!( - current_overlap - .iter() - .map(|(range, _): &IndexChunk| *range.end()) - .max() - .unwrap_or_default(), - *current_range.end(), - ); - - if current_overlap.is_empty() { - // We haven't found overlap yet. - let last_chunk_end = output.last().unwrap().range_end(); - if *chunk.0.start() <= last_chunk_end { - // We have found overlap. - match output.pop().unwrap() { - RawIndexChunk::NonOverlapping(chunk) => { - current_overlap.push(chunk); - } - _ => unreachable!(), - } - current_overlap.push(chunk); - - let range_start = *current_overlap.first().unwrap().0.start(); - let range_end = *current_overlap - .last() - .unwrap() - .0 - .end() - .max(current_overlap.first().unwrap().0.end()); - current_range = range_start..=range_end; - } else { - // We are still in non-overlapping space. - output.push(RawIndexChunk::NonOverlapping(chunk)); - } - } else { - // We are making an overlap chunk - if chunk.0.start() <= current_range.end() { - // We are still in overlap. - let range_end = *chunk.0.end().max(current_range.end()); - current_range = *current_range.start()..=range_end; - - current_overlap.push(chunk); - } else { - // We have exited overlap. - output.push(RawIndexChunk::Overlapping( - std::mem::replace(&mut current_range, 0..=0), - std::mem::take(&mut current_overlap), - )); - output.push(RawIndexChunk::NonOverlapping(chunk)); - } - } - } - debug_assert_eq!( - current_overlap - .iter() - .map(|(range, _): &IndexChunk| *range.start()) - .min() - .unwrap_or_default(), - *current_range.start(), - ); - debug_assert_eq!( - current_overlap - .iter() - .map(|(range, _): &IndexChunk| *range.end()) - .max() - .unwrap_or_default(), - *current_range.end(), - ); - - if !current_overlap.is_empty() { - output.push(RawIndexChunk::Overlapping( - current_range.clone(), - current_overlap, - )); - } - - output.into_iter() -} - -fn merge_overlapping_chunks(overlapping_chunks: Vec) -> Result { - let total_capacity = overlapping_chunks - .iter() - .map(|(_, (row_ids, _))| row_ids.len()) - .sum(); - let mut values = Vec::with_capacity(total_capacity); - for (_, (row_ids, row_addrs)) in overlapping_chunks.iter() { - values.extend(row_ids.iter().zip(row_addrs.iter())); - } - values.sort_by_key(|(row_id, _)| *row_id); - // A duplicate row id here means two fragments claim the same live id: a - // corrupt index, not a resolvable sparse-coverage case. - if let Some(w) = values.windows(2).find(|w| w[0].0 == w[1].0) { - return Err(Error::internal(format!( - "row id index corrupt: stable row id {} is live in multiple fragments", - w[0].0 - ))); - } - let row_id_segment = U64Segment::from_iter(values.iter().map(|(row_id, _)| *row_id)); - let address_segment = U64Segment::from_iter(values.iter().map(|(_, row_addr)| *row_addr)); - - let range = row_id_segment.range().unwrap(); - - Ok((range, (row_id_segment, address_segment))) -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::{prelude::Strategy, prop_assert_eq}; - - #[test] - fn test_new_index() { - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![ - U64Segment::Range(0..10), - U64Segment::RangeWithHoles { - range: 10..17, - holes: vec![12, 15].into(), - }, - U64Segment::SortedArray(vec![20, 25, 30].into()), - ])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 20, - row_id_sequence: Arc::new(RowIdSequence(vec![ - U64Segment::RangeWithBitmap { - range: 17..20, - bitmap: [true, false, true].as_slice().into(), - }, - U64Segment::Array(vec![40, 50, 60].into()), - ])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - // Check various queries. - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(15), None); - assert_eq!(index.get(16), Some(RowAddress::new_from_parts(10, 14))); - assert_eq!(index.get(17), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(25), Some(RowAddress::new_from_parts(10, 16))); - assert_eq!(index.get(40), Some(RowAddress::new_from_parts(20, 2))); - assert_eq!(index.get(60), Some(RowAddress::new_from_parts(20, 4))); - assert_eq!(index.get(61), None); - } - - #[test] - fn test_new_index_overlap() { - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 23, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray( - vec![3, 6, 9].into(), - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 42, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray( - vec![2, 5, 8].into(), - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray( - vec![1, 4, 7].into(), - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - // Check various queries. - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(42, 0))); - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(23, 0))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(42, 1))); - assert_eq!(index.get(6), Some(RowAddress::new_from_parts(23, 1))); - assert_eq!(index.get(7), Some(RowAddress::new_from_parts(10, 2))); - assert_eq!(index.get(8), Some(RowAddress::new_from_parts(42, 2))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(23, 2))); - } - - #[test] - fn test_new_index_unsorted_row_ids() { - // Test case with unsorted row ids within fragments - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Array( - vec![9, 3, 6].into(), // Unsorted array - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 20, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Array( - vec![8, 2, 5].into(), // Unsorted array - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 30, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Array( - vec![7, 1, 4].into(), // Unsorted array - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - // Check that all row ids can be found regardless of their order in the segments - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(30, 1))); - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(20, 1))); - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(30, 2))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(20, 2))); - assert_eq!(index.get(6), Some(RowAddress::new_from_parts(10, 2))); - assert_eq!(index.get(7), Some(RowAddress::new_from_parts(30, 0))); - assert_eq!(index.get(8), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(10, 0))); - - // Check that non-existent row ids return None - assert_eq!(index.get(0), None); - assert_eq!(index.get(10), None); - } - - #[test] - fn test_new_index_partial_overlap() { - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 0, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::RangeWithHoles { - range: 0..100, - holes: vec![50].into(), - }])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 1, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(50..51)])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - // Check various queries. - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(0, 0))); - assert_eq!(index.get(49), Some(RowAddress::new_from_parts(0, 49))); - assert_eq!(index.get(50), Some(RowAddress::new_from_parts(1, 0))); - assert_eq!(index.get(51), Some(RowAddress::new_from_parts(0, 50))); - assert_eq!(index.get(99), Some(RowAddress::new_from_parts(0, 98))); - } - - #[test] - fn test_overlapping_chunks_sparse_with_deletions() { - // Interleaved (overlapping) id ranges plus a deletion that leaves a hole, - // so the union doesn't tile the span. Every live id must still resolve. - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray( - vec![1, 3, 5, 7, 9].into(), - )])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 20, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray( - vec![0, 2, 4, 6, 8].into(), - )])), - // Delete offset 2 (id 4) -> a hole in the span. - deletion_vector: Arc::new(DeletionVector::from_iter(vec![2])), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(20, 1))); - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(4), None); - // Surviving ids keep their original offsets (the hole is not compacted). - assert_eq!(index.get(6), Some(RowAddress::new_from_parts(20, 3))); - assert_eq!(index.get(8), Some(RowAddress::new_from_parts(20, 4))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(10, 4))); - } - - #[test] - fn test_index_with_deletion_vector() { - let deletion_vector = DeletionVector::from_iter(vec![2, 3]); - - let fragment_indices = vec![FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(0..6)])), - deletion_vector: Arc::new(deletion_vector), - }]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(10, 4))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(10, 5))); - - assert_eq!(index.get(2), None); - assert_eq!(index.get(3), None); - } - - #[test] - fn test_empty_fragment_sequences() { - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 20, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(5..8)])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(7), Some(RowAddress::new_from_parts(20, 2))); - assert_eq!(index.get(4), None); - } - - #[test] - fn test_completely_empty_index() { - let fragment_indices = vec![]; - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - assert_eq!(index.get(0), None); - assert_eq!(index.get(100), None); - } - - #[test] - fn test_non_overlapping_ranges() { - let fragment_indices = vec![ - FragmentRowIdIndex { - fragment_id: 10, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(0..5)])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 20, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(5..10)])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - FragmentRowIdIndex { - fragment_id: 30, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range(10..15)])), - deletion_vector: Arc::new(DeletionVector::default()), - }, - ]; - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(10, 4))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(20, 4))); - assert_eq!(index.get(10), Some(RowAddress::new_from_parts(30, 0))); - assert_eq!(index.get(14), Some(RowAddress::new_from_parts(30, 4))); - } - - fn arbitrary_row_ids( - num_fragments_range: std::ops::Range, - frag_size_range: std::ops::Range, - ) -> impl Strategy)>> { - let fragment_sizes = proptest::collection::vec(frag_size_range, num_fragments_range); - fragment_sizes.prop_flat_map(|fragment_sizes| { - let num_rows = fragment_sizes.iter().sum::() as u64; - let row_ids = 0..num_rows; - let row_ids = row_ids.collect::>(); - let row_ids_shuffled = proptest::strategy::Just(row_ids).prop_shuffle(); - row_ids_shuffled.prop_map(move |row_ids| { - let mut sequences = Vec::with_capacity(fragment_sizes.len()); - let mut i = 0; - for size in &fragment_sizes { - let end = i + size; - let sequence = - RowIdSequence(vec![U64Segment::from_slice(row_ids[i..end].into())]); - sequences.push((i as u32, Arc::new(sequence))); - i = end; - } - sequences - }) - }) - } - - #[test] - fn test_large_range_segments_no_deletions() { - // Simulates a real-world scenario: many fragments with large Range segments - // and no deletions. Before optimization, this would iterate over all rows - // (O(total_rows)). After optimization, it's O(num_fragments). - let rows_per_fragment = 250_000u64; - let num_fragments = 100u32; - let mut offset = 0u64; - - let fragment_indices: Vec = (0..num_fragments) - .map(|frag_id| { - let start = offset; - offset += rows_per_fragment; - FragmentRowIdIndex { - fragment_id: frag_id, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range( - start..start + rows_per_fragment, - )])), - deletion_vector: Arc::new(DeletionVector::default()), - } - }) - .collect(); - - let start = std::time::Instant::now(); - let index = RowIdIndex::new(&fragment_indices).unwrap(); - let elapsed = start.elapsed(); - - // Verify correctness at boundaries - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(0, 0))); - assert_eq!( - index.get(rows_per_fragment - 1), - Some(RowAddress::new_from_parts(0, rows_per_fragment as u32 - 1)) - ); - assert_eq!( - index.get(rows_per_fragment), - Some(RowAddress::new_from_parts(1, 0)) - ); - let last_row = num_fragments as u64 * rows_per_fragment - 1; - assert_eq!( - index.get(last_row), - Some(RowAddress::new_from_parts( - num_fragments - 1, - rows_per_fragment as u32 - 1 - )) - ); - assert_eq!(index.get(last_row + 1), None); - - // With the optimization, building an index for 25M rows across 100 fragments - // should complete in well under 1 second (typically < 1ms). - assert!( - elapsed.as_secs() < 1, - "Index build took {:?} for {} fragments x {} rows = {} total rows. \ - This suggests the O(rows) -> O(fragments) optimization is not working.", - elapsed, - num_fragments, - rows_per_fragment, - num_fragments as u64 * rows_per_fragment, - ); - } - - #[test] - fn test_large_range_segments_with_deletions() { - let rows_per_fragment = 1_000u64; - let num_fragments = 10u32; - let mut offset = 0u64; - - let fragment_indices: Vec = (0..num_fragments) - .map(|frag_id| { - let start = offset; - offset += rows_per_fragment; - - // Delete every 3rd row (offsets 0, 3, 6, ...) within each fragment. - let mut deleted = roaring::RoaringBitmap::new(); - for i in (0..rows_per_fragment as u32).step_by(3) { - deleted.insert(i); - } - - FragmentRowIdIndex { - fragment_id: frag_id, - row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::Range( - start..start + rows_per_fragment, - )])), - deletion_vector: Arc::new(DeletionVector::Bitmap(deleted)), - } - }) - .collect(); - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - - // Deleted rows (offset 0, 3, 6, ...) should not be found. - // Row ID 0 has offset 0 in fragment 0 -> deleted. - assert_eq!(index.get(0), None); - // Row ID 3 has offset 3 in fragment 0 -> deleted. - assert_eq!(index.get(3), None); - - // Non-deleted rows should resolve correctly. - // Row ID 1 has offset 1 in fragment 0 -> address (frag=0, row=1). - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(0, 1))); - // Row ID 2 has offset 2 in fragment 0 -> address (frag=0, row=2). - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(0, 2))); - // Row ID 4 has offset 4 in fragment 0 -> address (frag=0, row=4). - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(0, 4))); - - // Check second fragment: row IDs start at 1000. - // Row ID 1000 has offset 0 in fragment 1 -> deleted. - assert_eq!(index.get(rows_per_fragment), None); - // Row ID 1001 has offset 1 in fragment 1 -> address (frag=1, row=1). - assert_eq!( - index.get(rows_per_fragment + 1), - Some(RowAddress::new_from_parts(1, 1)) - ); - - // Last fragment, last non-deleted row. - // Row ID 9999 has offset 999 in fragment 9 -> 999 % 3 == 0 -> deleted. - let last_row = num_fragments as u64 * rows_per_fragment - 1; - assert_eq!(index.get(last_row), None); - // Row ID 9998 has offset 998 -> 998 % 3 == 2 -> not deleted. - assert_eq!( - index.get(last_row - 1), - Some(RowAddress::new_from_parts(num_fragments - 1, 998)) - ); - - // Out of range. - assert_eq!(index.get(last_row + 1), None); - } - - proptest::proptest! { - #[test] - fn test_new_index_robustness(row_ids in arbitrary_row_ids(0..5, 0..32)) { - let fragment_indices: Vec = row_ids - .iter() - .map(|(frag_id, sequence)| FragmentRowIdIndex { - fragment_id: *frag_id, - row_id_sequence: sequence.clone(), - deletion_vector: Arc::new(DeletionVector::default()), - }) - .collect(); - - let index = RowIdIndex::new(&fragment_indices).unwrap(); - for (frag_id, sequence) in row_ids.iter() { - for (local_offset, row_id) in sequence.iter().enumerate() { - prop_assert_eq!( - index.get(row_id), - Some(RowAddress::new_from_parts(*frag_id, local_offset as u32)), - "Row id {} in sequence {:?} not found in index {:?}", - row_id, - sequence, - index - ); - } - } - } - } -} diff --git a/vendor/lance-table/src/rowids/segment.rs b/vendor/lance-table/src/rowids/segment.rs deleted file mode 100644 index a02acd8a..00000000 --- a/vendor/lance-table/src/rowids/segment.rs +++ /dev/null @@ -1,1141 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::ops::{Range, RangeInclusive}; - -use super::{bitmap::Bitmap, encoded_array::EncodedU64Array}; -use deepsize::DeepSizeOf; - -/// Convert an estimated serialized byte cost from `u128` to `usize`, saturating -/// at [`usize::MAX`] when the value does not fit (infeasible encodings). -#[inline] -fn u128_byte_cost_to_usize(v: u128) -> usize { - usize::try_from(v).unwrap_or(usize::MAX) -} - -/// Different ways to represent a sequence of distinct u64s. -/// -/// This is designed to be especially efficient for sequences that are sorted, -/// but not meaningfully larger than a `Vec` in the worst case. -/// -/// The representation is chosen based on the properties of the sequence: -/// -/// Sorted?───►Yes ───►Contiguous?─► Yes─► Range -/// │ â–¼ -/// │ No -/// │ â–¼ -/// │ Dense?─────► Yes─► RangeWithBitmap/RangeWithHoles -/// │ â–¼ -/// │ No─────────────► SortedArray -/// â–¼ -/// No──────────────────────────────► Array -/// -/// "Dense" is decided based on the estimated byte size of the representation. -/// -/// Size of RangeWithBitMap for N values: -/// 8 bytes + 8 bytes + ceil((max - min) / 8) bytes -/// Size of SortedArray for N values (assuming u16 packed): -/// 8 bytes + 8 bytes + 8 bytes + 2 bytes * N -/// -#[derive(Debug, PartialEq, Eq, Clone)] -pub enum U64Segment { - /// A contiguous sorted range of row ids. - /// - /// Total size: 16 bytes - Range(Range), - /// A sorted range of row ids, that is mostly contiguous. - /// - /// Total size: 24 bytes + n_holes * 4 bytes - /// Use when: 32 * n_holes < max - min - RangeWithHoles { - range: Range, - /// Bitmap of offsets from the start of the range that are holes. - /// This is sorted, so binary search can be used. It's typically - /// relatively small. - holes: EncodedU64Array, - }, - /// A sorted range of row ids, that is mostly contiguous. - /// - /// Bitmap is 1 when the value is present, 0 when it's missing. - /// - /// Total size: 24 bytes + ceil((max - min) / 8) bytes - /// Use when: max - min > 16 * len - RangeWithBitmap { range: Range, bitmap: Bitmap }, - /// A sorted array of row ids, that is sparse. - /// - /// Total size: 24 bytes + 2 * n_values bytes - SortedArray(EncodedU64Array), - /// An array of row ids, that is not sorted. - Array(EncodedU64Array), -} - -impl DeepSizeOf for U64Segment { - fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize { - match self { - Self::Range(_) => 0, - Self::RangeWithHoles { holes, .. } => holes.deep_size_of_children(context), - Self::RangeWithBitmap { bitmap, .. } => bitmap.deep_size_of_children(context), - Self::SortedArray(array) => array.deep_size_of_children(context), - Self::Array(array) => array.deep_size_of_children(context), - } - } -} - -/// Statistics about a segment of u64s. -#[derive(Debug)] -struct SegmentStats { - /// Min value in the segment. - min: u64, - /// Max value in the segment - max: u64, - /// Total number of values in the segment - count: u64, - /// Whether the segment is sorted - sorted: bool, -} - -impl SegmentStats { - /// Number of missing values ("holes") in the range `[min, max]`. - /// - /// Returns `u128` because the total slot count `max - min + 1` can be up - /// to `2^64` (when `min = 0, max = u64::MAX`), which exceeds `u64::MAX`. - fn n_holes(&self) -> u128 { - debug_assert!(self.sorted); - if self.count == 0 { - 0 - } else { - let total_slots = self.max as u128 - self.min as u128 + 1; - total_slots - self.count as u128 - } - } -} - -impl U64Segment { - /// Return the values that are missing from the slice. - fn holes_in_slice<'a>( - range: RangeInclusive, - existing: impl IntoIterator + 'a, - ) -> impl Iterator + 'a { - let mut existing = existing.into_iter().peekable(); - range.filter(move |val| { - if let Some(&existing_val) = existing.peek() - && existing_val == *val - { - existing.next(); - return false; - } - true - }) - } - - fn compute_stats(values: impl IntoIterator) -> SegmentStats { - let mut sorted = true; - let mut min = u64::MAX; - let mut max = 0; - let mut count = 0; - - for val in values { - count += 1; - if val < min { - min = val; - } - if val > max { - max = val; - } - if sorted && count > 1 && val < max { - sorted = false; - } - } - - if count == 0 { - min = 0; - max = 0; - } - - SegmentStats { - min, - max, - count, - sorted, - } - } - - /// Estimate the serialized byte size of each sorted encoding variant. - /// - /// All arithmetic is performed in `u128` to avoid overflow when the range - /// span `max - min + 1` approaches or exceeds `2^64`. Infeasible sizes - /// saturate to `usize::MAX` so they always lose the `min()` comparison. - fn sorted_sequence_sizes(stats: &SegmentStats) -> [usize; 3] { - let n_holes = stats.n_holes(); - let total_slots = stats.max as u128 - stats.min as u128 + 1; - - let range_with_holes = 24u128.saturating_add(4u128.saturating_mul(n_holes)); - let range_with_bitmap = 24u128.saturating_add(total_slots.div_ceil(8)); - let sorted_array = 24u128.saturating_add(2u128.saturating_mul(stats.count as u128)); - - [ - u128_byte_cost_to_usize(range_with_holes), - u128_byte_cost_to_usize(range_with_bitmap), - u128_byte_cost_to_usize(sorted_array), - ] - } - - fn from_stats_and_sequence( - stats: SegmentStats, - sequence: impl IntoIterator, - ) -> Self { - if stats.sorted { - let n_holes = stats.n_holes(); - // Range-backed encodings store an exclusive end as `Range`, - // which cannot represent `u64::MAX + 1`. Compute the end once and - // gate all range-backed branches on its representability. - let exclusive_end = stats.max.checked_add(1); - if stats.count == 0 { - Self::Range(0..0) - } else if n_holes == 0 && exclusive_end.is_some() { - Self::Range(stats.min..exclusive_end.unwrap()) - } else if let Some(end) = exclusive_end { - let sizes = Self::sorted_sequence_sizes(&stats); - let min_size = sizes.iter().min().unwrap(); - if min_size == &sizes[0] { - let range = stats.min..end; - let mut holes = - Self::holes_in_slice(stats.min..=stats.max, sequence).collect::>(); - holes.sort_unstable(); - let holes = EncodedU64Array::from(holes); - Self::RangeWithHoles { range, holes } - } else if min_size == &sizes[1] { - let range = stats.min..end; - let mut bitmap = Bitmap::new_full((stats.max - stats.min) as usize + 1); - for hole in Self::holes_in_slice(stats.min..=stats.max, sequence) { - let offset = (hole - stats.min) as usize; - bitmap.clear(offset); - } - Self::RangeWithBitmap { range, bitmap } - } else { - Self::SortedArray(EncodedU64Array::from_iter(sequence)) - } - } else { - // max == u64::MAX: exclusive end is unrepresentable in Range, - // so no range-backed encoding can be used. - Self::SortedArray(EncodedU64Array::from_iter(sequence)) - } - } else { - Self::Array(EncodedU64Array::from_iter(sequence)) - } - } - - pub fn from_slice(slice: &[u64]) -> Self { - Self::from_iter(slice.iter().copied()) - } -} - -impl FromIterator for U64Segment { - fn from_iter>(iter: T) -> Self { - let values: Vec = iter.into_iter().collect(); - let stats = Self::compute_stats(values.iter().copied()); - Self::from_stats_and_sequence(stats, values) - } -} - -impl U64Segment { - pub fn iter(&self) -> Box + '_> { - match self { - Self::Range(range) => Box::new(range.clone()), - Self::RangeWithHoles { range, holes } => { - Box::new((range.start..range.end).filter(move |&val| { - // TODO: we could write a more optimal version of this - // iterator, but would need special handling to make it - // double ended. - holes.binary_search(val).is_err() - })) - } - Self::RangeWithBitmap { range, bitmap } => { - Box::new((range.start..range.end).filter(|val| { - let offset = (val - range.start) as usize; - bitmap.get(offset) - })) - } - Self::SortedArray(array) => Box::new(array.iter()), - Self::Array(array) => Box::new(array.iter()), - } - } - - pub fn len(&self) -> usize { - match self { - Self::Range(range) => (range.end - range.start) as usize, - Self::RangeWithHoles { range, holes } => { - let holes = holes.iter().count(); - (range.end - range.start) as usize - holes - } - Self::RangeWithBitmap { range, bitmap } => { - let holes = bitmap.count_zeros(); - (range.end - range.start) as usize - holes - } - Self::SortedArray(array) => array.len(), - Self::Array(array) => array.len(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Get the min and max value of the segment, excluding tombstones. - pub fn range(&self) -> Option> { - match self { - Self::Range(range) if range.is_empty() => None, - Self::Range(range) - | Self::RangeWithBitmap { range, .. } - | Self::RangeWithHoles { range, .. } => Some(range.start..=(range.end - 1)), - Self::SortedArray(array) => { - // We can assume that the array is sorted. - let min_value = array.first().unwrap(); - let max_value = array.last().unwrap(); - Some(min_value..=max_value) - } - Self::Array(array) => { - let min_value = array.min().unwrap(); - let max_value = array.max().unwrap(); - Some(min_value..=max_value) - } - } - } - - pub fn slice(&self, offset: usize, len: usize) -> Self { - if len == 0 { - return Self::Range(0..0); - } - - let values: Vec = self.iter().skip(offset).take(len).collect(); - - // `from_slice` will compute stats and select the best representation. - Self::from_slice(&values) - } - - pub fn position(&self, val: u64) -> Option { - match self { - Self::Range(range) => { - if range.contains(&val) { - Some((val - range.start) as usize) - } else { - None - } - } - Self::RangeWithHoles { range, holes } => { - if !range.contains(&val) { - return None; - } - // binary_search returns Err(idx) where idx is the count of holes - // strictly less than val (holes are unique and sorted). - match holes.binary_search(val) { - Ok(_) => None, - Err(num_holes_before) => { - let offset = (val - range.start) as usize; - Some(offset - num_holes_before) - } - } - } - Self::RangeWithBitmap { range, bitmap } => { - if range.contains(&val) && bitmap.get((val - range.start) as usize) { - let offset = (val - range.start) as usize; - let num_zeros = bitmap.slice(0, offset).count_zeros(); - Some(offset - num_zeros) - } else { - None - } - } - Self::SortedArray(array) => array.binary_search(val).ok(), - Self::Array(array) => array.iter().position(|v| v == val), - } - } - - pub fn get(&self, i: usize) -> Option { - match self { - Self::Range(range) => match range.start.checked_add(i as u64) { - Some(val) if val < range.end => Some(val), - _ => None, - }, - Self::RangeWithHoles { range, holes } => { - let len = (range.end - range.start) as usize - holes.len(); - if i >= len { - return None; - } - // The i-th surviving value v satisfies v = range.start + i + k, - // where k = |{h ∈ holes : h < v}|. holes[k] - k is monotone - // non-decreasing in k (holes are sorted and unique), so binary - // search for the smallest k such that holes[k] - k > range.start + i. - let target = range.start + i as u64; - let mut lo = 0usize; - let mut hi = holes.len(); - while lo < hi { - let mid = (lo + hi) / 2; - let h = holes.get(mid).unwrap(); - if h.saturating_sub(mid as u64) > target { - hi = mid; - } else { - lo = mid + 1; - } - } - Some(range.start + i as u64 + lo as u64) - } - Self::RangeWithBitmap { range, bitmap } => { - // Find the i-th set bit (a "select1") via byte-wise popcount. - // Bytes past `bitmap.len()` are zero-padded by construction - // (Bitmap::new_full), so popcount counts only valid positions. - let mut remaining = i; - for (byte_idx, &byte) in bitmap.data.iter().enumerate() { - let ones = byte.count_ones() as usize; - if remaining < ones { - let mut b = byte; - for _ in 0..remaining { - b &= b - 1; // clear lowest set bit - } - let bit = b.trailing_zeros() as usize; - return Some(range.start + (byte_idx * 8 + bit) as u64); - } - remaining -= ones; - } - None - } - Self::SortedArray(array) => array.get(i), - Self::Array(array) => array.get(i), - } - } - - /// Check if a value is contained in the segment - pub fn contains(&self, val: u64) -> bool { - match self { - Self::Range(range) => range.contains(&val), - Self::RangeWithHoles { range, holes } => { - if !range.contains(&val) { - return false; - } - // Check if the value is not in the holes - !holes.iter().any(|hole| hole == val) - } - Self::RangeWithBitmap { range, bitmap } => { - if !range.contains(&val) { - return false; - } - // Check if the bitmap has the value set (not cleared) - let idx = (val - range.start) as usize; - bitmap.get(idx) - } - Self::SortedArray(array) => array.binary_search(val).is_ok(), - Self::Array(array) => array.iter().any(|v| v == val), - } - } - - /// Produce a new segment that has `val` as the new highest value in the segment - pub fn with_new_high(self, val: u64) -> lance_core::Result { - // Check that the new value is higher than the current maximum - if let Some(range) = self.range() - && val <= *range.end() - { - return Err(lance_core::Error::invalid_input(format!( - "New value {} must be higher than current maximum {}", - val, - range.end() - ))); - } - - Ok(match self { - Self::Range(range) => { - // Special case for empty range: create a range containing only the new value - if range.start == range.end { - Self::Range(Range { - start: val, - end: val + 1, - }) - } else if val == range.end { - Self::Range(Range { - start: range.start, - end: val + 1, - }) - } else { - Self::RangeWithHoles { - range: Range { - start: range.start, - end: val + 1, - }, - holes: EncodedU64Array::U64((range.end..val).collect()), - } - } - } - Self::RangeWithHoles { range, holes } => { - if val == range.end { - Self::RangeWithHoles { - range: Range { - start: range.start, - end: val + 1, - }, - holes, - } - } else { - let mut new_holes: Vec = holes.iter().collect(); - new_holes.extend(range.end..val); - Self::RangeWithHoles { - range: Range { - start: range.start, - end: val + 1, - }, - holes: EncodedU64Array::U64(new_holes), - } - } - } - Self::RangeWithBitmap { range, bitmap } => { - let new_range = Range { - start: range.start, - end: val + 1, - }; - let gap_size = (val - range.end) as usize; - let new_bitmap = bitmap - .iter() - .chain(std::iter::repeat_n(false, gap_size)) - .chain(std::iter::once(true)) - .collect::>(); - - Self::RangeWithBitmap { - range: new_range, - bitmap: Bitmap::from(new_bitmap.as_slice()), - } - } - Self::SortedArray(array) => match array { - EncodedU64Array::U64(mut vec) => { - vec.push(val); - Self::SortedArray(EncodedU64Array::U64(vec)) - } - EncodedU64Array::U16 { base, offsets } => { - if let Some(offset) = val.checked_sub(base) { - if offset <= u16::MAX as u64 { - let mut offsets = offsets; - offsets.push(offset as u16); - return Ok(Self::SortedArray(EncodedU64Array::U16 { base, offsets })); - } else if offset <= u32::MAX as u64 { - let mut u32_offsets: Vec = - offsets.into_iter().map(|o| o as u32).collect(); - u32_offsets.push(offset as u32); - return Ok(Self::SortedArray(EncodedU64Array::U32 { - base, - offsets: u32_offsets, - })); - } - } - let mut new_array: Vec = - offsets.into_iter().map(|o| base + o as u64).collect(); - new_array.push(val); - Self::SortedArray(EncodedU64Array::from(new_array)) - } - EncodedU64Array::U32 { base, mut offsets } => { - if let Some(offset) = val.checked_sub(base) - && offset <= u32::MAX as u64 - { - offsets.push(offset as u32); - return Ok(Self::SortedArray(EncodedU64Array::U32 { base, offsets })); - } - let mut new_array: Vec = - offsets.into_iter().map(|o| base + o as u64).collect(); - new_array.push(val); - Self::SortedArray(EncodedU64Array::from(new_array)) - } - }, - Self::Array(array) => match array { - EncodedU64Array::U64(mut vec) => { - vec.push(val); - Self::Array(EncodedU64Array::U64(vec)) - } - EncodedU64Array::U16 { base, offsets } => { - if let Some(offset) = val.checked_sub(base) { - if offset <= u16::MAX as u64 { - let mut offsets = offsets; - offsets.push(offset as u16); - return Ok(Self::Array(EncodedU64Array::U16 { base, offsets })); - } else if offset <= u32::MAX as u64 { - let mut u32_offsets: Vec = - offsets.into_iter().map(|o| o as u32).collect(); - u32_offsets.push(offset as u32); - return Ok(Self::Array(EncodedU64Array::U32 { - base, - offsets: u32_offsets, - })); - } - } - let mut new_array: Vec = - offsets.into_iter().map(|o| base + o as u64).collect(); - new_array.push(val); - Self::Array(EncodedU64Array::from(new_array)) - } - EncodedU64Array::U32 { base, mut offsets } => { - if let Some(offset) = val.checked_sub(base) - && offset <= u32::MAX as u64 - { - offsets.push(offset as u32); - return Ok(Self::Array(EncodedU64Array::U32 { base, offsets })); - } - let mut new_array: Vec = - offsets.into_iter().map(|o| base + o as u64).collect(); - new_array.push(val); - Self::Array(EncodedU64Array::from(new_array)) - } - }, - }) - } - - /// Delete a set of row ids from the segment. - /// The row ids are assumed to be in the segment. (within the range, not - /// already deleted.) - /// They are also assumed to be ordered by appearance in the segment. - pub fn delete(&self, vals: &[u64]) -> Self { - // TODO: can we enforce these assumptions? or make them safer? - debug_assert!(vals.iter().all(|&val| self.range().unwrap().contains(&val))); - - let make_new_iter = || { - let mut vals_iter = vals.iter().copied().peekable(); - self.iter().filter(move |val| { - if let Some(&next_val) = vals_iter.peek() - && next_val == *val - { - vals_iter.next(); - return false; - } - true - }) - }; - let stats = Self::compute_stats(make_new_iter()); - Self::from_stats_and_sequence(stats, make_new_iter()) - } - - pub fn mask(&mut self, positions: &[u32]) { - if positions.is_empty() { - return; - } - if positions.len() == self.len() { - *self = Self::Range(0..0); - return; - } - let count = (self.len() - positions.len()) as u64; - let sorted = match self { - Self::Range(_) => true, - Self::RangeWithHoles { .. } => true, - Self::RangeWithBitmap { .. } => true, - Self::SortedArray(_) => true, - Self::Array(_) => false, - }; - // To get minimum, need to find the first value that is not masked. - let first_unmasked = (0..self.len()) - .zip(positions.iter().cycle()) - .find(|(sequential_i, i)| **i != *sequential_i as u32) - .map(|(sequential_i, _)| sequential_i) - .unwrap(); - let min = self.get(first_unmasked).unwrap(); - - let last_unmasked = (0..self.len()) - .rev() - .zip(positions.iter().rev().cycle()) - .filter(|(sequential_i, i)| **i != *sequential_i as u32) - .map(|(sequential_i, _)| sequential_i) - .next() - .unwrap(); - let max = self.get(last_unmasked).unwrap(); - - let stats = SegmentStats { - min, - max, - count, - sorted, - }; - - let mut positions = positions.iter().copied().peekable(); - let sequence = self.iter().enumerate().filter_map(move |(i, val)| { - if let Some(next_pos) = positions.peek() - && *next_pos == i as u32 - { - positions.next(); - return None; - } - Some(val) - }); - *self = Self::from_stats_and_sequence(stats, sequence) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_segments() { - fn check_segment(values: &[u64], expected: &U64Segment) { - let segment = U64Segment::from_slice(values); - assert_eq!(segment, *expected); - assert_eq!(values.len(), segment.len()); - - let roundtripped = segment.iter().collect::>(); - assert_eq!(roundtripped, values); - - let expected_min = values.iter().copied().min(); - let expected_max = values.iter().copied().max(); - match segment.range() { - Some(range) => { - assert_eq!(range.start(), &expected_min.unwrap()); - assert_eq!(range.end(), &expected_max.unwrap()); - } - None => { - assert_eq!(expected_min, None); - assert_eq!(expected_max, None); - } - } - - for (i, value) in values.iter().enumerate() { - assert_eq!(segment.get(i), Some(*value), "i = {}", i); - assert_eq!(segment.position(*value), Some(i), "i = {}", i); - } - - check_segment_iter(&segment); - } - - fn check_segment_iter(segment: &U64Segment) { - // Should be able to iterate forwards and backwards, and get the same thing. - let forwards = segment.iter().collect::>(); - let mut backwards = segment.iter().rev().collect::>(); - backwards.reverse(); - assert_eq!(forwards, backwards); - - // Should be able to pull from both sides in lockstep. - let mut expected = Vec::with_capacity(segment.len()); - let mut actual = Vec::with_capacity(segment.len()); - let mut iter = segment.iter(); - // Alternating forwards and backwards - for i in 0..segment.len() { - if i % 2 == 0 { - actual.push(iter.next().unwrap()); - expected.push(segment.get(i / 2).unwrap()); - } else { - let i = segment.len() - 1 - i / 2; - actual.push(iter.next_back().unwrap()); - expected.push(segment.get(i).unwrap()); - }; - } - assert_eq!(expected, actual); - } - - // Empty - check_segment(&[], &U64Segment::Range(0..0)); - - // Single value - check_segment(&[42], &U64Segment::Range(42..43)); - - // Contiguous range - check_segment( - &(100..200).collect::>(), - &U64Segment::Range(100..200), - ); - - // Range with a hole - let values = (0..1000).filter(|&x| x != 100).collect::>(); - check_segment( - &values, - &U64Segment::RangeWithHoles { - range: 0..1000, - holes: vec![100].into(), - }, - ); - - // Range with every other value missing - let values = (0..1000).filter(|&x| x % 2 == 0).collect::>(); - check_segment( - &values, - &U64Segment::RangeWithBitmap { - range: 0..999, - bitmap: Bitmap::from((0..999).map(|x| x % 2 == 0).collect::>().as_slice()), - }, - ); - - // Sparse but sorted sequence - check_segment( - &[1, 7000, 24000], - &U64Segment::SortedArray(vec![1, 7000, 24000].into()), - ); - - // Sparse unsorted sequence - check_segment( - &[7000, 1, 24000], - &U64Segment::Array(vec![7000, 1, 24000].into()), - ); - } - - #[test] - fn test_segment_overflow_boundary() { - // Sparse range spanning i64::MAX — the original overflow reproducer. - // n_holes ≈ 2^63, which overflows `4 * n_holes as usize` without u128 arithmetic. - let values: Vec = vec![0, 1, 2, 100, i64::MAX as u64]; - let segment = U64Segment::from_slice(&values); - assert!( - matches!(segment, U64Segment::SortedArray(_)), - "sparse range spanning i64::MAX should be SortedArray, got {:?}", - std::mem::discriminant(&segment) - ); - assert_eq!(segment.len(), 5); - assert_eq!(segment.iter().collect::>(), values); - - // Two values at u64 extremes — triggers n_holes() total_slots overflow - // (u64::MAX - 0 + 1 wraps to 0 without u128). - let values: Vec = vec![0, u64::MAX]; - let segment = U64Segment::from_slice(&values); - assert!( - matches!(segment, U64Segment::SortedArray(_)), - "full u64 span should be SortedArray, got {:?}", - std::mem::discriminant(&segment) - ); - assert_eq!(segment.len(), 2); - assert_eq!(segment.iter().collect::>(), values); - - // Small dense set near u64::MAX — cost estimation correctly prefers a - // range-backed encoding, but Range cannot represent u64::MAX + 1 - // as the exclusive end. Must fall back to SortedArray. - let values: Vec = vec![u64::MAX - 3, u64::MAX - 1, u64::MAX]; - let segment = U64Segment::from_slice(&values); - assert!( - matches!(segment, U64Segment::SortedArray(_)), - "dense set near u64::MAX should be SortedArray (exclusive end unrepresentable), got {:?}", - std::mem::discriminant(&segment) - ); - assert_eq!(segment.len(), 3); - assert_eq!(segment.iter().collect::>(), values); - - // Single value at u64::MAX — contiguous range with n_holes == 0, but - // exclusive end u64::MAX + 1 overflows. - let values: Vec = vec![u64::MAX]; - let segment = U64Segment::from_slice(&values); - assert!( - matches!(segment, U64Segment::SortedArray(_)), - "single u64::MAX should be SortedArray, got {:?}", - std::mem::discriminant(&segment) - ); - assert_eq!(segment.len(), 1); - assert_eq!(segment.iter().collect::>(), values); - - // Contiguous range ending just below u64::MAX — exclusive end is - // representable, so Range encoding should still be used. - let values: Vec = vec![u64::MAX - 3, u64::MAX - 2, u64::MAX - 1]; - let segment = U64Segment::from_slice(&values); - assert_eq!(segment, U64Segment::Range((u64::MAX - 3)..u64::MAX)); - assert_eq!(segment.len(), 3); - assert_eq!(segment.iter().collect::>(), values); - - // Regression: normal dense range with few holes still picks RangeWithHoles. - // Needs total_slots > 32 * n_holes for RangeWithHoles to beat RangeWithBitmap. - let values: Vec = (100..1100).filter(|&x| x != 500).collect(); - let segment = U64Segment::from_slice(&values); - assert_eq!( - segment, - U64Segment::RangeWithHoles { - range: 100..1100, - holes: vec![500].into(), - } - ); - assert_eq!(segment.len(), 999); - assert_eq!(segment.iter().collect::>(), values); - - // Regression: small dense range with hole picks RangeWithBitmap. - let values: Vec = vec![100, 101, 102, 103, 105]; - let segment = U64Segment::from_slice(&values); - assert!( - matches!(segment, U64Segment::RangeWithBitmap { .. }), - "small dense range with hole should be RangeWithBitmap, got {:?}", - std::mem::discriminant(&segment) - ); - assert_eq!(segment.len(), 5); - assert_eq!(segment.iter().collect::>(), values); - } - - #[test] - fn test_u128_byte_cost_to_usize() { - assert_eq!(super::u128_byte_cost_to_usize(0), 0); - assert_eq!(super::u128_byte_cost_to_usize(42), 42); - assert_eq!( - super::u128_byte_cost_to_usize(usize::MAX as u128), - usize::MAX - ); - assert_eq!(super::u128_byte_cost_to_usize(u128::MAX), usize::MAX); - } - - #[test] - fn test_sorted_sequence_sizes_sparse_span_saturates_range_with_holes_cost() { - let stats = super::SegmentStats { - min: 0, - max: i64::MAX as u64, - count: 5, - sorted: true, - }; - let sizes = U64Segment::sorted_sequence_sizes(&stats); - assert_eq!(sizes[0], usize::MAX); - assert!(sizes[2] < sizes[0]); - } - - #[test] - fn test_sorted_sequence_sizes_sorted_array_cost_saturates() { - // Nearly full [0, u64::MAX] with one hole: count = u64::MAX, n_holes = 1. - // SortedArray cost 24 + 2 * u64::MAX does not fit in usize on 64-bit. - let stats = super::SegmentStats { - min: 0, - max: u64::MAX, - count: u64::MAX, - sorted: true, - }; - let sizes = U64Segment::sorted_sequence_sizes(&stats); - assert_eq!(sizes[2], usize::MAX); - } - - #[test] - fn test_sorted_sequence_sizes_full_span_bitmap_cost() { - // Synthetic stats: full [0, u64::MAX] slot space; exercises `range_with_bitmap` - // cost path (always fits in `usize` on 64-bit targets). - let stats = super::SegmentStats { - min: 0, - max: u64::MAX, - count: 1, - sorted: true, - }; - let sizes = U64Segment::sorted_sequence_sizes(&stats); - assert!(sizes[1] < sizes[0]); - assert!(sizes[1] < usize::MAX); - } - - #[test] - fn test_with_new_high() { - // Test Range: contiguous sequence - let segment = U64Segment::Range(10..20); - - // Test adding value that extends the range - let result = segment.clone().with_new_high(20).unwrap(); - assert_eq!(result, U64Segment::Range(10..21)); - - // Test adding value that creates holes - let result = segment.with_new_high(25).unwrap(); - assert_eq!( - result, - U64Segment::RangeWithHoles { - range: 10..26, - holes: EncodedU64Array::U64(vec![20, 21, 22, 23, 24]), - } - ); - - // Test RangeWithHoles: sequence with existing holes - let segment = U64Segment::RangeWithHoles { - range: 10..20, - holes: EncodedU64Array::U64(vec![15, 17]), - }; - - // Test adding value that extends the range without new holes - let result = segment.clone().with_new_high(20).unwrap(); - assert_eq!( - result, - U64Segment::RangeWithHoles { - range: 10..21, - holes: EncodedU64Array::U64(vec![15, 17]), - } - ); - - // Test adding value that creates additional holes - let result = segment.with_new_high(25).unwrap(); - assert_eq!( - result, - U64Segment::RangeWithHoles { - range: 10..26, - holes: EncodedU64Array::U64(vec![15, 17, 20, 21, 22, 23, 24]), - } - ); - - // Test RangeWithBitmap: sequence with bitmap representation - let mut bitmap = Bitmap::new_full(10); - bitmap.clear(3); // Clear position 3 (value 13) - bitmap.clear(7); // Clear position 7 (value 17) - let segment = U64Segment::RangeWithBitmap { - range: 10..20, - bitmap, - }; - - // Test adding value that extends the range without new holes - let result = segment.clone().with_new_high(20).unwrap(); - let expected_bitmap = { - let mut b = Bitmap::new_full(11); - b.clear(3); // Clear position 3 (value 13) - b.clear(7); // Clear position 7 (value 17) - b - }; - assert_eq!( - result, - U64Segment::RangeWithBitmap { - range: 10..21, - bitmap: expected_bitmap, - } - ); - - // Test adding value that creates additional holes - let result = segment.with_new_high(25).unwrap(); - let expected_bitmap = { - let mut b = Bitmap::new_full(16); - b.clear(3); // Clear position 3 (value 13) - b.clear(7); // Clear position 7 (value 17) - // Clear positions 10-14 (values 20-24) - for i in 10..15 { - b.clear(i); - } - b - }; - assert_eq!( - result, - U64Segment::RangeWithBitmap { - range: 10..26, - bitmap: expected_bitmap, - } - ); - - // Test SortedArray: sparse sorted sequence - let segment = U64Segment::SortedArray(EncodedU64Array::U64(vec![1, 5, 10])); - - let result = segment.with_new_high(15).unwrap(); - assert_eq!( - result, - U64Segment::SortedArray(EncodedU64Array::U64(vec![1, 5, 10, 15])) - ); - - // Test Array: unsorted sequence - let segment = U64Segment::Array(EncodedU64Array::U64(vec![10, 5, 1])); - - let result = segment.with_new_high(15).unwrap(); - assert_eq!( - result, - U64Segment::Array(EncodedU64Array::U64(vec![10, 5, 1, 15])) - ); - - // Test edge cases - // Empty segment - let segment = U64Segment::Range(0..0); - let result = segment.with_new_high(5).unwrap(); - assert_eq!(result, U64Segment::Range(5..6)); - - // Single value segment - let segment = U64Segment::Range(42..43); - let result = segment.with_new_high(50).unwrap(); - assert_eq!( - result, - U64Segment::RangeWithHoles { - range: 42..51, - holes: EncodedU64Array::U64(vec![43, 44, 45, 46, 47, 48, 49]), - } - ); - } - - #[test] - fn test_with_new_high_assertion() { - let segment = U64Segment::Range(10..20); - // This should return an error because 15 is not higher than the current maximum 19 - let result = segment.with_new_high(15); - assert!(result.is_err()); - let error = result.unwrap_err(); - assert!( - error - .to_string() - .contains("New value 15 must be higher than current maximum 19") - ); - } - - #[test] - fn test_with_new_high_assertion_equal() { - let segment = U64Segment::Range(1..6); - // This should return an error because 5 is not higher than the current maximum 5 - let result = segment.with_new_high(5); - assert!(result.is_err()); - let error = result.unwrap_err(); - assert!( - error - .to_string() - .contains("New value 5 must be higher than current maximum 5") - ); - } - - #[test] - fn test_contains() { - // Test Range: contiguous sequence - let segment = U64Segment::Range(10..20); - assert!(segment.contains(10), "Should contain 10"); - assert!(segment.contains(15), "Should contain 15"); - assert!(segment.contains(19), "Should contain 19"); - assert!(!segment.contains(9), "Should not contain 9"); - assert!(!segment.contains(20), "Should not contain 20"); - assert!(!segment.contains(25), "Should not contain 25"); - - // Test RangeWithHoles: sequence with holes - let segment = U64Segment::RangeWithHoles { - range: 10..20, - holes: EncodedU64Array::U64(vec![15, 17]), - }; - assert!(segment.contains(10), "Should contain 10"); - assert!(segment.contains(14), "Should contain 14"); - assert!(!segment.contains(15), "Should not contain 15 (hole)"); - assert!(segment.contains(16), "Should contain 16"); - assert!(!segment.contains(17), "Should not contain 17 (hole)"); - assert!(segment.contains(18), "Should contain 18"); - assert!( - !segment.contains(20), - "Should not contain 20 (out of range)" - ); - - // Test RangeWithBitmap: sequence with bitmap - let mut bitmap = Bitmap::new_full(10); - bitmap.clear(3); // Clear position 3 (value 13) - bitmap.clear(7); // Clear position 7 (value 17) - let segment = U64Segment::RangeWithBitmap { - range: 10..20, - bitmap, - }; - assert!(segment.contains(10), "Should contain 10"); - assert!(segment.contains(12), "Should contain 12"); - assert!( - !segment.contains(13), - "Should not contain 13 (cleared in bitmap)" - ); - assert!(segment.contains(16), "Should contain 16"); - assert!( - !segment.contains(17), - "Should not contain 17 (cleared in bitmap)" - ); - assert!(segment.contains(19), "Should contain 19"); - assert!( - !segment.contains(20), - "Should not contain 20 (out of range)" - ); - - // Test SortedArray: sparse sorted sequence - let segment = U64Segment::SortedArray(EncodedU64Array::U64(vec![1, 5, 10])); - assert!(segment.contains(1), "Should contain 1"); - assert!(segment.contains(5), "Should contain 5"); - assert!(segment.contains(10), "Should contain 10"); - assert!(!segment.contains(0), "Should not contain 0"); - assert!(!segment.contains(3), "Should not contain 3"); - assert!(!segment.contains(15), "Should not contain 15"); - - // Test Array: unsorted sequence - let segment = U64Segment::Array(EncodedU64Array::U64(vec![10, 5, 1])); - assert!(segment.contains(1), "Should contain 1"); - assert!(segment.contains(5), "Should contain 5"); - assert!(segment.contains(10), "Should contain 10"); - assert!(!segment.contains(0), "Should not contain 0"); - assert!(!segment.contains(3), "Should not contain 3"); - assert!(!segment.contains(15), "Should not contain 15"); - - // Test empty segment - let segment = U64Segment::Range(0..0); - assert!( - !segment.contains(0), - "Empty segment should not contain anything" - ); - assert!( - !segment.contains(5), - "Empty segment should not contain anything" - ); - } -} diff --git a/vendor/lance-table/src/rowids/serde.rs b/vendor/lance-table/src/rowids/serde.rs deleted file mode 100644 index c087fa60..00000000 --- a/vendor/lance-table/src/rowids/serde.rs +++ /dev/null @@ -1,239 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use crate::{format::pb, rowids::bitmap::Bitmap}; -use lance_core::{Error, Result}; - -use super::{RowIdSequence, U64Segment, encoded_array::EncodedU64Array}; -use prost::Message; - -impl TryFrom for RowIdSequence { - type Error = Error; - - fn try_from(pb: pb::RowIdSequence) -> Result { - Ok(Self( - pb.segments - .into_iter() - .map(U64Segment::try_from) - .collect::>>()?, - )) - } -} - -impl TryFrom for U64Segment { - type Error = Error; - - fn try_from(pb: pb::U64Segment) -> Result { - use pb::u64_segment as pb_seg; - use pb::u64_segment::Segment::*; - match pb.segment { - Some(Range(pb_seg::Range { start, end })) => Ok(Self::Range(start..end)), - Some(RangeWithHoles(pb_seg::RangeWithHoles { start, end, holes })) => { - let holes = holes - .ok_or_else(|| Error::invalid_input("missing hole"))? - .try_into()?; - Ok(Self::RangeWithHoles { - range: start..end, - holes, - }) - } - Some(RangeWithBitmap(pb_seg::RangeWithBitmap { start, end, bitmap })) => { - Ok(Self::RangeWithBitmap { - range: start..end, - bitmap: Bitmap { - data: bitmap, - len: (end - start) as usize, - }, - }) - } - Some(SortedArray(array)) => Ok(Self::SortedArray(EncodedU64Array::try_from(array)?)), - Some(Array(array)) => Ok(Self::Array(EncodedU64Array::try_from(array)?)), - // TODO: why non-exhaustive? - // Some(_) => Err(Error::invalid_input("unknown segment type")), - None => Err(Error::invalid_input("missing segment type")), - } - } -} - -impl TryFrom for EncodedU64Array { - type Error = Error; - - fn try_from(pb: pb::EncodedU64Array) -> Result { - use pb::encoded_u64_array as pb_arr; - use pb::encoded_u64_array::Array::*; - match pb.array { - Some(U16Array(pb_arr::U16Array { base, offsets })) => { - assert!( - offsets.len() % 2 == 0, - "Must have even number of bytes to store u16 array" - ); - let offsets = offsets - .chunks_exact(2) - .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) - .collect(); - Ok(Self::U16 { base, offsets }) - } - Some(U32Array(pb_arr::U32Array { base, offsets })) => { - assert!( - offsets.len() % 4 == 0, - "Must have even number of bytes to store u32 array" - ); - let offsets = offsets - .chunks_exact(4) - .map(|chunk| u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) - .collect(); - Ok(Self::U32 { base, offsets }) - } - Some(U64Array(pb_arr::U64Array { values })) => { - assert!( - values.len() % 8 == 0, - "Must have even number of bytes to store u64 array" - ); - let values = values - .chunks_exact(8) - .map(|chunk| { - u64::from_le_bytes([ - chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], - chunk[7], - ]) - }) - .collect(); - Ok(Self::U64(values)) - } - // TODO: shouldn't this enum be non-exhaustive? - // Some(_) => Err(Error::invalid_input("unknown array type")), - None => Err(Error::invalid_input("missing array type")), - } - } -} - -impl From for pb::RowIdSequence { - fn from(sequence: RowIdSequence) -> Self { - Self { - segments: sequence.0.into_iter().map(pb::U64Segment::from).collect(), - } - } -} - -impl From for pb::U64Segment { - fn from(segment: U64Segment) -> Self { - match segment { - U64Segment::Range(range) => Self { - segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range { - start: range.start, - end: range.end, - })), - }, - U64Segment::RangeWithHoles { range, holes } => Self { - segment: Some(pb::u64_segment::Segment::RangeWithHoles( - pb::u64_segment::RangeWithHoles { - start: range.start, - end: range.end, - holes: Some(holes.into()), - }, - )), - }, - U64Segment::RangeWithBitmap { range, bitmap } => Self { - segment: Some(pb::u64_segment::Segment::RangeWithBitmap( - pb::u64_segment::RangeWithBitmap { - start: range.start, - end: range.end, - bitmap: bitmap.data, - }, - )), - }, - U64Segment::SortedArray(array) => Self { - segment: Some(pb::u64_segment::Segment::SortedArray(array.into())), - }, - U64Segment::Array(array) => Self { - segment: Some(pb::u64_segment::Segment::Array(array.into())), - }, - } - } -} - -impl From for pb::EncodedU64Array { - fn from(array: EncodedU64Array) -> Self { - match array { - EncodedU64Array::U16 { base, offsets } => Self { - array: Some(pb::encoded_u64_array::Array::U16Array( - pb::encoded_u64_array::U16Array { - base, - offsets: offsets - .iter() - .flat_map(|&offset| offset.to_le_bytes().to_vec()) - .collect(), - }, - )), - }, - EncodedU64Array::U32 { base, offsets } => Self { - array: Some(pb::encoded_u64_array::Array::U32Array( - pb::encoded_u64_array::U32Array { - base, - offsets: offsets - .iter() - .flat_map(|&offset| offset.to_le_bytes().to_vec()) - .collect(), - }, - )), - }, - EncodedU64Array::U64(values) => Self { - array: Some(pb::encoded_u64_array::Array::U64Array( - pb::encoded_u64_array::U64Array { - values: values - .iter() - .flat_map(|&value| value.to_le_bytes().to_vec()) - .collect(), - }, - )), - }, - } - } -} - -/// Serialize a rowid sequence to a buffer. -pub fn write_row_ids(sequence: &RowIdSequence) -> Vec { - let pb_sequence = pb::RowIdSequence::from(sequence.clone()); - pb_sequence.encode_to_vec() -} - -/// Deserialize a rowid sequence from some bytes. -pub fn read_row_ids(reader: &[u8]) -> Result { - let pb_sequence = pb::RowIdSequence::decode(reader)?; - RowIdSequence::try_from(pb_sequence) -} - -#[cfg(test)] -mod test { - use super::*; - use pretty_assertions::assert_eq; - - #[test] - fn test_write_read_row_ids() { - let mut sequence = RowIdSequence::from(0..20); - sequence.0.push(U64Segment::Range(30..100)); - sequence.0.push(U64Segment::RangeWithHoles { - range: 100..200, - holes: EncodedU64Array::U64(vec![104, 108, 150]), - }); - sequence.0.push(U64Segment::RangeWithBitmap { - range: 200..300, - bitmap: Bitmap::new_empty(100), - }); - sequence - .0 - .push(U64Segment::SortedArray(EncodedU64Array::U16 { - base: 200, - offsets: vec![1, 2, 3], - })); - sequence - .0 - .push(U64Segment::Array(EncodedU64Array::U64(vec![1, 2, 3]))); - - let serialized = write_row_ids(&sequence); - - let sequence2 = read_row_ids(&serialized).unwrap(); - - assert_eq!(sequence.0, sequence2.0); - } -} diff --git a/vendor/lance-table/src/rowids/version.rs b/vendor/lance-table/src/rowids/version.rs deleted file mode 100644 index 80f3d06d..00000000 --- a/vendor/lance-table/src/rowids/version.rs +++ /dev/null @@ -1,713 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Row version tracking for cross-version diff functionality -//! -//! This module provides data structures and functionality to track the latest -//! update version for each row in a Lance dataset, enabling efficient -//! cross-version diff operations. - -use std::sync::Arc; - -use deepsize::DeepSizeOf; -use lance_core::Error; -use lance_core::Result; -use prost::Message; -use serde::de::Deserializer; -use serde::ser::Serializer; -use serde::{Deserialize, Serialize}; - -use crate::format::{ExternalFile, Fragment, pb}; -use crate::rowids::segment::U64Segment; -use crate::rowids::{RowIdSequence, read_row_ids}; - -/// A run of identical versions over a contiguous span of row positions. -/// -/// Span is expressed as a U64Segment over row offsets (0..N within a fragment), -/// not over row IDs. This keeps the encoding aligned with RowIdSequence order -/// and enables zipped iteration without building a map. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] -pub struct RowDatasetVersionRun { - pub span: U64Segment, - pub version: u64, -} - -impl RowDatasetVersionRun { - /// Number of rows covered by this run. - pub fn len(&self) -> usize { - self.span.len() - } - - /// Whether this run covers no rows. - pub fn is_empty(&self) -> bool { - self.span.is_empty() - } - - /// The version value of this run. - pub fn version(&self) -> u64 { - self.version - } -} - -/// Sequence of dataset versions -/// -/// Stores version runs aligned to the positional order of RowIdSequence. -/// Provides sequential iterators and optional lightweight indexing for -/// efficient random access. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf, Default)] -pub struct RowDatasetVersionSequence { - pub runs: Vec, -} - -impl RowDatasetVersionSequence { - /// Create a new empty version sequence - pub fn new() -> Self { - Self { runs: Vec::new() } - } - - /// Create a version sequence with a single uniform run of `row_count` rows. - pub fn from_uniform_row_count(row_count: u64, version: u64) -> Self { - if row_count == 0 { - return Self::new(); - } - let run = RowDatasetVersionRun { - span: U64Segment::Range(0..row_count), - version, - }; - Self { runs: vec![run] } - } - - /// Number of rows tracked by this sequence (sum of run lengths). - pub fn len(&self) -> u64 { - self.runs.iter().map(|s| s.len() as u64).sum() - } - - /// Empty if there are no runs or all runs are empty. - pub fn is_empty(&self) -> bool { - self.runs.is_empty() || self.runs.iter().all(|s| s.is_empty()) - } - - /// Returns a forward iterator over versions, expanding runs lazily. - pub fn versions(&self) -> VersionsIter<'_> { - VersionsIter::new(&self.runs) - } - - /// Random access: get the version at global row position `index`. - pub fn version_at(&self, index: usize) -> Option { - let mut offset = 0usize; - for run in &self.runs { - let len = run.len(); - if index < offset + len { - return Some(run.version()); - } - offset += len; - } - None - } - - /// Get the version associated with a specific row id. - /// This reconstructs the positional offset from RowIdSequence and then - /// performs `version_at` lookup. - pub fn get_version_for_row_id(&self, row_ids: &RowIdSequence, row_id: u64) -> Option { - let mut offset = 0usize; - for seg in &row_ids.0 { - if seg.range().is_some_and(|r| r.contains(&row_id)) - && let Some(local) = seg.position(row_id) - { - return self.version_at(offset + local); - } - offset += seg.len(); - } - None - } - - /// Convenience: collect row IDs with version strictly greater than `threshold`. - pub fn rows_with_version_greater_than( - &self, - row_ids: &RowIdSequence, - threshold: u64, - ) -> Vec { - row_ids - .iter() - .zip(self.versions()) - .filter_map(|(rid, v)| if v > threshold { Some(rid) } else { None }) - .collect() - } - - /// Delete rows by positional offsets (e.g., from a deletion vector) - pub fn mask(&mut self, positions: impl IntoIterator) -> Result<()> { - let mut local_positions: Vec = Vec::new(); - let mut positions_iter = positions.into_iter(); - let mut curr_position = positions_iter.next(); - let mut offset: usize = 0; - let mut cutoff: usize = 0; - - for run in self.runs.iter_mut() { - cutoff += run.span.len(); - while let Some(position) = curr_position { - if position as usize >= cutoff { - break; - } - local_positions.push(position - offset as u32); - curr_position = positions_iter.next(); - } - - if !local_positions.is_empty() { - run.span.mask(local_positions.as_slice()); - local_positions.clear(); - } - offset = cutoff; - } - - self.runs.retain(|r| !r.span.is_empty()); - Ok(()) - } -} - -/// Iterator over versions expanding runs lazily. -pub struct VersionsIter<'a> { - runs: &'a [RowDatasetVersionRun], - run_idx: usize, - remaining_in_run: usize, - current_version: u64, -} - -impl<'a> VersionsIter<'a> { - fn new(runs: &'a [RowDatasetVersionRun]) -> Self { - let mut it = Self { - runs, - run_idx: 0, - remaining_in_run: 0, - current_version: 0, - }; - it.advance_run(); - it - } - - fn advance_run(&mut self) { - if self.run_idx < self.runs.len() { - let run = &self.runs[self.run_idx]; - self.remaining_in_run = run.len(); - self.current_version = run.version(); - } else { - self.remaining_in_run = 0; - } - } -} - -impl<'a> Iterator for VersionsIter<'a> { - type Item = u64; - - fn next(&mut self) -> Option { - if self.remaining_in_run == 0 { - // Move to next run - self.run_idx += 1; - if self.run_idx >= self.runs.len() { - return None; - } - self.advance_run(); - } - self.remaining_in_run = self.remaining_in_run.saturating_sub(1); - Some(self.current_version) - } -} - -/// Metadata about the location of dataset version sequence data -/// Following the same pattern as RowIdMeta -/// -/// When stored inline, identical byte sequences are shared across fragments -/// via `Arc<[u8]>` to reduce manifest memory for large tables. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] -pub enum RowDatasetVersionMeta { - /// Small sequences stored inline in the fragment metadata - Inline(Arc<[u8]>), - /// Large sequences stored in external files - External(ExternalFile), -} - -// Custom Serialize: convert Arc<[u8]> to slice for transparent JSON output -impl Serialize for RowDatasetVersionMeta { - fn serialize(&self, serializer: S) -> std::result::Result { - #[derive(Serialize)] - #[serde(untagged)] - enum Helper<'a> { - Inline { inline: &'a [u8] }, - External { external: &'a ExternalFile }, - } - - match self { - Self::Inline(data) => Helper::Inline { - inline: data.as_ref(), - } - .serialize(serializer), - Self::External(file) => Helper::External { external: file }.serialize(serializer), - } - } -} - -// Custom Deserialize: read Vec and convert to Arc<[u8]> -impl<'de> Deserialize<'de> for RowDatasetVersionMeta { - fn deserialize>(deserializer: D) -> std::result::Result { - #[derive(Deserialize)] - #[serde(untagged)] - enum Helper { - Inline { inline: Vec }, - External { external: ExternalFile }, - } - - match Helper::deserialize(deserializer)? { - Helper::Inline { inline } => Ok(Self::Inline(Arc::from(inline))), - Helper::External { external } => Ok(Self::External(external)), - } - } -} - -impl RowDatasetVersionMeta { - /// Create inline metadata from a version sequence - pub fn from_sequence(sequence: &RowDatasetVersionSequence) -> lance_core::Result { - let bytes = write_dataset_versions(sequence); - Ok(Self::Inline(Arc::from(bytes))) - } - - /// Create external metadata reference - pub fn from_external_file(path: String, offset: u64, size: u64) -> Self { - Self::External(ExternalFile { path, offset, size }) - } - - /// Load the version sequence from this metadata - pub fn load_sequence(&self) -> lance_core::Result { - match self { - Self::Inline(data) => read_dataset_versions(data), - Self::External(_file) => { - todo!("External file loading not yet implemented") - } - } - } -} - -/// Helper function to convert RowDatasetVersionMeta to protobuf format for last_updated_at -pub fn last_updated_at_version_meta_to_pb( - meta: &Option, -) -> Option { - meta.as_ref().map(|m| match m { - RowDatasetVersionMeta::Inline(data) => { - pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( - data.to_vec(), - ) - } - RowDatasetVersionMeta::External(file) => { - pb::data_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( - pb::ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - }, - ) - } - }) -} - -/// Helper function to convert RowDatasetVersionMeta to protobuf format for created_at -pub fn created_at_version_meta_to_pb( - meta: &Option, -) -> Option { - meta.as_ref().map(|m| match m { - RowDatasetVersionMeta::Inline(data) => { - pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data.to_vec()) - } - RowDatasetVersionMeta::External(file) => { - pb::data_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions( - pb::ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - }, - ) - } - }) -} - -/// Serialize a dataset version sequence to a buffer (following RowIdSequence pattern) -pub fn write_dataset_versions(sequence: &RowDatasetVersionSequence) -> Vec { - // Convert to protobuf sequence - let pb_sequence = pb::RowDatasetVersionSequence { - runs: sequence - .runs - .iter() - .map(|run| pb::RowDatasetVersionRun { - span: Some(pb::U64Segment::from(run.span.clone())), - version: run.version, - }) - .collect(), - }; - - pb_sequence.encode_to_vec() -} - -/// Deserialize a dataset version sequence from bytes (following RowIdSequence pattern) -pub fn read_dataset_versions(data: &[u8]) -> lance_core::Result { - let pb_sequence = pb::RowDatasetVersionSequence::decode(data).map_err(|e| { - Error::internal(format!("Failed to decode RowDatasetVersionSequence: {}", e)) - })?; - - let segments = pb_sequence - .runs - .into_iter() - .map(|pb_run| { - let positions_pb = pb_run.span.ok_or_else(|| { - Error::internal("Missing positions in RowDatasetVersionRun".to_string()) - })?; - let segment = U64Segment::try_from(positions_pb)?; - Ok(RowDatasetVersionRun { - span: segment, - version: pb_run.version, - }) - }) - .collect::>>()?; - - Ok(RowDatasetVersionSequence { runs: segments }) -} - -/// Re-chunk a sequence of dataset version runs into new chunk sizes (aligned with RowIdSequence rechunking) -pub fn rechunk_version_sequences( - sequences: impl IntoIterator, - chunk_sizes: impl IntoIterator, - allow_incomplete: bool, -) -> Result> { - let chunk_sizes_vec: Vec = chunk_sizes.into_iter().collect(); - let total_chunks = chunk_sizes_vec.len(); - let mut chunked_sequences: Vec = Vec::with_capacity(total_chunks); - - let mut run_iter = sequences - .into_iter() - .flat_map(|sequence| sequence.runs.into_iter()) - .peekable(); - - let too_few_segments_error = |chunk_index: usize, expected_chunk_size: u64, remaining: u64| { - Error::invalid_input(format!( - "Got too few version runs for chunk {}. Expected chunk size: {}, remaining needed: {}", - chunk_index, expected_chunk_size, remaining - )) - }; - - let too_many_segments_error = |processed_chunks: usize, total_chunk_sizes: usize| { - Error::invalid_input(format!( - "Got too many version runs for the provided chunk lengths. Processed {} chunks out of {} expected", - processed_chunks, total_chunk_sizes - )) - }; - - let mut segment_offset = 0_u64; - - for (chunk_index, chunk_size) in chunk_sizes_vec.iter().enumerate() { - let chunk_size = *chunk_size; - let mut out_seq = RowDatasetVersionSequence::new(); - let mut remaining = chunk_size; - - while remaining > 0 { - let remaining_in_segment = run_iter - .peek() - .map_or(0, |run| run.span.len() as u64 - segment_offset); - - if remaining_in_segment == 0 { - if run_iter.next().is_some() { - segment_offset = 0; - continue; - } else if allow_incomplete { - break; - } else { - return Err(too_few_segments_error(chunk_index, chunk_size, remaining)); - } - } - - match remaining_in_segment.cmp(&remaining) { - std::cmp::Ordering::Greater => { - let run = run_iter.peek().unwrap(); - let seg = run.span.slice(segment_offset as usize, remaining as usize); - out_seq.runs.push(RowDatasetVersionRun { - span: seg, - version: run.version, - }); - segment_offset += remaining; - remaining = 0; - } - std::cmp::Ordering::Equal | std::cmp::Ordering::Less => { - let run = run_iter.next().ok_or_else(|| { - too_few_segments_error(chunk_index, chunk_size, remaining) - })?; - let seg = run - .span - .slice(segment_offset as usize, remaining_in_segment as usize); - out_seq.runs.push(RowDatasetVersionRun { - span: seg, - version: run.version, - }); - segment_offset = 0; - remaining -= remaining_in_segment; - } - } - } - - chunked_sequences.push(out_seq); - } - - if run_iter.peek().is_some() { - return Err(too_many_segments_error( - chunked_sequences.len(), - total_chunks, - )); - } - - Ok(chunked_sequences) -} - -/// Build version metadata for a fragment if it has physical rows and no existing metadata. -pub fn build_version_meta( - fragment: &Fragment, - current_version: u64, -) -> Option { - if let Some(physical_rows) = fragment.physical_rows - && physical_rows > 0 - { - // Verify row_id_meta exists (sanity check for stable row IDs) - if fragment.row_id_meta.is_none() { - panic!("Can not find row id meta, please make sure you have enabled stable row id.") - } - - // Use physical_rows directly as the authoritative row count - // This is correct even for compacted fragments where row_id_meta might - // have been partially copied - let version_sequence = RowDatasetVersionSequence::from_uniform_row_count( - physical_rows as u64, - current_version, - ); - - return Some(RowDatasetVersionMeta::from_sequence(&version_sequence).unwrap()); - } - None -} - -/// Refresh row-level latest update version metadata for a full fragment rewrite-column update. -/// -/// This sets a uniform version sequence for all rows in the fragment to `current_version`. -pub fn refresh_row_latest_update_meta_for_full_frag_rewrite_cols( - fragment: &mut Fragment, - current_version: u64, -) -> Result<()> { - let row_count = if let Some(pr) = fragment.physical_rows { - pr as u64 - } else if let Some(row_id_meta) = fragment.row_id_meta.as_ref() { - match row_id_meta { - crate::format::RowIdMeta::Inline(data) => { - let sequence = read_row_ids(data).unwrap(); - sequence.len() - } - // Follow existing behavior: external sequence not yet supported here - crate::format::RowIdMeta::External(_file) => 0, - } - } else { - 0 - }; - - if row_count > 0 { - let version_seq = - RowDatasetVersionSequence::from_uniform_row_count(row_count, current_version); - let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq)?; - fragment.last_updated_at_version_meta = Some(version_meta); - } - - Ok(()) -} - -/// Refresh row-level latest update version metadata for a partial fragment rewrite-column update. -/// -/// `updated_offsets` are local row offsets (within the fragment) that have been updated. -/// Existing version metadata is preserved and only the updated positions are set to `current_version`. -/// If no existing metadata is present, positions default to `prev_version`. -pub fn refresh_row_latest_update_meta_for_partial_frag_rewrite_cols( - fragment: &mut Fragment, - updated_offsets: &[usize], - current_version: u64, - prev_version: u64, -) -> Result<()> { - // Determine row count for fragment - let row_count_u64: u64 = if let Some(pr) = fragment.physical_rows { - pr as u64 - } else if let Some(row_id_meta) = fragment.row_id_meta.as_ref() { - match row_id_meta { - crate::format::RowIdMeta::Inline(data) => { - let sequence = read_row_ids(data).unwrap(); - sequence.len() - } - crate::format::RowIdMeta::External(_file) => { - // Preserve original behavior for external sequences - todo!("External file loading not yet implemented") - } - } - } else { - 0 - }; - - if row_count_u64 > 0 { - // Build base version vector from existing meta or previous dataset version - let mut base_versions: Vec = Vec::with_capacity(row_count_u64 as usize); - if let Some(meta) = fragment.last_updated_at_version_meta.as_ref() { - if let Ok(base_seq) = meta.load_sequence() { - for pos in 0..(row_count_u64 as usize) { - base_versions.push(base_seq.version_at(pos).unwrap_or(prev_version)); - } - } else { - base_versions.resize(row_count_u64 as usize, prev_version); - } - } else { - base_versions.resize(row_count_u64 as usize, prev_version); - } - - // Apply updates to updated positions - for &pos in updated_offsets { - if pos < base_versions.len() { - base_versions[pos] = current_version; - } - } - - // Compress into runs - let mut runs: Vec = Vec::new(); - if !base_versions.is_empty() { - let mut start = 0usize; - let mut curr_ver = base_versions[0]; - for (idx, &ver) in base_versions.iter().enumerate().skip(1) { - if ver != curr_ver { - runs.push(RowDatasetVersionRun { - span: U64Segment::Range(start as u64..idx as u64), - version: curr_ver, - }); - start = idx; - curr_ver = ver; - } - } - runs.push(RowDatasetVersionRun { - span: U64Segment::Range(start as u64..base_versions.len() as u64), - version: curr_ver, - }); - } - let new_seq = RowDatasetVersionSequence { runs }; - let new_meta = RowDatasetVersionMeta::from_sequence(&new_seq)?; - fragment.last_updated_at_version_meta = Some(new_meta); - } - - Ok(()) -} - -// Protobuf conversion implementations -impl TryFrom for RowDatasetVersionMeta { - type Error = Error; - - fn try_from(value: pb::data_fragment::LastUpdatedAtVersionSequence) -> Result { - match value { - pb::data_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions(data) => { - Ok(Self::Inline(Arc::from(data))) - } - pb::data_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( - file, - ) => Ok(Self::External(ExternalFile { - path: file.path, - offset: file.offset, - size: file.size, - })), - } - } -} - -impl TryFrom for RowDatasetVersionMeta { - type Error = Error; - - fn try_from(value: pb::data_fragment::CreatedAtVersionSequence) -> Result { - match value { - pb::data_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => { - Ok(Self::Inline(Arc::from(data))) - } - pb::data_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => { - Ok(Self::External(ExternalFile { - path: file.path, - offset: file.offset, - size: file.size, - })) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_version_random_access() { - let seq = RowDatasetVersionSequence { - runs: vec![ - RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 1, - }, - RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 2, - }, - RowDatasetVersionRun { - span: U64Segment::Range(0..1), - version: 3, - }, - ], - }; - assert_eq!(seq.version_at(0), Some(1)); - assert_eq!(seq.version_at(2), Some(1)); - assert_eq!(seq.version_at(3), Some(2)); - assert_eq!(seq.version_at(4), Some(2)); - assert_eq!(seq.version_at(5), Some(3)); - assert_eq!(seq.version_at(6), None); - } - - #[test] - fn test_serialization_round_trip() { - let seq = RowDatasetVersionSequence { - runs: vec![ - RowDatasetVersionRun { - span: U64Segment::Range(0..4), - version: 42, - }, - RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 99, - }, - ], - }; - let bytes = write_dataset_versions(&seq); - let seq2 = read_dataset_versions(&bytes).unwrap(); - assert_eq!(seq2.runs.len(), 2); - assert_eq!(seq2.len(), 7); - assert_eq!(seq2.version_at(0), Some(42)); - assert_eq!(seq2.version_at(5), Some(99)); - } - - #[test] - fn test_get_version_for_row_id() { - let seq = RowDatasetVersionSequence { - runs: vec![ - RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 8, - }, - RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 9, - }, - ], - }; - let rows = RowIdSequence::from(10..14); // row ids: 10,11,12,13 - assert_eq!(seq.get_version_for_row_id(&rows, 10), Some(8)); - assert_eq!(seq.get_version_for_row_id(&rows, 11), Some(8)); - assert_eq!(seq.get_version_for_row_id(&rows, 12), Some(9)); - assert_eq!(seq.get_version_for_row_id(&rows, 13), Some(9)); - assert_eq!(seq.get_version_for_row_id(&rows, 99), None); - } -} diff --git a/vendor/lance-table/src/utils.rs b/vendor/lance-table/src/utils.rs deleted file mode 100644 index 01c64f78..00000000 --- a/vendor/lance-table/src/utils.rs +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -pub mod stream; - -pub trait LanceIteratorExtension { - fn exact_size(self, size: usize) -> ExactSize - where - Self: Sized; -} - -impl LanceIteratorExtension for I { - fn exact_size(self, size: usize) -> ExactSize - where - Self: Sized, - { - ExactSize { inner: self, size } - } -} - -/// A iterator that is tagged with a known size. This is useful when we are -/// able to pre-compute the size of the iterator but the iterator implementation -/// isn't able to itself. A common example is when using `flatten()`. -/// -/// This is inspired by discussion in -pub struct ExactSize { - inner: I, - size: usize, -} - -impl Iterator for ExactSize { - type Item = I::Item; - - fn next(&mut self) -> Option { - match self.inner.next() { - None => None, - Some(x) => { - self.size -= 1; - Some(x) - } - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.size, Some(self.size)) - } -} diff --git a/vendor/lance-table/src/utils/stream.rs b/vendor/lance-table/src/utils/stream.rs deleted file mode 100644 index f6fbbd45..00000000 --- a/vendor/lance-table/src/utils/stream.rs +++ /dev/null @@ -1,806 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::sync::Arc; - -use arrow_array::{BooleanArray, RecordBatch, RecordBatchOptions, UInt64Array, make_array}; -use arrow_buffer::NullBuffer; -use futures::{ - FutureExt, Stream, StreamExt, - future::BoxFuture, - stream::{BoxStream, FuturesOrdered}, -}; -use lance_arrow::RecordBatchExt; -use lance_core::{ - ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, - ROW_LAST_UPDATED_AT_VERSION_FIELD, Result, - utils::{address::RowAddress, deletion::DeletionVector}, -}; -use lance_io::ReadBatchParams; -use tracing::instrument; - -use crate::rowids::RowIdSequence; - -pub type ReadBatchFut = BoxFuture<'static, Result>; -/// A task, emitted by a file reader, that will produce a batch (of the -/// given size) -pub struct ReadBatchTask { - pub task: ReadBatchFut, - pub num_rows: u32, -} -pub type ReadBatchTaskStream = BoxStream<'static, ReadBatchTask>; -pub type ReadBatchFutStream = BoxStream<'static, ReadBatchFut>; - -struct MergeStream { - streams: Vec, - next_batch: FuturesOrdered, - next_num_rows: u32, - index: usize, -} - -impl MergeStream { - fn emit(&mut self) -> ReadBatchTask { - let mut iter = std::mem::take(&mut self.next_batch); - let task = async move { - let mut batch = iter.next().await.unwrap()?; - while let Some(next) = iter.next().await { - let next = next?; - batch = batch.merge(&next)?; - } - Ok(batch) - } - .boxed(); - let num_rows = self.next_num_rows; - self.next_num_rows = 0; - ReadBatchTask { task, num_rows } - } -} - -impl Stream for MergeStream { - type Item = ReadBatchTask; - - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - loop { - let index = self.index; - match self.streams[index].poll_next_unpin(cx) { - std::task::Poll::Ready(Some(batch_task)) => { - if self.index == 0 { - self.next_num_rows = batch_task.num_rows; - } else { - debug_assert_eq!(self.next_num_rows, batch_task.num_rows); - } - self.next_batch.push_back(batch_task.task); - self.index += 1; - if self.index == self.streams.len() { - self.index = 0; - let next_batch = self.emit(); - return std::task::Poll::Ready(Some(next_batch)); - } - } - std::task::Poll::Ready(None) => { - return std::task::Poll::Ready(None); - } - std::task::Poll::Pending => { - return std::task::Poll::Pending; - } - } - } - } -} - -/// Given multiple streams of batch tasks, merge them into a single stream -/// -/// This pulls one batch from each stream and then combines the columns from -/// all of the batches into a single batch. The order of the batches in the -/// streams is maintained and the merged batch columns will be in order from -/// first to last stream. -/// -/// This stream ends as soon as any of the input streams ends (we do not -/// verify that the other input streams are finished as well) -/// -/// This will panic if any of the input streams return a batch with a different -/// number of rows than the first stream. -pub fn merge_streams(streams: Vec) -> ReadBatchTaskStream { - MergeStream { - streams, - next_batch: FuturesOrdered::new(), - next_num_rows: 0, - index: 0, - } - .boxed() -} - -/// Apply a mask to the batch, where rows are "deleted" by the _rowid column null. -/// -/// This is used partly as a performance optimization (cheaper to null than to filter) -/// but also because there are cases where we want to load the physical rows. For example, -/// we may be replacing a column based on some UDF and we want to provide a value for the -/// deleted rows to ensure the fragments are aligned. -fn apply_deletions_as_nulls(batch: RecordBatch, mask: &BooleanArray) -> Result { - // Transform mask into null buffer. Null means deleted, though note that - // null buffers are actually validity buffers, so True means not null - // and thus not deleted. - let mask_buffer = NullBuffer::new(mask.values().clone()); - - if mask_buffer.null_count() == 0 { - // No rows are deleted - return Ok(batch); - } - - // For each column convert to data - let new_columns = batch - .schema() - .fields() - .iter() - .zip(batch.columns()) - .map(|(field, col)| { - if field.name() == ROW_ID || field.name() == ROW_ADDR { - let col_data = col.to_data(); - // If it already has a validity bitmap, then AND it with the mask. - // Otherwise, use the boolean buffer as the mask. - let null_buffer = NullBuffer::union(col_data.nulls(), Some(&mask_buffer)); - - Ok(col_data - .into_builder() - .null_bit_buffer(null_buffer.map(|b| b.buffer().clone())) - .build() - .map(make_array)?) - } else { - Ok(col.clone()) - } - }) - .collect::>>()?; - - Ok(RecordBatch::try_new_with_options( - batch.schema(), - new_columns, - &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), - )?) -} - -/// Extract version values for a batch selection by binary-searching over -/// precomputed RLE run offsets. Single-run fragments (the common case) -/// take the O(1) fast path. -fn version_values_for_selection( - sequence: &crate::rowids::version::RowDatasetVersionSequence, - params: &ReadBatchParams, - batch_offset: u32, - num_rows: u32, -) -> Result> { - let selection = params - .slice(batch_offset as usize, num_rows as usize) - .unwrap() - .to_ranges() - .unwrap(); - - if sequence.runs.len() == 1 { - return Ok(vec![sequence.runs[0].version(); num_rows as usize]); - } - - let mut versions = Vec::with_capacity(num_rows as usize); - let run_offsets: Vec = sequence - .runs - .iter() - .scan(0usize, |acc, run| { - let start = *acc; - *acc += run.len(); - Some(start) - }) - .collect(); - let total_len: usize = sequence.runs.iter().map(|r| r.len()).sum(); - - for r in &selection { - for pos in r.start..r.end { - let pos = pos as usize; - if pos >= total_len { - return Err(lance_core::Error::internal(format!( - "version column position {} out of range (total_len={})", - pos, total_len - ))); - } - let run_idx = match run_offsets.binary_search(&pos) { - Ok(idx) => idx, - Err(idx) => idx - 1, - }; - versions.push(sequence.runs[run_idx].version()); - } - } - Ok(versions) -} - -/// Configuration needed to apply row ids and deletions to a batch -#[derive(Debug)] -pub struct RowIdAndDeletesConfig { - /// The row ids that were requested - pub params: ReadBatchParams, - /// Whether to include the row id column in the final batch - pub with_row_id: bool, - /// Whether to include the row address column in the final batch - pub with_row_addr: bool, - /// Whether to include the last updated at version column in the final batch - pub with_row_last_updated_at_version: bool, - /// Whether to include the created at version column in the final batch - pub with_row_created_at_version: bool, - /// An optional deletion vector to apply to the batch - pub deletion_vector: Option>, - /// An optional row id sequence to use for the row id column. - pub row_id_sequence: Option>, - /// The last_updated_at version sequence - pub last_updated_at_sequence: Option>, - /// The created_at version sequence - pub created_at_sequence: Option>, - /// Whether to make deleted rows null instead of filtering them out - pub make_deletions_null: bool, - /// The total number of rows that will be loaded - /// - /// This is needed to convert ReadbatchParams::RangeTo into a valid range - pub total_num_rows: u32, -} - -impl RowIdAndDeletesConfig { - fn has_system_cols(&self) -> bool { - self.with_row_id - || self.with_row_addr - || self.with_row_last_updated_at_version - || self.with_row_created_at_version - } -} - -#[instrument(level = "debug", skip_all)] -pub fn apply_row_id_and_deletes( - batch: RecordBatch, - batch_offset: u32, - fragment_id: u32, - config: &RowIdAndDeletesConfig, -) -> Result { - let mut deletion_vector = config.deletion_vector.as_ref(); - // Convert Some(NoDeletions) into None to simplify logic below - if let Some(deletion_vector_inner) = deletion_vector - && matches!(deletion_vector_inner.as_ref(), DeletionVector::NoDeletions) - { - deletion_vector = None; - } - let has_deletions = deletion_vector.is_some(); - debug_assert!(batch.num_columns() > 0 || config.has_system_cols() || has_deletions); - - // If row id sequence is None, then row id IS row address. - let should_fetch_row_addr = config.with_row_addr - || (config.with_row_id && config.row_id_sequence.is_none()) - || has_deletions; - - let num_rows = batch.num_rows() as u32; - - let row_addrs = - if should_fetch_row_addr { - let _rowaddrs = tracing::span!(tracing::Level::DEBUG, "fetch_row_addrs").entered(); - let mut row_addrs = Vec::with_capacity(num_rows as usize); - for offset_range in config - .params - .slice(batch_offset as usize, num_rows as usize) - .unwrap() - .iter_offset_ranges()? - { - row_addrs.extend(offset_range.map(|row_offset| { - u64::from(RowAddress::new_from_parts(fragment_id, row_offset)) - })); - } - - Some(Arc::new(UInt64Array::from(row_addrs))) - } else { - None - }; - - let row_ids = if config.with_row_id { - let _rowids = tracing::span!(tracing::Level::DEBUG, "fetch_row_ids").entered(); - if let Some(row_id_sequence) = &config.row_id_sequence { - let selection = config - .params - .slice(batch_offset as usize, num_rows as usize) - .unwrap() - .to_ranges() - .unwrap(); - let row_ids = row_id_sequence - .select( - selection - .iter() - .flat_map(|r| r.start as usize..r.end as usize), - ) - .collect::(); - Some(Arc::new(row_ids)) - } else { - // If we don't have a row id sequence, can assume the row ids are - // the same as the row addresses. - row_addrs.clone() - } - } else { - None - }; - - let span = tracing::span!(tracing::Level::DEBUG, "apply_deletions"); - let _enter = span.enter(); - let deletion_mask = deletion_vector.and_then(|v| { - let row_addrs: &[u64] = row_addrs.as_ref().unwrap().values(); - v.build_predicate(row_addrs.iter()) - }); - - let batch = if config.with_row_id { - let row_id_arr = row_ids.unwrap(); - batch.try_with_column(ROW_ID_FIELD.clone(), row_id_arr)? - } else { - batch - }; - - let batch = if config.with_row_addr { - let row_addr_arr = row_addrs.unwrap(); - batch.try_with_column(ROW_ADDR_FIELD.clone(), row_addr_arr)? - } else { - batch - }; - - // Add version columns if requested - let batch = if config.with_row_last_updated_at_version || config.with_row_created_at_version { - let mut batch = batch; - - if config.with_row_last_updated_at_version { - let version_arr = if let Some(sequence) = &config.last_updated_at_sequence { - Arc::new(UInt64Array::from(version_values_for_selection( - sequence, - &config.params, - batch_offset, - num_rows, - )?)) - } else { - // Default to version 1 if sequence not provided - Arc::new(UInt64Array::from(vec![1u64; num_rows as usize])) - }; - batch = - batch.try_with_column(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone(), version_arr)?; - } - - if config.with_row_created_at_version { - let version_arr = if let Some(sequence) = &config.created_at_sequence { - Arc::new(UInt64Array::from(version_values_for_selection( - sequence, - &config.params, - batch_offset, - num_rows, - )?)) - } else { - // Default to version 1 if sequence not provided - Arc::new(UInt64Array::from(vec![1u64; num_rows as usize])) - }; - batch = batch.try_with_column(ROW_CREATED_AT_VERSION_FIELD.clone(), version_arr)?; - } - - batch - } else { - batch - }; - - match (deletion_mask, config.make_deletions_null) { - (None, _) => Ok(batch), - (Some(mask), false) => Ok(arrow::compute::filter_record_batch(&batch, &mask)?), - (Some(mask), true) => Ok(apply_deletions_as_nulls(batch, &mask)?), - } -} - -/// Given a stream of batch tasks this function will add a row ids column (if requested) -/// and also apply a deletions vector to the batch. -/// -/// This converts from BatchTaskStream to BatchFutStream because, if we are applying a -/// deletion vector, it is impossible to know how many output rows we will have. -pub fn wrap_with_row_id_and_delete( - stream: ReadBatchTaskStream, - fragment_id: u32, - config: RowIdAndDeletesConfig, -) -> ReadBatchFutStream { - let config = Arc::new(config); - let mut offset = 0; - stream - .map(move |batch_task| { - let config = config.clone(); - let this_offset = offset; - let num_rows = batch_task.num_rows; - offset += num_rows; - batch_task - .task - .map(move |batch| { - apply_row_id_and_deletes(batch?, this_offset, fragment_id, config.as_ref()) - }) - .boxed() - }) - .boxed() -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use arrow::{array::AsArray, datatypes::UInt64Type}; - use arrow_array::{RecordBatch, UInt32Array, types::Int32Type}; - use arrow_schema::ArrowError; - use futures::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream}; - use lance_core::{ - ROW_ID, - utils::{address::RowAddress, deletion::DeletionVector}, - }; - use lance_datagen::{BatchCount, RowCount}; - use lance_io::{ReadBatchParams, stream::arrow_stream_to_lance_stream}; - use roaring::RoaringBitmap; - - use crate::utils::stream::ReadBatchTask; - - use super::RowIdAndDeletesConfig; - - fn batch_task_stream( - datagen_stream: BoxStream<'static, std::result::Result>, - ) -> super::ReadBatchTaskStream { - arrow_stream_to_lance_stream(datagen_stream) - .map(|batch| ReadBatchTask { - num_rows: batch.as_ref().unwrap().num_rows() as u32, - task: std::future::ready(batch).boxed(), - }) - .boxed() - } - - #[tokio::test] - async fn test_basic_zip() { - let left = batch_task_stream( - lance_datagen::gen_batch() - .col("x", lance_datagen::array::step::()) - .into_reader_stream(RowCount::from(100), BatchCount::from(10)) - .0, - ); - let right = batch_task_stream( - lance_datagen::gen_batch() - .col("y", lance_datagen::array::step::()) - .into_reader_stream(RowCount::from(100), BatchCount::from(10)) - .0, - ); - - let merged = super::merge_streams(vec![left, right]) - .map(|batch_task| batch_task.task) - .buffered(1) - .try_collect::>() - .await - .unwrap(); - - let expected = lance_datagen::gen_batch() - .col("x", lance_datagen::array::step::()) - .col("y", lance_datagen::array::step::()) - .into_reader_rows(RowCount::from(100), BatchCount::from(10)) - .collect::, ArrowError>>() - .unwrap(); - assert_eq!(merged, expected); - } - - async fn check_row_id(params: ReadBatchParams, expected: impl IntoIterator) { - let expected = Vec::from_iter(expected); - - for has_columns in [false, true] { - for fragment_id in [0, 10] { - // 100 rows across 10 batches of 10 rows - let mut datagen = lance_datagen::gen_batch(); - if has_columns { - datagen = datagen.col("x", lance_datagen::array::rand::()); - } - let data = batch_task_stream( - datagen - .into_reader_stream(RowCount::from(10), BatchCount::from(10)) - .0, - ); - - let config = RowIdAndDeletesConfig { - params: params.clone(), - with_row_id: true, - with_row_addr: false, - with_row_last_updated_at_version: false, - with_row_created_at_version: false, - deletion_vector: None, - row_id_sequence: None, - last_updated_at_sequence: None, - created_at_sequence: None, - make_deletions_null: false, - total_num_rows: 100, - }; - let stream = super::wrap_with_row_id_and_delete(data, fragment_id, config); - let batches = stream.buffered(1).try_collect::>().await.unwrap(); - - let mut offset = 0; - let expected = expected.clone(); - for batch in batches { - let actual_row_ids = - batch[ROW_ID].as_primitive::().values().to_vec(); - let expected_row_ids = expected[offset..offset + 10] - .iter() - .map(|row_offset| { - RowAddress::new_from_parts(fragment_id, *row_offset).into() - }) - .collect::>(); - assert_eq!(actual_row_ids, expected_row_ids); - offset += batch.num_rows(); - } - } - } - } - - #[tokio::test] - async fn test_row_id() { - let some_indices = (0..100).rev().collect::>(); - let some_indices_arr = UInt32Array::from(some_indices.clone()); - check_row_id(ReadBatchParams::RangeFull, 0..100).await; - check_row_id(ReadBatchParams::Indices(some_indices_arr), some_indices).await; - check_row_id(ReadBatchParams::Range(1000..1100), 1000..1100).await; - check_row_id( - ReadBatchParams::RangeFrom(std::ops::RangeFrom { start: 1000 }), - 1000..1100, - ) - .await; - check_row_id( - ReadBatchParams::RangeTo(std::ops::RangeTo { end: 1000 }), - 0..100, - ) - .await; - } - - #[tokio::test] - async fn test_deletes() { - let no_deletes: Option> = None; - let no_deletes_2 = Some(Arc::new(DeletionVector::NoDeletions)); - let delete_some_bitmap = Some(Arc::new(DeletionVector::Bitmap(RoaringBitmap::from_iter( - 0..35, - )))); - let delete_some_set = Some(Arc::new(DeletionVector::Set((0..35).collect()))); - - for deletion_vector in [ - no_deletes, - no_deletes_2, - delete_some_bitmap, - delete_some_set, - ] { - for has_columns in [false, true] { - for with_row_id in [false, true] { - for make_deletions_null in [false, true] { - for frag_id in [0, 1] { - let has_deletions = if let Some(dv) = &deletion_vector { - !matches!(dv.as_ref(), DeletionVector::NoDeletions) - } else { - false - }; - if !has_columns && !has_deletions && !with_row_id { - // This is an invalid case and should be prevented upstream, - // no meaningful work is being done! - continue; - } - if make_deletions_null && !with_row_id { - // This is an invalid case and should be prevented upstream - // we cannot make the row_id column null if it isn't present - continue; - } - - let mut datagen = lance_datagen::gen_batch(); - if has_columns { - datagen = - datagen.col("x", lance_datagen::array::rand::()); - } - // 100 rows across 10 batches of 10 rows - let data = batch_task_stream( - datagen - .into_reader_stream(RowCount::from(10), BatchCount::from(10)) - .0, - ); - - let config = RowIdAndDeletesConfig { - params: ReadBatchParams::RangeFull, - with_row_id, - with_row_addr: false, - with_row_last_updated_at_version: false, - with_row_created_at_version: false, - deletion_vector: deletion_vector.clone(), - row_id_sequence: None, - last_updated_at_sequence: None, - created_at_sequence: None, - make_deletions_null, - total_num_rows: 100, - }; - let stream = super::wrap_with_row_id_and_delete(data, frag_id, config); - let batches = stream - .buffered(1) - .filter_map(|batch| { - std::future::ready( - batch - .map(|batch| { - if batch.num_rows() == 0 { - None - } else { - Some(batch) - } - }) - .transpose(), - ) - }) - .try_collect::>() - .await - .unwrap(); - - let total_num_rows = - batches.iter().map(|b| b.num_rows()).sum::(); - let total_num_nulls = if make_deletions_null { - batches - .iter() - .map(|b| b[ROW_ID].null_count()) - .sum::() - } else { - 0 - }; - let total_actually_deleted = total_num_nulls + (100 - total_num_rows); - - let expected_deletions = match &deletion_vector { - None => 0, - Some(deletion_vector) => match deletion_vector.as_ref() { - DeletionVector::NoDeletions => 0, - DeletionVector::Bitmap(b) => b.len() as usize, - DeletionVector::Set(s) => s.len(), - }, - }; - assert_eq!(total_actually_deleted, expected_deletions); - if expected_deletions > 0 && with_row_id { - if make_deletions_null { - // If we make deletions null we get 3 batches of all-null and then - // a batch of half-null - assert_eq!( - batches[3][ROW_ID].as_primitive::().value(0), - u64::from(RowAddress::new_from_parts(frag_id, 30)) - ); - assert_eq!(batches[3][ROW_ID].null_count(), 5); - } else { - // If we materialize deletions the first row will be 35 - assert_eq!( - batches[0][ROW_ID].as_primitive::().value(0), - u64::from(RowAddress::new_from_parts(frag_id, 35)) - ); - } - } - if !with_row_id { - assert!(batches[0].column_by_name(ROW_ID).is_none()); - } - } - } - } - } - } - } - - #[tokio::test] - async fn test_version_column_with_deletions() { - use crate::rowids::segment::U64Segment; - use crate::rowids::version::{RowDatasetVersionRun, RowDatasetVersionSequence}; - - let seq = Arc::new(RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..100), - version: 42, - }], - }); - - let data = batch_task_stream( - lance_datagen::gen_batch() - .col("x", lance_datagen::array::rand::()) - .into_reader_stream(RowCount::from(10), BatchCount::from(10)) - .0, - ); - - let config = RowIdAndDeletesConfig { - params: ReadBatchParams::RangeFull, - with_row_id: true, - with_row_addr: false, - with_row_last_updated_at_version: false, - with_row_created_at_version: true, - deletion_vector: Some(Arc::new(DeletionVector::Bitmap(RoaringBitmap::from_iter( - 0..35, - )))), - row_id_sequence: None, - last_updated_at_sequence: None, - created_at_sequence: Some(seq), - make_deletions_null: false, - total_num_rows: 100, - }; - let stream = super::wrap_with_row_id_and_delete(data, 0, config); - let batches: Vec<_> = stream - .buffered(1) - .try_filter(|b| std::future::ready(b.num_rows() > 0)) - .try_collect() - .await - .unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 65); - - for batch in &batches { - let versions = batch - .column_by_name("_row_created_at_version") - .unwrap() - .as_primitive::() - .values(); - assert!(versions.iter().all(|&v| v == 42)); - } - } - - #[tokio::test] - async fn test_version_column_multi_run() { - use crate::rowids::segment::U64Segment; - use crate::rowids::version::{RowDatasetVersionRun, RowDatasetVersionSequence}; - - // 3 runs: 0..40 v1, 40..70 v2, 70..100 v3 - let seq = Arc::new(RowDatasetVersionSequence { - runs: vec![ - RowDatasetVersionRun { - span: U64Segment::Range(0..40), - version: 1, - }, - RowDatasetVersionRun { - span: U64Segment::Range(40..70), - version: 2, - }, - RowDatasetVersionRun { - span: U64Segment::Range(70..100), - version: 3, - }, - ], - }); - - // Delete 0..20 and 60..80 (spans run boundary). - // Survivors: 20..40 (v1), 40..60 (v2), 80..100 (v3) = 60 rows - let mut deletions = RoaringBitmap::from_iter(0..20); - deletions.extend(60..80); - - let data = batch_task_stream( - lance_datagen::gen_batch() - .col("x", lance_datagen::array::rand::()) - .into_reader_stream(RowCount::from(10), BatchCount::from(10)) - .0, - ); - - let config = RowIdAndDeletesConfig { - params: ReadBatchParams::RangeFull, - with_row_id: true, - with_row_addr: false, - with_row_last_updated_at_version: false, - with_row_created_at_version: true, - deletion_vector: Some(Arc::new(DeletionVector::Bitmap(deletions))), - row_id_sequence: None, - last_updated_at_sequence: None, - created_at_sequence: Some(seq), - make_deletions_null: false, - total_num_rows: 100, - }; - let stream = super::wrap_with_row_id_and_delete(data, 0, config); - let batches: Vec<_> = stream - .buffered(1) - .try_filter(|b| std::future::ready(b.num_rows() > 0)) - .try_collect() - .await - .unwrap(); - - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 60); - - let all_versions: Vec = batches - .iter() - .flat_map(|b| { - b.column_by_name("_row_created_at_version") - .unwrap() - .as_primitive::() - .values() - .to_vec() - }) - .collect(); - - assert!(all_versions[..20].iter().all(|&v| v == 1)); - assert!(all_versions[20..40].iter().all(|&v| v == 2)); - assert!(all_versions[40..60].iter().all(|&v| v == 3)); - } -}