omnigraph/vendor/lance-table/protos/transaction.proto

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

syntax = "proto3";

import "file.proto";
import "table.proto";
import "google/protobuf/any.proto";

package lance.table;

// A transaction represents the changes to a dataset.
//
// This has two purposes:
// 1. When retrying a commit, the transaction can be used to re-build an updated
//    manifest.
// 2. When there's a conflict, this can be used to determine whether the other
//    transaction is compatible with this one.
message Transaction {
  // The version of the dataset this transaction was built from.
  //
  // For example, for a delete transaction this means the version of the dataset
  // that was read from while evaluating the deletion predicate.
  uint64 read_version = 1;

  // The UUID that unique identifies a transaction.
  string uuid = 2;

  // Optional version tag.
  string tag = 3;

  // Optional properties for the transaction
  // __lance_commit_message is a reserved key
  map<string, string> transaction_properties = 4;

  // Add new rows to the dataset.
  message Append {
    // The new fragments to append.
    //
    // Fragment IDs are not yet assigned.
    repeated DataFragment fragments = 1;
  }

  // Mark rows as deleted.
  message Delete {
    // The fragments to update
    //
    // The fragment IDs will match existing fragments in the dataset.
    repeated DataFragment updated_fragments = 1;
    // The fragments to delete entirely.
    repeated uint64 deleted_fragment_ids = 2;
    // The predicate that was evaluated
    //
    // This may be used to determine whether the delete would have affected 
    // files written by a concurrent transaction.
    string predicate = 3;
  }

  // Create or overwrite the entire dataset.
  message Overwrite {
    // The new fragments
    //
    // Fragment IDs are not yet assigned.
    repeated DataFragment fragments = 1;
    // The new schema
    repeated lance.file.Field schema = 2;
    // Schema metadata.
    map<string, bytes> schema_metadata = 3;
    // Key-value pairs to merge with existing config.
    map<string, string> config_upsert_values = 4;
    // The base paths to be added for the initial dataset creation
    repeated BasePath initial_bases = 5;
  }

  // Add or replace a new secondary index.
  //
  // This is also used to remove an index (we are replacing it with nothing)
  //
  // - new_indices: the modified indices, empty if dropping indices only
  // - removed_indices: the indices that are being replaced
  message CreateIndex {
    repeated IndexMetadata new_indices = 1;
    repeated IndexMetadata removed_indices = 2;
  }

  // An operation that rewrites but does not change the data in the table. These
  // kinds of operations just rearrange data.
  message Rewrite {
    // The old fragments that are being replaced
    //
    // DEPRECATED: use groups instead.
    //
    // These should all have existing fragment IDs.
    repeated DataFragment old_fragments = 1;
    // The new fragments
    //
    // DEPRECATED: use groups instead.
    //
    // These fragments IDs are not yet assigned.
    repeated DataFragment new_fragments = 2;

    // During a rewrite an index may be rewritten.  We only serialize the UUID
    // since a rewrite should not change the other index parameters.
    message RewrittenIndex {
      // The id of the index that will be replaced
      UUID old_id = 1;
      // the id of the new index
      UUID new_id = 2;
      // the new index details
      google.protobuf.Any new_index_details = 3;
      // the version of the new index
      uint32 new_index_version = 4;
      // Files in the new index with their sizes.
      // Empty if file sizes are not available (e.g. older writers).
      repeated IndexFile new_index_files = 5;
    }

    // A group of rewrite files that are all part of the same rewrite.
    message RewriteGroup {
      // The old fragment that is being replaced
      //
      // This should have an existing fragment ID.
      repeated DataFragment old_fragments = 1;
      // The new fragment
      //
      // The ID should have been reserved by an earlier
      // reserve operation
      repeated DataFragment new_fragments = 2;
    }

    // Groups of files that have been rewritten
    repeated RewriteGroup groups = 3;
    // Indices that have been rewritten
    repeated RewrittenIndex rewritten_indices = 4;
  }

  // An operation that merges in a new column, altering the schema.
  message Merge {
    // The updated fragments
    //
    // These should all have existing fragment IDs.
    repeated DataFragment fragments = 1;
    // The new schema
    repeated lance.file.Field schema = 2;
    // Schema metadata.
    map<string, bytes> schema_metadata = 3;
  }

  // An operation that projects a subset of columns, altering the schema.
  message Project {
    // The new schema
    repeated lance.file.Field schema = 1;
  }

  // An operation that restores a dataset to a previous version.
  message Restore {
    // The version to restore to
    uint64 version = 1;
  }

  // An operation that reserves fragment ids for future use in
  // a rewrite operation.
  message ReserveFragments {
    uint32 num_fragments = 1;
  }

  // An operation that clones a dataset.
  message Clone {
    // - true:  Performs a metadata-only clone (copies manifest without data files).
    //          The cloned dataset references original data through `base_paths`,
    //          suitable for experimental scenarios or rapid metadata migration.
    // - false: Performs a full deep clone using the underlying object storage's native
    //          copy API (e.g., S3 CopyObject, GCS rewrite). This leverages server-side
    //          bulk copy operations to bypass download/upload bottlenecks, achieving
    //          near-linear speedup for large datasets (typically 3-10x faster than
    //          manual file transfers). The operation maintains atomicity and data
    //          integrity guarantees provided by the storage backend.
    bool is_shallow = 1;
    // the reference name in the source dataset
    // in most cases it should be the branch or tag name in the source dataset
    optional string ref_name = 2;
    // the version of the source dataset for cloning
    uint64 ref_version = 3;
    // the absolute base path of the source dataset for cloning
    string ref_path = 4;
    // if the target dataset is a branch, this is the branch name of the target dataset
    optional string branch_name = 5;
  }
  
  // Exact set of key hashes for conflict detection.
  // Used when the number of inserted rows is small.
  message ExactKeySetFilter {
    // 64-bit hashes of the inserted row keys.
    repeated uint64 key_hashes = 1;
  }

  // Bloom filter for key existence tests.
  // Used when the number of rows is large.
  message BloomFilter {
    // Bitset backing the bloom filter (SBBF format).
    bytes bitmap = 1;
    // Number of bits in the bitmap.
    uint32 num_bits = 2;
    // Number of items the filter was sized for.
    // Used for intersection validation (filters with different sizes cannot be compared).
    // Default: 8192
    uint64 number_of_items = 3;
    // False positive probability the filter was sized for.
    // Used for intersection validation (filters with different parameters cannot be compared).
    // Default: 0.00057
    double probability = 4;
  }

  // A filter for checking key existence in set of rows inserted by a merge insert operation.
  // Only created when the merge insert's ON columns match the schema's unenforced primary key.
  // The presence of this filter indicates strict primary key conflict detection should be used.
  // Can use either an exact set (for small row counts) or a Bloom filter (for large row counts).
  message KeyExistenceFilter {
    // Field IDs of columns participating in the key (must match unenforced primary key).
    repeated int32 field_ids = 1;
    // The underlying data structure storing the key hashes.
    oneof data {
      // Exact set of key hashes (used for small number of rows).
      ExactKeySetFilter exact = 2;
      // Bloom filter (used for large number of rows).
      BloomFilter bloom = 3;
    }
  }

  // Serialized as sorted distinct local physical row offsets within the fragment (0-based).
  message UInt32List {
    repeated uint32 values = 1;
  }

  // An operation that updates rows but does not add or remove rows.
  message Update {
    // The fragments that have been removed. These are fragments where all rows
    // have been updated and moved to a new fragment.
    repeated uint64 removed_fragment_ids = 1;
    // The fragments that have been updated.
    repeated DataFragment updated_fragments = 2;
    // The new fragments where updated rows have been moved to.
    repeated DataFragment new_fragments = 3;
    // The ids of the fields that have been modified.
    repeated uint32 fields_modified = 4;
    /// List of MemWAL shard generations to mark as merged after this transaction
    repeated MergedGeneration merged_generations = 5;
    /// The fields that used to judge whether to preserve the new frag's id into
    /// the frag bitmap of the specified indices.
    repeated uint32 fields_for_preserving_frag_bitmap = 6;
    // The mode of update
    UpdateMode update_mode = 7;
    // Filter for checking existence of keys in newly inserted rows, used for conflict detection.
    // Only tracks keys from INSERT operations during merge insert, not updates.
    optional KeyExistenceFilter inserted_rows = 8;
    // Per-fragment physical row offsets that matched an update_columns hash join (RewriteColumns).
    map<uint64, UInt32List> updated_fragment_offsets = 9;
  }

  // The mode of update operation
  enum UpdateMode {

    /// rows are deleted in current fragments and rewritten in new fragments.
    /// This is most optimal when the majority of columns are being rewritten
    /// or only a few rows are being updated.
    REWRITE_ROWS = 0;

    /// within each fragment, columns are fully rewritten and inserted as new data files.
    /// Old versions of columns are tombstoned. This is most optimal when most rows are affected
    /// but a small subset of columns are affected.
    REWRITE_COLUMNS = 1;
  }

  // An entry for a map update. If value is not set, the key will be removed from the map.
  message UpdateMapEntry {
    // The key of the map entry to update.
    string key = 1;
    // The value to set for the key.
    optional string value = 2;
  }

  message UpdateMap {
    repeated UpdateMapEntry update_entries = 1;
    // If true, the map will be replaced entirely with the new entries.
    // If false, the new entries will be merged with the existing map.
    bool replace = 2;
  }
  
  // An operation that updates the table config, table metadata, schema metadata,
  // or field metadata.
  message UpdateConfig {
    UpdateMap config_updates = 6;
    UpdateMap table_metadata_updates = 7;
    UpdateMap schema_metadata_updates = 8;
    map<int32, UpdateMap> field_metadata_updates = 9;

    // Deprecated -------------------------------
    map<string, string> upsert_values = 1;
    repeated string delete_keys = 2;
    map<string, string> schema_metadata = 3;
    map<uint32, FieldMetadataUpdate> field_metadata = 4;

    message FieldMetadataUpdate {
      map<string, string> metadata = 5;
    }
  }

  message DataReplacementGroup {
    uint64 fragment_id = 1;
    DataFile new_file = 2;
  }

  // An operation that replaces the data in a region of the table with new data.
  message DataReplacement {
    repeated DataReplacementGroup replacements = 1;
  }

  // Update the merged generations in MemWAL index.
  // This operation is used during merge-insert to atomically record which
  // generations have been merged to the base table.
  message UpdateMemWalState {
    // Shards and generations being marked as merged.
    repeated MergedGeneration merged_generations = 1;
  }

  // An operation that updates base paths in the dataset.
  message UpdateBases {
    // The new base paths to add to the manifest.
    repeated BasePath new_bases = 1;
  }

  // The operation of this transaction.
  oneof operation {
    Append append = 100;
    Delete delete = 101;
    Overwrite overwrite = 102;
    CreateIndex create_index = 103;
    Rewrite rewrite = 104;
    Merge merge = 105;
    Restore restore = 106;
    ReserveFragments reserve_fragments = 107;
    Update update = 108;
    Project project = 109;
    UpdateConfig update_config = 110;
    DataReplacement data_replacement = 111;
    UpdateMemWalState update_mem_wal_state = 112;
    Clone clone = 113;
    UpdateBases update_bases = 114;
  }

  // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops.
  reserved 200, 202;
  reserved "blob_append", "blob_overwrite";
}