From e895ae141adc0d20f78678dae5039058cfafe35a Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Fri, 5 Jun 2026 21:07:57 -0700 Subject: [PATCH 01/83] docs: add MongoDB storage backend design document Architecture design for the extenddb-storage-mongodb crate covering: - Collection schema (catalog_db + data_db) - Document structure (_id, pk, sk_*, item_data) - Concurrency model (transactions + optimistic versioning) - GSI synchronous propagation strategy - Stream record storage - Bootstrapper and configuration --- docs/design/13-storage-mongodb.md | 753 ++++++++++++++++++++++++++++++ 1 file changed, 753 insertions(+) create mode 100644 docs/design/13-storage-mongodb.md diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md new file mode 100644 index 00000000..f3218db2 --- /dev/null +++ b/docs/design/13-storage-mongodb.md @@ -0,0 +1,753 @@ +# Design: MongoDB Storage Backend + +## 1. Overview + +The MongoDB backend (`extenddb-storage-mongodb`) implements the same trait surface as +`extenddb-storage-postgres`: the 6 engine traits (`TableEngine`, `DataEngine`, +`MetadataEngine`, `StreamEngine`, `BackupEngine`, `WorkerStore`) and the catalog +traits (`ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, +`RateLimitStore`, `AuthorizationStore`). + +**Driver:** `mongodb` (official Rust driver, async, supports multi-document ACID +transactions on replica sets). + +**Minimum MongoDB version:** 6.0 (for multi-document transactions and snapshot +reads). + +## 2. Database Layout + +Two databases, mirroring the PostgreSQL backend's catalog/data separation: + +| Database | Purpose | +|----------------------|------------------------------------------------------| +| `extenddb_catalog` | Table metadata, IAM, settings, metrics | +| `extenddb_data` | Per-table item collections, idempotency tokens | + +DynamoDB Streams are implemented via inline stream record writes during data +operations. GSI updates are propagated synchronously inline during writes. + +## 3. Catalog Database Collections + +### 3.1 `accounts` +```json +{ "_id": "", "account_name": "...", "created_at": ISODate } +``` +Unique index on `account_name`. + +### 3.2 `tables` +```json +{ + "_id": { "account_id": "...", "table_name": "..." }, + "key_schema": [...], + "attribute_definitions": [...], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": { ... }, + "stream_specification": { ... }, + "table_status": "ACTIVE", + "creation_date_time": ISODate, + "table_size_bytes": NumberLong, + "item_count": NumberLong, + "table_arn": "...", + "table_id": "...", + "ttl_attribute": null, + "deletion_protection_enabled": false, + "status_transition_at": null, + "stream_label": null, + "ttl_index_ready": false +} +``` +Unique index on `table_id`. Partial index on `status_transition_at` where not null. + +### 3.3 `indexes` +```json +{ + "_id": { "table_id": "...", "index_name": "..." }, + "index_id": "...", + "index_type": "GSI|LSI", + "key_schema": [...], + "projection": { ... }, + "index_status": "ACTIVE", + "provisioned_throughput": { ... } +} +``` + +### 3.4 `tags` +```json +{ "_id": { "resource_arn": "...", "tag_key": "..." }, "tag_value": "..." } +``` + +### 3.5 `settings` +```json +{ "_id": "", "value": "..." } +``` + +### 3.6 `admin_users` +```json +{ "_id": "", "password_hash": "...", "created_at": ISODate } +``` + +### 3.7 `iam_users` +```json +{ + "_id": { "account_id": "...", "user_name": "..." }, + "user_arn": "...", + "password_hash": null, + "tags": { "": "", ... }, + "created_at": ISODate +} +``` +Unique index on `user_arn`. + +### 3.8 `access_keys` +```json +{ + "_id": "", + "secret_key_encrypted": BinData, + "account_id": "...", + "user_name": "...", + "is_active": true, + "created_at": ISODate +} +``` +Index on `(account_id, user_name)`. + +### 3.9 `iam_groups` +```json +{ + "_id": { "account_id": "...", "group_name": "..." }, + "group_arn": "...", + "members": ["user1", "user2"], + "created_at": ISODate +} +``` +Unique index on `group_arn`. + +### 3.10 `iam_roles` +```json +{ + "_id": { "account_id": "...", "role_name": "..." }, + "role_arn": "...", + "trust_policy": { ... }, + "permissions_boundary_arn": null, + "tags": { "": "", ... }, + "created_at": ISODate +} +``` +Unique index on `role_arn`. + +### 3.11 `iam_sessions` +```json +{ + "_id": "", + "access_key_id": "...", + "secret_key_encrypted": BinData, + "account_id": "...", + "role_name": "...", + "session_name": "...", + "session_tags": { ... }, + "session_policy": { ... }, + "expires_at": ISODate, + "created_at": ISODate +} +``` +Unique index on `access_key_id`. TTL index on `expires_at`. + +### 3.12 `iam_policies` +```json +{ + "_id": { "account_id": "...", "principal_type": "...", "principal_name": "...", "policy_name": "..." }, + "policy_document": { ... }, + "created_at": ISODate +} +``` + +### 3.13 `iam_permissions_boundaries` +```json +{ + "_id": { "account_id": "...", "principal_type": "...", "principal_name": "..." }, + "policy_document": { ... } +} +``` + +### 3.14 `metrics` +```json +{ + "_id": { "bucket": ISODate, "metric": "...", "table_name": "...", "index_name": "...", "operation": "..." }, + "sum": 0.0, + "count": NumberLong(0), + "min": Infinity, + "max": -Infinity +} +``` +Index on `bucket` for pruning. + +### 3.15 `login_attempts` +```json +{ + "principal": "...", + "attempted_at": ISODate, + "success": false, + "source_ip": "..." +} +``` +Compound index on `(principal, attempted_at)`. +Partial index on `(source_ip, attempted_at)` where source_ip exists. + +### 3.16 `backups` (metadata only) +```json +{ + "_id": "", + "backup_name": "...", + "table_id": "...", + "table_name": "...", + "account_id": "...", + "backup_status": "AVAILABLE", + "backup_type": "USER", + "backup_size_bytes": NumberLong, + "item_count": NumberLong, + "key_schema": [...], + "attribute_definitions": [...], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": null, + "stream_specification": null, + "backup_collection": "_backup_{backup_id}", + "created_at": ISODate +} +``` +Index on `(account_id, table_name)`. + +Backup item data is stored in a cloned collection (see Section 5.7). + +### 3.17 `continuous_backups` +```json +{ + "_id": { "account_id": "...", "table_name": "..." }, + "pitr_enabled": false, + "earliest_restorable": null, + "latest_restorable": null +} +``` + +### 3.18 `schema_history` +```json +{ "_id": "", "applied_at": ISODate } +``` + +## 4. Data Database Collections + +### 4.1 Per-Table Item Collections: `_ddb_{table_id}` + +Each DynamoDB virtual table maps to a MongoDB collection. + +**Document structure:** +```json +{ + "_id": "#", + "pk": "...", + "sk_s": "...", + "sk_n": Decimal128, + "sk_b": BinData, + "item_data": { ... } +} +``` + +Fields: +- `_id` — deterministic compound key for upserts +- `pk` — partition key value (string-encoded) +- `sk_s` — sort key (string type), null if not applicable +- `sk_n` — sort key (numeric type, native BSON Decimal128), null if not applicable +- `sk_b` — sort key (binary type), null if not applicable +- `item_data` — full DynamoDB item serialized as BSON + +**Indexes:** +- `{ pk: 1, sk_s: 1 }` or `{ pk: 1, sk_n: 1 }` or `{ pk: 1, sk_b: 1 }` depending + on sort key type, or just `{ pk: 1 }` for PK-only tables + +**Sort key ordering:** +- **String (`sk_s`):** Collection uses `collation: { locale: "simple" }` for + byte-order sorting (matches DynamoDB's UTF-8 byte-order comparison). +- **Numeric (`sk_n`):** Native BSON Decimal128. MongoDB sorts numbers by value — + no encoding tricks needed. DynamoDB supports 38 significant digits; Decimal128 + provides 34, which covers all practical use cases. + +### 4.2 Per-Index Collections: `_ddb_{index_id}` + +Same structure as item collections. GSI/LSI data is projected and stored here. +Written synchronously inline during data operations (PutItem, UpdateItem, DeleteItem). + +### 4.3 `idempotency_tokens` +```json +{ + "_id": "", + "fingerprint": "...", + "created_at": ISODate +} +``` +TTL index on `created_at` (10 minutes) — MongoDB automatically cleans up expired +tokens. + +## 5. Key Design Decisions + +### 5.1 Transactions + +MongoDB multi-document ACID transactions are used **only** for: + +1. **TransactWriteItems** — all operations in a single transaction. +2. **TransactGetItems** — snapshot read using a session with `snapshot` read concern. + +Everything else is transaction-free: +- Single-item conditional writes use filter pushdown (Section 5.2) +- GSI updates are done synchronously inline during the write operation (Section 5.4) +- Stream records are written inline during data operations (Section 5.5) + +### 5.2 Condition Evaluation — Filter Pushdown + +DynamoDB condition expressions are compiled to MongoDB query filters and pushed into +the write operation itself. This exploits MongoDB's single-document atomicity: a +`findOneAndReplace`/`findOneAndUpdate`/`findOneAndDelete` with a filter is atomic +without an explicit transaction. + +**Flow:** +1. Compile `ConditionExpression` AST → MongoDB filter document +2. Combine with primary key filter: `{ pk: X, sk_s: Y, ...condition_filter... }` +3. Execute as `findOneAndReplace` (PutItem), `findOneAndUpdate` (UpdateItem), or + `findOneAndDelete` (DeleteItem) +4. If result is `None` and the item exists → condition failed → + `StorageError::ConditionFailed` + +**Condition-to-filter translation:** + +| DynamoDB condition | MongoDB filter | +|---|---| +| `attribute_exists(foo)` | `{ "item_data.foo": { $exists: true } }` | +| `attribute_not_exists(foo)` | `{ "item_data.foo": { $exists: false } }` | +| `foo = :val` | `{ "item_data.foo.S": val }` (typed) | +| `foo <> :val` | `{ "item_data.foo.S": { $ne: val } }` | +| `foo < :val` | `{ "item_data.foo.N": { $lt: val } }` | +| `foo > :val` | `{ "item_data.foo.N": { $gt: val } }` | +| `begins_with(foo, :p)` | `{ "item_data.foo.S": { $regex: "^

" } }` | +| `contains(foo, :v)` | `{ "item_data.foo.S": { $regex: "" } }` | +| `size(foo) = :n` | `{ $expr: { $eq: [{ $size: "$item_data.foo.L" }, n] } }` | +| `cond1 AND cond2` | `{ $and: [filter1, filter2] }` | +| `cond1 OR cond2` | `{ $or: [filter1, filter2] }` | +| `NOT cond` | `{ $nor: [filter] }` | + +**Implementation:** `condition_to_filter(expr: &Expr, maps: &ExpressionMaps) -> bson::Document` +walks the expression AST and emits a MongoDB filter. + +**Common patterns (all transaction-free):** + +| DynamoDB pattern | MongoDB operation | +|---|---| +| PutItem + `attribute_not_exists(pk)` | `updateOne({ pk, sk, "item_data.pk": {$exists: false} }, $setOnInsert, upsert)` | +| UpdateItem + `version = :v` | `findOneAndUpdate({ pk, sk, "item_data.version.N": v }, $set)` | +| DeleteItem + `status = :val` | `findOneAndDelete({ pk, sk, "item_data.status.S": val })` | +| PutItem (unconditional) | `replaceOne({ pk, sk }, doc, upsert: true)` | + +**Returning the old item:** + +`findOneAndReplace`/`findOneAndDelete` atomically returns the pre-modification +document when `return_old = true`. No transaction needed. + +For `ConditionFailed` with `ReturnValuesOnConditionCheckFailure`, a follow-up +`find_one` fetches the existing item. This is acceptable — DynamoDB has the same +best-effort semantics for the returned item. + +### 5.3 Query and Scan + +**Query:** Translates `KeyCondition` to a MongoDB `find()` filter: +- Partition key equality: `{ pk: "" }` +- Sort key conditions: + - `=` → `{ sk_s: value }` + - `<` → `{ sk_s: { $lt: value } }` + - `begins_with` → `{ sk_s: { $gte: prefix, $lt: prefix_upper } }` + - `BETWEEN` → `{ sk_s: { $gte: low, $lte: high } }` + +Sort direction: `.sort({ sk_s: 1 })` for forward, `.sort({ sk_s: -1 })` for reverse. + +Pagination: `exclusive_start_key` translates to an additional `$gt`/`$lt` filter on +the sort key (or partition key for scans). + +**Scan:** Full collection scan with `.find({})`, paginated via sort-key-based cursor. + +**Parallel scan:** Segments are handled by filtering in application +(`crc32(pk) % total_segments == segment`). Each segment scans the full collection. +This is a known tradeoff — the only way to avoid redundant scans is a pre-bucketed +field on every document, which adds write-path overhead for a feature that is rarely +used in practice. + +### 5.4 GSI Propagation (Synchronous Inline) + +GSI updates are performed synchronously inline during each write operation. There is +no background worker, no Change Stream consumer, and no resume token tracking for GSI +propagation. + +**How it works:** + +On each write (PutItem, UpdateItem, DeleteItem), after writing to the base table +collection, the `sync_indexes` method: + +1. Checks the in-memory `gsi_cache` (`DashMap`) keyed by `table_id`. + If the cache entry is `false`, skip the catalog query entirely (fast path for + tables with no GSIs). +2. If the cache misses or is `true`, query the `indexes` collection in the catalog + database for all indexes belonging to this `table_id`. +3. For each GSI found: + - If an old item exists and has the index keys: delete the old entry from + `_ddb_{index_id}` + - If a new item exists and has the index keys: project the relevant attributes + (respecting the GSI's `Projection` setting) and upsert into `_ddb_{index_id}` +4. Update the cache: `gsi_cache.insert(table_id, found_any)`. + +**Cache invalidation:** +- On table delete (`delete_table`): `gsi_cache.remove(table_id)` +- On GSI create (`update_table`): `gsi_cache.insert(table_id, true)` +- On GSI delete (`update_table`): `gsi_cache.remove(table_id)` (will be re-populated + on next write) + +**Consistency model:** +- GSI reads are strongly consistent (index is updated before write returns to client) +- This is stricter than DynamoDB's eventual consistency model for GSIs, which is + acceptable (stronger guarantees never break application code) + +**Rationale:** Synchronous inline propagation avoids the complexity of Change Stream +recovery, resume token management, and eventual consistency bugs. The overhead is one +catalog query per write for tables with GSIs (cached to zero for tables without GSIs). + +### 5.5 DynamoDB Streams (Inline Record Storage) + +DynamoDB Streams are implemented by writing stream records inline during data +operations, using the same storage model as the PostgreSQL backend. Stream records +are stored in MongoDB collections (`stream_records` and `stream_shards` in the data +database) with explicit sequence numbers and shard assignment. This approach provides +behavioral parity with the PostgreSQL backend rather than relying on MongoDB Change +Streams. + +**Data model:** + +- `stream_shards` — one document per shard (4 shards per stream-enabled table), + keyed by `shard_id` + `table_id` +- `stream_records` — one document per event, containing `sequence_number`, `shard_id`, + `table_id`, `event_name`, `record_data` (full `StreamRecord` serialized as BSON), + and `created_at` + +**Write path:** + +When `StreamCapture` is provided to a data operation (PutItem, UpdateItem, DeleteItem), +the `write_stream_inline` helper: + +1. Determines the event type (INSERT/MODIFY/REMOVE) from old/new item presence +2. Builds key images and old/new images based on `StreamViewType` +3. Assigns a shard using `crc32(partition_key) % shard_count` +4. Obtains a sequence number via atomic `findOneAndUpdate` on a counter document +5. Writes the stream record to the `stream_records` collection + +**Shard assignment:** `crc32(pk) % SHARDS_PER_STREAM` (currently 4 shards per table). + +**Sequence numbers:** Global monotonic counter stored in `counters` collection, using +`findOneAndUpdate` with `$inc` for atomic increment. Format: zero-padded 21 digits. + +**`StreamEngine` trait mapping:** + +| Trait method | Implementation | +|----------------------------------|------------------------------------------------------------------| +| `write_stream_record` | Insert record document into `stream_records` collection | +| `get_stream_records` | Query `stream_records` by `shard_id`, ordered by `sequence_number` | +| `describe_stream` | Query `tables` + `stream_shards`, return shard list | +| `list_streams` | Query `tables` where `stream_label` is not null | +| `cleanup_expired_stream_records` | Delete records older than retention cutoff | +| `assign_shard` | `crc32(pk) % shard_count` over shards for the table | +| `next_sequence_number` | Atomic `$inc` on counter document in `counters` collection | +| `validate_shard` | Check table+stream exist and shard_id belongs to the stream | +| `latest_sequence_number` | Query last record in shard by descending `sequence_number` | + +**Retention:** `cleanup_expired_stream_records` deletes records with `created_at` +older than the configured retention period. + +### 5.6 TTL Handling + +MongoDB's built-in TTL indexes handle automatic cleanup for: +- `idempotency_tokens` — expire after 10 minutes +- `iam_sessions` — expire at `expires_at` + +For DynamoDB-level TTL (user-configured `TimeToLive`), the application-level TTL +worker is still needed because TTL deletion must emit stream records with a specific +`UserIdentity`. When TTL is enabled on a table, a sparse index is created on the TTL +attribute path for efficient expired-item lookup: + +```rust +db.collection("_ddb_{table_id}") + .create_index(IndexModel::builder() + .keys(doc! { format!("item_data.{ttl_attribute}.N"): 1 }) + .options(IndexOptions::builder().sparse(true).build()) + .build()) +``` + +### 5.7 Backups + +`CreateBackup` clones the source collection server-side using `$out`: + +```rust +// CreateBackup — server-side collection clone +data_db.collection("_ddb_{table_id}") + .aggregate([doc! { "$out": "_backup_{backup_id}" }]) + .await?; + +// RestoreTableFromBackup — clone back to new table +data_db.collection("_backup_{backup_id}") + .aggregate([doc! { "$out": "_ddb_{new_table_id}" }]) + .await?; + +// DeleteBackup — drop the backup collection +data_db.collection("_backup_{backup_id}").drop().await?; +``` + +No document size limits, handles tables of any size, no client-side data transfer. + +### 5.8 Write Conflict Handling + +**UpdateItem (optimistic concurrency):** + +`UpdateItem` uses a read-modify-write pattern with a `_v` version field for conflict +detection: + +1. Read the existing document and note its `_v` (version) value (defaults to 0 if + absent) +2. Apply update expressions in memory to produce the new item +3. Set `_v = current_version + 1` on the new document +4. Execute `replaceOne` with a filter matching both the primary key AND the expected + `_v` value +5. If `matched_count == 0`, a concurrent writer incremented the version first — + retry with jittered exponential backoff (base 100us, up to 50 attempts) + +This avoids multi-document transactions for single-item updates while preventing +lost updates. + +**Conditional writes (PutItem, DeleteItem):** + +These use a find-then-write pattern. For PutItem, the condition is evaluated +client-side against the fetched document, then `findOneAndReplace` (or `insert_one` +for new items) is used. Duplicate key errors on insert are caught and mapped to +`ConditionFailed`. + +**Explicit transactions (`TransactWriteItems`):** + +All operations in a `TransactWriteItems` call execute within a single MongoDB +multi-document transaction with snapshot read concern and majority write concern. +If the transaction fails, it is not retried — the error propagates as +`StorageError::TransactionCanceled`. + +### 5.9 Account ID Validation + +Defense against MongoDB operator injection: +- Reject `$` (operator injection) +- Reject `.` (field path traversal) +- Reject null bytes +- Reject non-ASCII + +### 5.10 Catalog Version Check + +Read `catalog_version` from the `settings` collection and compare against the +compiled-in constant. Same pattern as PostgreSQL. + +## 6. Crate Structure + +``` +crates/storage-mongodb/ +├── Cargo.toml +└── src/ + ├── lib.rs # MongoEngine struct, inventory registrations + ├── config.rs # Configuration parsing + ├── bootstrapper.rs # Database initialization (init/destroy) + ├── table_engine.rs # CreateTable, DeleteTable, DescribeTable, UpdateTable + ├── data_engine.rs # PutItem, GetItem, DeleteItem, UpdateItem, Query, Scan, Transactions + ├── data/mod.rs # Document <-> Item conversion helpers + ├── condition.rs # DynamoDB condition expressions -> MongoDB filters + ├── stream_engine.rs # DynamoDB Streams (shard management, sequence numbers) + ├── metadata_engine.rs # TTL, tags, table size tracking + ├── ttl_worker.rs # Background TTL cleanup + ├── backup_engine.rs # Backup/restore via collection cloning + ├── management_store.rs # IAM management, settings, metrics, rate limiting + ├── authorization_store.rs # Policy evaluation, boundaries, sessions + ├── credential_store.rs # Access key lookup with AES-GCM decryption + ├── catalog_store.rs # Catalog and diagnostics + ├── admin_store.rs # Admin operations + └── worker_store.rs # Control plane state transitions +``` + +## 7. MongoEngine Struct + +```rust +pub struct MongoEngine { + client: mongodb::Client, + catalog_db: mongodb::Database, + data_db: mongodb::Database, + region: String, + max_connections: u32, + /// Cache of `table_id` -> `has_gsi`. Avoids catalog queries on every write + /// for tables with no GSIs. + gsi_cache: dashmap::DashMap, +} +``` + +MongoDB's driver manages connection pooling internally (configurable via +`ClientOptions`). A single `Client` is shared; `Database` handles are lightweight +references. The `gsi_cache` provides a fast path to skip GSI catalog lookups for +tables known to have no indexes. + +## 8. Configuration + +```toml +[storage.mongodb] +connection_string = "mongodb://localhost:27017" +pool_size = 20 +``` + +```rust +#[derive(Debug, Clone, Deserialize)] +pub struct MongoStorageConfig { + #[serde(default = "default_connection_string")] + pub connection_string: String, + #[serde(default = "default_pool_size")] + pub pool_size: u32, +} +``` + +## 9. Bootstrapper Flow + +**`extenddb init`:** +1. Connect to MongoDB (databases created implicitly on first write) +2. Create catalog collections with indexes +3. Seed `settings` with `catalog_version` +4. Generate and store encryption key +5. Create default account +6. Create admin user +7. Create data database `idempotency_tokens` collection with TTL index +8. Record data database name in catalog settings + +**`extenddb destroy`:** +1. Drop data database +2. Drop catalog database + +## 10. Inventory Registrations + +```rust +inventory::submit! { BackendRegistration { name: "mongodb", .. } } +inventory::submit! { OperationsEngineRegistration { name: "mongodb", .. } } +inventory::submit! { StorageConfigRegistration { backend: "mongodb", .. } } +inventory::submit! { ServerComponentsRegistration { backend: "mongodb", .. } } +inventory::submit! { SettingsStoreRegistration { backend: "mongodb", .. } } +``` + +## 11. Dependencies + +```toml +[dependencies] +mongodb = { version = "3", features = ["tokio-runtime"] } +bson = "2" +dashmap = "6" +tokio = { workspace = true, features = ["sync"] } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +toml = { workspace = true } +tracing = { workspace = true } +time = { workspace = true } +uuid = { workspace = true } +base64 = { workspace = true } +rand = { workspace = true } +bcrypt = { workspace = true } +aes-gcm = { workspace = true } +async-trait = { workspace = true } +zeroize = { workspace = true } +inventory = { workspace = true } +extenddb-core = { workspace = true } +extenddb-storage = { workspace = true } +extenddb-auth = { workspace = true } +crc32fast = { workspace = true } +``` + +## 12. Implementation Phases + +### Phase 1: Core (MVP) -- Complete +- `MongoEngine` struct and connection setup +- `TableEngine` (create/delete/describe/list/update) +- `DataEngine` (put/get/delete/update/query/scan) with condition filter compiler +- `Bootstrapper` (init, destroy) +- `StorageConfig` and inventory registrations +- Unit tests against a local MongoDB replica set + +### Phase 2: Management & Auth -- Complete +- `CatalogStore` (ManagementStore, AdminStore, SettingsStore, MetricsStore, + RateLimitStore) +- `AuthorizationStore` +- `MongoCredentialStore` +- Web console and management API working + +### Phase 3: Streams & Transactions -- Complete +- `StreamEngine` (inline stream record writes, shard management, sequence numbers) +- `TransactGetItems` / `TransactWriteItems` +- Idempotency tokens + +### Phase 4: Advanced Features -- Complete +- `BackupEngine` (collection cloning via `$out`) +- `WorkerStore` (control plane state transitions) +- Synchronous inline GSI propagation with `DashMap` cache +- TTL worker (application-level DynamoDB TTL) +- `MetadataEngine` (full TTL lifecycle) + +### Phase 5: Testing & Production Readiness -- Complete +- Full pytest integration suite passes against MongoDB backend +- Performance benchmarking vs PostgreSQL backend +- Documentation + +## 13. Testing Strategy + +- **Unit tests:** Mock the MongoDB client for pure logic tests +- **Integration tests:** Single-node replica set in Docker (`mongod --replSet rs0`) +- **Existing pytest suite:** Passes unchanged (speaks DynamoDB wire protocol) +- **CI:** GitHub Actions job with MongoDB replica set, runs + `cargo test -p extenddb-storage-mongodb` + +## 14. Deployment Requirements + +- MongoDB **6.0+** in **replica set** mode (required for multi-document transactions) +- Single-node replica set is fine for development/testing +- For production: 3-node replica set +- Target scale: < 500 DynamoDB tables. At 500 tables with 2 GSIs each (~1500 + collections), WiredTiger handles this comfortably with default settings. + Ensure `ulimit -n` ≥ 65536. + +## 15. Design Decisions Summary + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Conditional writes | Filter pushdown (no transaction) | Single-document atomicity, no tx overhead on hot path | +| GSI updates | Synchronous inline | Simplicity, no Change Stream recovery complexity, strongly consistent | +| DynamoDB Streams | Inline record writes to MongoDB collections | Behavioral parity with PostgreSQL backend, explicit sequence numbers | +| Stream shards | 4 per table, CRC32 hash assignment | Predictable parallelism for consumers | +| Sort key numbers | Native BSON Decimal128 | Correct ordering by value, zero encoding overhead | +| Backups | `$out` collection clone | Server-side, no size limits | +| Parallel scan | Filter in application | Rarely used, not worth write-path overhead of `_seg` field | +| Write conflict (UpdateItem) | Optimistic concurrency with `_v` field + jittered backoff | Avoids transactions for single-item updates | + +## 16. Performance Characteristics + +**Hot path (single-item writes):** Transaction-free. A PutItem with condition is a +single `findOneAndReplace` with a filter — one network roundtrip, one WiredTiger +document write. No locking, no multi-phase commit. + +**GSI overhead on write path:** One catalog query per write for tables with GSIs +(to fetch index definitions), plus one upsert/delete per GSI. For tables with no +GSIs, the `gsi_cache` short-circuits to zero overhead (no catalog query, no I/O). + +**Stream overhead on write path:** When streaming is enabled, one counter increment +(atomic `findOneAndUpdate`) plus one document insert to `stream_records` per write +operation. + +**Query/Scan:** Direct index lookups on `{ pk, sk_* }`. Same performance +characteristics as any indexed MongoDB query. + +**TransactWriteItems:** Multi-collection transaction. Rare in practice (most +workloads are single-item operations). Limited to 100 operations per DynamoDB +API spec. From 04c7543fd21e834d836b25fb97b257ccb1d67672 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Fri, 5 Jun 2026 21:08:14 -0700 Subject: [PATCH 02/83] feat: add MongoDB storage backend (extenddb-storage-mongodb) Implements the full TableEngine, DataEngine, MetadataEngine, StreamEngine, BackupEngine, WorkerStore, and catalog traits against MongoDB 6.0+. Key design decisions: - Single-item writes (put/delete/update) use MongoDB transactions with snapshot read concern and majority write concern for atomicity - Stream records and GSI sync are in the same transaction as the data write - UpdateItem uses optimistic concurrency (_v version field) with session reuse across retries for performance under contention - Condition expressions compiled to MongoDB query filters via condition.rs - Numbers stored as strings in item_data to preserve DynamoDB 38-digit decimal precision - Binary sort key begins_with uses post-fetch filtering (BSON Binary comparison sorts by length first, making $gte/$lt unreliable for prefix matching) - Simple unconditional SET/REMOVE updates use native MongoDB operators via findOneAndUpdate for lower latency Wiring: adds mongodb feature flag to bin crate, registers backend via inventory, and generalizes cmd_serve backend validation. Requires: MongoDB 6.0+ configured as a replica set (even single-node) for multi-document transactions and snapshot reads. --- Cargo.toml | 5 + crates/storage-mongodb/Cargo.toml | 56 + crates/storage-mongodb/src/admin_store.rs | 4 + .../src/authorization_store.rs | 361 +++ crates/storage-mongodb/src/backup_engine.rs | 579 ++++ crates/storage-mongodb/src/bootstrapper.rs | 547 ++++ crates/storage-mongodb/src/catalog_store.rs | 110 + crates/storage-mongodb/src/condition.rs | 747 +++++ crates/storage-mongodb/src/config.rs | 58 + .../storage-mongodb/src/credential_store.rs | 216 ++ crates/storage-mongodb/src/data/mod.rs | 193 ++ crates/storage-mongodb/src/data_engine.rs | 2156 +++++++++++++++ crates/storage-mongodb/src/lib.rs | 270 ++ .../storage-mongodb/src/management_store.rs | 2400 +++++++++++++++++ crates/storage-mongodb/src/metadata_engine.rs | 546 ++++ crates/storage-mongodb/src/operations.rs | 129 + crates/storage-mongodb/src/stream_engine.rs | 526 ++++ crates/storage-mongodb/src/table_engine.rs | 1038 +++++++ crates/storage-mongodb/src/ttl_worker.rs | 208 ++ crates/storage-mongodb/src/worker_store.rs | 148 + 20 files changed, 10297 insertions(+) create mode 100644 crates/storage-mongodb/Cargo.toml create mode 100644 crates/storage-mongodb/src/admin_store.rs create mode 100644 crates/storage-mongodb/src/authorization_store.rs create mode 100644 crates/storage-mongodb/src/backup_engine.rs create mode 100644 crates/storage-mongodb/src/bootstrapper.rs create mode 100644 crates/storage-mongodb/src/catalog_store.rs create mode 100644 crates/storage-mongodb/src/condition.rs create mode 100644 crates/storage-mongodb/src/config.rs create mode 100644 crates/storage-mongodb/src/credential_store.rs create mode 100644 crates/storage-mongodb/src/data/mod.rs create mode 100644 crates/storage-mongodb/src/data_engine.rs create mode 100644 crates/storage-mongodb/src/lib.rs create mode 100644 crates/storage-mongodb/src/management_store.rs create mode 100644 crates/storage-mongodb/src/metadata_engine.rs create mode 100644 crates/storage-mongodb/src/operations.rs create mode 100644 crates/storage-mongodb/src/stream_engine.rs create mode 100644 crates/storage-mongodb/src/table_engine.rs create mode 100644 crates/storage-mongodb/src/ttl_worker.rs create mode 100644 crates/storage-mongodb/src/worker_store.rs diff --git a/Cargo.toml b/Cargo.toml index 76f909b6..7b3faece 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/storage", "crates/config", "crates/storage-postgres", + "crates/storage-mongodb", "crates/auth", "crates/server", "crates/app", @@ -29,6 +30,7 @@ extenddb-engine = { path = "crates/engine" } extenddb-storage = { path = "crates/storage" } extenddb-config = { path = "crates/config" } extenddb-storage-postgres = { path = "crates/storage-postgres" } +extenddb-storage-mongodb = { path = "crates/storage-mongodb" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } extenddb-app = { path = "crates/app" } @@ -60,6 +62,9 @@ moka = { version = "0.12", features = ["future"] } # Database sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "bigdecimal"] } +mongodb = "3" +bson = "2.13" +dashmap = "6" # Crypto & checksums crc32fast = "1" diff --git a/crates/storage-mongodb/Cargo.toml b/crates/storage-mongodb/Cargo.toml new file mode 100644 index 00000000..a2d00fff --- /dev/null +++ b/crates/storage-mongodb/Cargo.toml @@ -0,0 +1,56 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "extenddb-storage-mongodb" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +# Internal crates +extenddb-core.workspace = true +extenddb-storage.workspace = true +extenddb-auth.workspace = true + +# Serialization +serde.workspace = true +serde_json.workspace = true +toml.workspace = true +bson.workspace = true + +# Error handling +thiserror.workspace = true +anyhow.workspace = true + +# Async +tokio.workspace = true +async-trait.workspace = true +futures.workspace = true + +# Backend registry +inventory.workspace = true + +# MongoDB driver +mongodb.workspace = true + +# Crypto & checksums +uuid.workspace = true +base64.workspace = true +bcrypt.workspace = true +aes-gcm.workspace = true +rand.workspace = true +zeroize.workspace = true + +# Time +time.workspace = true + +# Logging +tracing.workspace = true + +# Checksums +crc32fast.workspace = true + +# In-process GSI existence cache +dashmap.workspace = true diff --git a/crates/storage-mongodb/src/admin_store.rs b/crates/storage-mongodb/src/admin_store.rs new file mode 100644 index 00000000..aa24f8fd --- /dev/null +++ b/crates/storage-mongodb/src/admin_store.rs @@ -0,0 +1,4 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Admin store stub module (implemented in `management_store.rs`). diff --git a/crates/storage-mongodb/src/authorization_store.rs b/crates/storage-mongodb/src/authorization_store.rs new file mode 100644 index 00000000..415249d9 --- /dev/null +++ b/crates/storage-mongodb/src/authorization_store.rs @@ -0,0 +1,361 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `AuthorizationStore` trait implementation for `MongoDB`. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{self, Document, doc}; +use mongodb::options::FindOptions; + +use extenddb_storage::authorization_store::{AuthorizationStore, SessionData}; +use extenddb_storage::management_store::{OpError, OpResult}; + +use crate::catalog_store::MongoCatalogStore; + +impl AuthorizationStore for MongoCatalogStore { + fn fetch_user_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_user_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + }) + .collect()) + }) + } + + fn fetch_user_group_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + // First get the groups this user belongs to + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let members_cursor = members_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("fetch_user_group_policies members: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let member_docs: Vec = members_cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_group_policies members cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let group_names: Vec<&str> = member_docs + .iter() + .filter_map(|d| d.get_str("group_name").ok()) + .collect(); + + if group_names.is_empty() { + return Ok(Vec::new()); + } + + // Now get all policies for those groups + let policies_coll = self.catalog_db().collection::("iam_policies"); + let cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "group", + "principal_name": { "$in": &group_names }, + }) + .await + .map_err(|e| { + tracing::error!("fetch_user_group_policies policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_group_policies policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + }) + .collect()) + }) + } + + fn fetch_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_user_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + })) + }) + } + + fn fetch_role_policies( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_role_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_role_policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + }) + .collect()) + }) + } + + fn fetch_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_role_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + })) + }) + } + + fn fetch_session_data( + &self, + account_id: &str, + role_name: &str, + session_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_sessions"); + let now_bson = mongodb::bson::DateTime::now(); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "role_name": &role_name, + "session_name": &session_name, + "expires_at": { "$gt": now_bson }, + }) + .await + .map_err(|e| { + tracing::error!("fetch_session_data: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(session_doc) = doc else { + return Ok(None); + }; + + let session_policy = session_doc.get("session_policy").and_then(|b| { + let json_val: serde_json::Value = bson::from_bson(b.clone()).ok()?; + Some(json_val.to_string()) + }); + + let mut session_tags = Vec::new(); + if let Some(tags_bson) = session_doc.get("session_tags") { + if let Ok(tags_val) = bson::from_bson::(tags_bson.clone()) { + if let Some(arr) = tags_val.as_array() { + for tag in arr { + if let (Some(k), Some(v)) = ( + tag.get("Key").and_then(|k| k.as_str()), + tag.get("Value").and_then(|v| v.as_str()), + ) { + session_tags.push((k.to_owned(), v.to_owned())); + } + } + } else if let Some(obj) = tags_val.as_object() { + for (k, v) in obj { + if let Some(v_str) = v.as_str() { + session_tags.push((k.clone(), v_str.to_owned())); + } + } + } + } + } + + Ok(Some(SessionData { + session_policy, + session_tags, + })) + }) + } + + fn fetch_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("fetch_user_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn fetch_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("fetch_role_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_role_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn fetch_resource_tags(&self, arn: &str) -> BoxFuture<'_, OpResult>> { + let arn = arn.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("tags"); + let cursor = coll + .find(doc! { "resource_arn": &arn }) + .await + .map_err(|e| { + tracing::error!("fetch_resource_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_resource_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } +} diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs new file mode 100644 index 00000000..05cdc1b6 --- /dev/null +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -0,0 +1,579 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `BackupEngine` implementation for `MongoDB`. +//! +//! Backups are stored as documents in a `backups` collection (metadata) and +//! a `backup_items` collection (snapshotted items). Uses `$out`-style cloning +//! approach: read all items from the data collection and bulk-insert into +//! the backup items collection tagged with `backup_arn`. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{Document, doc}; + +use extenddb_core::types::{ + BackupDescription, BackupDetails, BackupSummary, ContinuousBackupsDescription, + KeySchemaElement, PointInTimeRecoveryDescription, SourceTableDetails, TableDescription, +}; +use extenddb_storage::BackupEngine; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; + +use crate::MongoEngine; +use crate::data::data_collection_name; + +fn epoch_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +#[allow(clippy::cast_precision_loss)] +fn now_epoch_secs() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64 +} + +impl BackupEngine for MongoEngine { + fn create_backup( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let backup_name = backup_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { + "account_id": &account_id, + "table_name": &table_name, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))? + .to_owned(); + let table_arn = table_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + let key_schema_bson = table_doc + .get_array("key_schema") + .map_err(|_| StorageError::Internal("missing key_schema".to_string()))? + .clone(); + let attr_defs_bson = table_doc + .get_array("attribute_definitions") + .map_err(|_| StorageError::Internal("missing attribute_definitions".to_string()))? + .clone(); + let billing_mode = table_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST") + .to_owned(); + let table_size = table_doc.get_i64("table_size_bytes").unwrap_or(0); + let item_count = table_doc.get_i64("item_count").unwrap_or(0); + + let backup_arn = format!( + "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{ts}", + region = self.region, + ts = epoch_millis() + ); + + // Snapshot items from the data collection + let coll_name = data_collection_name(&table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let mut cursor = data_coll + .find(doc! {}) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let backup_items_coll = self.catalog_db.collection::("backup_items"); + let mut actual_count: i64 = 0; + + while let Some(item_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let mut backup_doc = Document::new(); + backup_doc.insert("backup_arn", &backup_arn); + backup_doc.insert( + "item_data", + item_doc + .get("item_data") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null), + ); + backup_doc.insert("pk", item_doc.get_str("pk").unwrap_or_default()); + if let Ok(sk) = item_doc.get_str("sk_s") { + backup_doc.insert("sk", sk); + } else if let Some(sk_n) = item_doc.get("sk_n") { + backup_doc.insert("sk_n", sk_n.clone()); + } + + backup_items_coll + .insert_one(backup_doc) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + actual_count += 1; + } + + let created_at = now_epoch_secs(); + + // Store backup metadata + let backups_coll = self.catalog_db.collection::("backups"); + let backup_meta = doc! { + "_id": &backup_arn, + "backup_name": &backup_name, + "backup_status": "AVAILABLE", + "backup_type": "USER", + "table_id": &table_id, + "table_name": &table_name, + "table_arn": &table_arn, + "account_id": &account_id, + "backup_size_bytes": table_size, + "item_count": actual_count, + "key_schema": key_schema_bson, + "attribute_definitions": attr_defs_bson, + "billing_mode": &billing_mode, + "created_at": mongodb::bson::DateTime::now(), + "table_creation_date_time": created_at, + }; + + backups_coll + .insert_one(backup_meta) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDetails { + backup_arn, + backup_name, + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: table_size, + backup_creation_date_time: created_at, + }) + }) + } + + fn describe_backup( + &self, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let backups_coll = self.catalog_db.collection::("backups"); + let backup_doc = backups_coll + .find_one(doc! { "_id": &backup_arn }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + let name = backup_doc + .get_str("backup_name") + .unwrap_or_default() + .to_owned(); + let status = backup_doc + .get_str("backup_status") + .unwrap_or("AVAILABLE") + .to_owned(); + let table_id = backup_doc + .get_str("table_id") + .unwrap_or_default() + .to_owned(); + let table_name = backup_doc + .get_str("table_name") + .unwrap_or_default() + .to_owned(); + let table_arn = backup_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + let size = backup_doc.get_i64("backup_size_bytes").unwrap_or(0); + let count = backup_doc.get_i64("item_count").unwrap_or(0); + let billing = backup_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST") + .to_owned(); + + let created_at = backup_doc + .get_datetime("created_at") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); + let table_created = backup_doc + .get_f64("table_creation_date_time") + .unwrap_or(created_at); + + let key_schema_bson = backup_doc + .get_array("key_schema") + .map_err(|_| StorageError::Internal("missing key_schema in backup".to_string()))?; + let key_schema_json = serde_json::to_value(key_schema_bson) + .map_err(|e| StorageError::Internal(format!("serialize key_schema: {e}")))?; + let key_schema: Vec = serde_json::from_value(key_schema_json) + .map_err(|e| StorageError::Internal(format!("parse key_schema: {e}")))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.clone(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: created_at, + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: table_created, + }, + }) + }) + } + + fn list_backups( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.map(std::string::ToString::to_string); + Box::pin(async move { + let backups_coll = self.catalog_db.collection::("backups"); + + let mut filter = doc! { + "account_id": &account_id, + "backup_status": { "$ne": "DELETED" }, + }; + if let Some(tn) = &table_name { + filter.insert("table_name", tn.as_str()); + } + + let mut cursor = backups_coll + .find(filter) + .sort(doc! { "created_at": -1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let arn = doc.get_str("_id").unwrap_or_default().to_owned(); + let name = doc.get_str("backup_name").unwrap_or_default().to_owned(); + let tn = doc.get_str("table_name").unwrap_or_default().to_owned(); + let table_arn = doc.get_str("table_arn").unwrap_or_default().to_owned(); + let status = doc + .get_str("backup_status") + .unwrap_or("AVAILABLE") + .to_owned(); + let size = doc.get_i64("backup_size_bytes").unwrap_or(0); + let created_at = doc + .get_datetime("created_at") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); + + results.push(BackupSummary { + backup_arn: arn, + backup_name: name, + table_name: tn, + table_arn, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: created_at, + }); + } + Ok(results) + }) + } + + fn delete_backup( + &self, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let desc = self.describe_backup(&backup_arn).await?; + + // Delete backup items + let backup_items_coll = self.catalog_db.collection::("backup_items"); + backup_items_coll + .delete_many(doc! { "backup_arn": &backup_arn }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Mark backup as deleted + let backups_coll = self.catalog_db.collection::("backups"); + backups_coll + .update_one( + doc! { "_id": &backup_arn }, + doc! { "$set": { "backup_status": "DELETED" } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details + }, + source_table_details: desc.source_table_details, + }) + }) + } + + fn restore_table_from_backup( + &self, + account_id: &str, + target_table_name: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let target_table_name = target_table_name.to_string(); + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let backups_coll = self.catalog_db.collection::("backups"); + let backup_doc = backups_coll + .find_one(doc! { "_id": &backup_arn, "backup_status": "AVAILABLE" }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + let key_schema_bson = backup_doc + .get_array("key_schema") + .map_err(|_| StorageError::Internal("missing key_schema".to_string()))?; + let attr_defs_bson = backup_doc + .get_array("attribute_definitions") + .map_err(|_| StorageError::Internal("missing attribute_definitions".to_string()))?; + let billing = backup_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST"); + + let ks_json = serde_json::to_value(key_schema_bson) + .map_err(|e| StorageError::Internal(format!("serialize key_schema: {e}")))?; + let ad_json = serde_json::to_value(attr_defs_bson) + .map_err(|e| StorageError::Internal(format!("serialize attr_defs: {e}")))?; + + let key_schema: Vec = + serde_json::from_value(ks_json) + .map_err(|e| StorageError::Internal(format!("parse key_schema: {e}")))?; + let attr_defs: Vec = + serde_json::from_value(ad_json) + .map_err(|e| StorageError::Internal(format!("parse attr_defs: {e}")))?; + + let billing_mode = if billing == "PAY_PER_REQUEST" { + Some(extenddb_core::types::BillingMode::PayPerRequest) + } else { + Some(extenddb_core::types::BillingMode::Provisioned) + }; + + let create_input = extenddb_core::types::CreateTableInput { + table_name: target_table_name.clone(), + key_schema, + attribute_definitions: attr_defs, + billing_mode, + provisioned_throughput: Some(extenddb_core::types::ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + global_secondary_indexes: None, + local_secondary_indexes: None, + stream_specification: None, + tags: None, + deletion_protection_enabled: None, + sse_specification: None, + table_class: None, + on_demand_throughput: None, + }; + + let desc = self.create_table(&account_id, create_input).await?; + + // Restore items from backup + let backup_items_coll = self.catalog_db.collection::("backup_items"); + let mut cursor = backup_items_coll + .find(doc! { "backup_arn": &backup_arn }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let new_coll_name = data_collection_name(&desc.table_id); + let new_data_coll = self.data_db.collection::(&new_coll_name); + + let mut item_count: i64 = 0; + while let Some(backup_item) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + // Re-insert using the original document structure + let mut restore_doc = Document::new(); + if let Some(pk) = backup_item.get("pk") { + restore_doc.insert("pk", pk.clone()); + } + if let Some(item_data) = backup_item.get("item_data") { + restore_doc.insert("item_data", item_data.clone()); + } + if let Ok(sk) = backup_item.get_str("sk") { + restore_doc.insert("sk_s", sk); + let pk_str = backup_item.get_str("pk").unwrap_or_default(); + restore_doc.insert("_id", format!("{pk_str}#{sk}")); + } else if let Some(sk_n) = backup_item.get("sk_n") { + restore_doc.insert("sk_n", sk_n.clone()); + let pk_str = backup_item.get_str("pk").unwrap_or_default(); + restore_doc.insert("_id", format!("{pk_str}#{sk_n}")); + } else { + let pk_str = backup_item.get_str("pk").unwrap_or_default(); + restore_doc.insert("_id", pk_str); + } + + new_data_coll + .insert_one(restore_doc) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + item_count += 1; + } + + // Update item count and mark table ACTIVE + let tables_coll = self.catalog_db.collection::("tables"); + tables_coll + .update_one( + doc! { "account_id": &account_id, "table_name": &target_table_name }, + doc! { "$set": { "item_count": item_count, "table_status": "ACTIVE" } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(desc) + }) + } + + fn describe_continuous_backups( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let exists = tables_coll + .find_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(table_name)); + } + + let cb_coll = self.catalog_db.collection::("continuous_backups"); + let pitr_doc = cb_coll + .find_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let pitr_enabled = pitr_doc + .as_ref() + .and_then(|d| d.get_bool("pitr_enabled").ok()) + .unwrap_or(false); + + let now_epoch = now_epoch_secs(); + + Ok(ContinuousBackupsDescription { + continuous_backups_status: "ENABLED".to_owned(), + point_in_time_recovery_description: Some(PointInTimeRecoveryDescription { + point_in_time_recovery_status: if pitr_enabled { + "ENABLED".to_owned() + } else { + "DISABLED".to_owned() + }, + earliest_restorable_date_time: if pitr_enabled { + Some(now_epoch - 35.0 * 24.0 * 3600.0) + } else { + None + }, + latest_restorable_date_time: if pitr_enabled { Some(now_epoch) } else { None }, + }), + }) + }) + } + + fn update_continuous_backups( + &self, + account_id: &str, + table_name: &str, + pitr_enabled: bool, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let exists = tables_coll + .find_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(table_name.clone())); + } + + let cb_coll = self.catalog_db.collection::("continuous_backups"); + cb_coll + .update_one( + doc! { "account_id": &account_id, "table_name": &table_name }, + doc! { "$set": { + "account_id": &account_id, + "table_name": &table_name, + "pitr_enabled": pitr_enabled, + }}, + ) + .upsert(true) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.describe_continuous_backups(&account_id, &table_name) + .await + }) + } + + fn restore_table_to_point_in_time( + &self, + account_id: &str, + source_table_name: &str, + target_table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let source_table_name = source_table_name.to_string(); + let target_table_name = target_table_name.to_string(); + Box::pin(async move { + let backup = self + .create_backup(&account_id, &source_table_name, "__pitr_restore__") + .await?; + let desc = self + .restore_table_from_backup(&account_id, &target_table_name, &backup.backup_arn) + .await?; + let _ = self.delete_backup(&backup.backup_arn).await; + Ok(desc) + }) + } +} diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs new file mode 100644 index 00000000..93657a82 --- /dev/null +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -0,0 +1,547 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Bootstrapper implementation for `MongoDB`. + +use async_trait::async_trait; +use bson::{Document, doc}; +use mongodb::IndexModel; +use mongodb::options::IndexOptions; + +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; +use extenddb_storage::error::StorageError; +use extenddb_storage::management_store::{OpError, OpResult}; + +/// `MongoDB` bootstrapper for init/destroy/migrate operations. +pub struct MongoBootstrapper { + client: mongodb::Client, + connection_string: String, +} + +impl MongoBootstrapper { + pub async fn from_config( + config_path: &str, + _cli_args: &[String], + ) -> Result { + // Read the config file to get the connection string + let config_content = std::fs::read_to_string(config_path).map_err(|e| { + StorageError::Internal(format!("Cannot read config file '{config_path}': {e}")) + })?; + + let config: toml::Value = config_content + .parse() + .map_err(|e| StorageError::Internal(format!("Cannot parse config: {e}")))?; + + let connection_string = config + .get("storage") + .and_then(|s| s.get("mongodb")) + .and_then(|m| m.get("connection_string")) + .and_then(|v| v.as_str()) + .unwrap_or("mongodb://localhost:27017") + .to_string(); + + let client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + + Ok(Self { + client, + connection_string, + }) + } + + fn catalog_db(&self) -> mongodb::Database { + self.client.database("extenddb_catalog") + } + + fn data_db(&self) -> mongodb::Database { + self.client.database("extenddb_data") + } +} + +#[async_trait] +impl Bootstrapper for MongoBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + // MongoDB uses connection-level auth; no separate app user needed + Ok(()) + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + // Not applicable for MongoDB + Ok(()) + } + + async fn create_catalog_db(&self) -> OpResult<()> { + // MongoDB creates databases implicitly on first write. + // We'll create a sentinel collection to materialize the database. + let db = self.catalog_db(); + db.create_collection("schema_history") + .await + .map_err(|e| OpError::Internal(format!("Failed to create catalog db: {e}")))?; + Ok(()) + } + + async fn create_data_db(&self) -> OpResult<()> { + // MongoDB creates databases implicitly on first write. + let db = self.data_db(); + db.create_collection("idempotency_tokens") + .await + .map_err(|e| OpError::Internal(format!("Failed to create data db: {e}")))?; + + // Create TTL index on idempotency_tokens.created_at (10 min expiry) + let coll = db.collection::("idempotency_tokens"); + let ttl_index = IndexModel::builder() + .keys(doc! { "created_at": 1 }) + .options( + IndexOptions::builder() + .expire_after(std::time::Duration::from_secs(600)) + .build(), + ) + .build(); + coll.create_index(ttl_index) + .await + .map_err(|e| OpError::Internal(format!("Failed to create TTL index: {e}")))?; + + Ok(()) + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + let db = self.catalog_db(); + + // Create all catalog collections + let collections = [ + "accounts", + "tables", + "indexes", + "tags", + "settings", + "admin_users", + "iam_users", + "access_keys", + "iam_groups", + "iam_roles", + "iam_sessions", + "iam_policies", + "iam_permissions_boundaries", + "metrics", + "login_attempts", + "backups", + "continuous_backups", + ]; + + for coll_name in collections { + // create_collection is idempotent in recent MongoDB versions + let _ = db.create_collection(coll_name).await; + } + + // Create indexes for catalog collections + + // accounts: unique index on account_name + let accounts = db.collection::("accounts"); + accounts + .create_index( + IndexModel::builder() + .keys(doc! { "account_name": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("accounts index: {e}")))?; + + // tables: unique index on table_id + let tables = db.collection::("tables"); + tables + .create_index( + IndexModel::builder() + .keys(doc! { "table_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("tables table_id index: {e}")))?; + + // iam_users: unique index on user_arn + let iam_users = db.collection::("iam_users"); + iam_users + .create_index( + IndexModel::builder() + .keys(doc! { "user_arn": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_users user_arn index: {e}")))?; + + // access_keys: index on (account_id, user_name) + let access_keys = db.collection::("access_keys"); + access_keys + .create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "user_name": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("access_keys index: {e}")))?; + + // iam_groups: unique index on group_arn + let iam_groups = db.collection::("iam_groups"); + iam_groups + .create_index( + IndexModel::builder() + .keys(doc! { "group_arn": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_groups index: {e}")))?; + + // iam_roles: unique index on role_arn + let iam_roles = db.collection::("iam_roles"); + iam_roles + .create_index( + IndexModel::builder() + .keys(doc! { "role_arn": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_roles index: {e}")))?; + + // iam_sessions: unique index on access_key_id, TTL on expires_at + let iam_sessions = db.collection::("iam_sessions"); + iam_sessions + .create_index( + IndexModel::builder() + .keys(doc! { "access_key_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_sessions access_key index: {e}")))?; + iam_sessions + .create_index( + IndexModel::builder() + .keys(doc! { "expires_at": 1 }) + .options( + IndexOptions::builder() + .expire_after(std::time::Duration::from_secs(0)) + .build(), + ) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_sessions TTL index: {e}")))?; + + // metrics: index on bucket + let metrics = db.collection::("metrics"); + metrics + .create_index(IndexModel::builder().keys(doc! { "_id.bucket": 1 }).build()) + .await + .map_err(|e| OpError::Internal(format!("metrics bucket index: {e}")))?; + + // login_attempts: compound index + let login_attempts = db.collection::("login_attempts"); + login_attempts + .create_index( + IndexModel::builder() + .keys(doc! { "principal": 1, "attempted_at": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("login_attempts index: {e}")))?; + + // backups: index on (account_id, table_name) + let backups = db.collection::("backups"); + backups + .create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "table_name": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("backups index: {e}")))?; + + // Seed catalog_version in settings + let settings = db.collection::("settings"); + let _ = settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$setOnInsert": { "value": "0.0.2" } }, + ) + .upsert(true) + .await; + + // Record migration + let schema_history = db.collection::("schema_history"); + let _ = schema_history + .insert_one(doc! { + "_id": "001_initial", + "applied_at": bson::DateTime::now(), + }) + .await; // Ignore E11000 (already applied) + + Ok(()) + } + + async fn run_data_migrations(&self) -> OpResult<()> { + // Data database schema is minimal for MongoDB (just idempotency_tokens) + // Table collections are created on-demand + Ok(()) + } + + async fn pending_data_migrations(&self) -> OpResult> { + // MongoDB has no versioned data-database migrations — table + // collections are created on-demand, and the idempotency_tokens + // collection is created in create_data_db. Nothing to apply. + Ok(Vec::new()) + } + + async fn record_data_connection(&self) -> OpResult<()> { + let db = self.catalog_db(); + let settings = db.collection::("settings"); + + let data_db_name = "extenddb_data".to_string(); + settings + .update_one( + doc! { "_id": "data_database_name" }, + doc! { "$set": { "value": &data_db_name } }, + ) + .upsert(true) + .await + .map_err(|e| OpError::Internal(format!("record_data_connection: {e}")))?; + + settings + .update_one( + doc! { "_id": "data_connection_string" }, + doc! { "$set": { "value": &self.connection_string } }, + ) + .upsert(true) + .await + .map_err(|e| OpError::Internal(format!("record data_connection_string: {e}")))?; + + Ok(()) + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + use base64::Engine; + use rand::RngCore; + + let db = self.catalog_db(); + let settings = db.collection::("settings"); + + // Check if already exists + let existing = settings + .find_one(doc! { "_id": "encryption_key" }) + .await + .map_err(|e| OpError::Internal(format!("check encryption key: {e}")))?; + + if existing.is_some() { + return Ok(()); + } + + // Generate a 256-bit encryption key + let mut key_bytes = [0u8; 32]; + rand::rng().fill_bytes(&mut key_bytes); + let key_b64 = base64::engine::general_purpose::STANDARD.encode(key_bytes); + + // Ignore E11000 (race: someone else created it first) + let _ = settings + .insert_one(doc! { + "_id": "encryption_key", + "value": &key_b64, + }) + .await; + + Ok(()) + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + let db = self.catalog_db(); + let accounts = db.collection::("accounts"); + + // Check if any account exists + let count = accounts + .count_documents(doc! {}) + .await + .map_err(|e| OpError::Internal(format!("count accounts: {e}")))?; + + if count > 0 { + return Ok(()); + } + + let account_id = uuid::Uuid::new_v4().to_string(); + accounts + .insert_one(doc! { + "_id": &account_id, + "account_name": "default", + "created_at": bson::DateTime::now(), + }) + .await + .map_err(|e| OpError::Internal(format!("create default account: {e}")))?; + + Ok(()) + } + + async fn bootstrap_admin_user( + &self, + env_user: Option<&str>, + env_password: Option<&str>, + ) -> OpResult { + let username = env_user.unwrap_or("admin").to_string(); + + let db = self.catalog_db(); + let admin_users = db.collection::("admin_users"); + + // Check if admin already exists + let existing = admin_users + .find_one(doc! { "_id": &username }) + .await + .map_err(|e| OpError::Internal(format!("check admin: {e}")))?; + + if existing.is_some() { + return Ok(AdminBootstrapResult { + username, + generated_password: None, + already_existed: true, + from_env: env_user.is_some(), + }); + } + + // Generate or use provided password + let (password, from_env) = if let Some(pw) = env_password { + (pw.to_string(), true) + } else { + use rand::Rng; + let pw: String = rand::rng() + .sample_iter(&rand::distr::Alphanumeric) + .take(24) + .map(char::from) + .collect(); + (pw, false) + }; + + let password_hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST) + .map_err(|e| OpError::Internal(format!("bcrypt hash: {e}")))?; + + admin_users + .insert_one(doc! { + "_id": &username, + "password_hash": &password_hash, + "created_at": bson::DateTime::now(), + }) + .await + .map_err(|e| OpError::Internal(format!("create admin: {e}")))?; + + Ok(AdminBootstrapResult { + username, + generated_password: if from_env { None } else { Some(password) }, + already_existed: false, + from_env, + }) + } + + async fn is_catalog_initialized(&self) -> OpResult { + let db = self.catalog_db(); + let collections = db + .list_collection_names() + .await + .map_err(|e| OpError::Internal(format!("list collections: {e}")))?; + Ok(collections.contains(&"settings".to_string())) + } + + async fn list_table_names(&self) -> OpResult> { + use futures::TryStreamExt; + + let db = self.catalog_db(); + let tables = db.collection::("tables"); + + let cursor = tables + .find(doc! {}) + .projection(doc! { "_id.table_name": 1 }) + .await + .map_err(|e| OpError::Internal(format!("list tables: {e}")))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| OpError::Internal(format!("collect tables: {e}")))?; + + let names: Vec = docs + .iter() + .filter_map(|d| { + d.get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .map(std::string::ToString::to_string) + }) + .collect(); + + Ok(names) + } + + async fn get_data_db_name(&self) -> OpResult> { + let db = self.catalog_db(); + let settings = db.collection::("settings"); + let doc = settings + .find_one(doc! { "_id": "data_database_name" }) + .await + .map_err(|e| OpError::Internal(format!("get data_db_name: {e}")))?; + Ok(doc.and_then(|d| { + d.get_str("value") + .ok() + .map(std::string::ToString::to_string) + })) + } + + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + self.data_db() + .drop() + .await + .map_err(|e| OpError::Internal(format!("drop data db: {e}")))?; + self.catalog_db() + .drop() + .await + .map_err(|e| OpError::Internal(format!("drop catalog db: {e}")))?; + Ok(()) + } + + async fn read_catalog_version(&self) -> OpResult> { + let db = self.catalog_db(); + let settings = db.collection::("settings"); + let doc = settings + .find_one(doc! { "_id": "catalog_version" }) + .await + .map_err(|e| OpError::Internal(format!("read catalog_version: {e}")))?; + Ok(doc.and_then(|d| { + d.get_str("value") + .ok() + .map(std::string::ToString::to_string) + })) + } + + fn expected_catalog_version(&self) -> String { + "0.0.2".to_string() + } + + fn catalog_database_name(&self) -> String { + "extenddb_catalog".to_string() + } + + fn endpoint_info(&self) -> String { + self.connection_string.clone() + } + + fn catalog_connection_url(&self) -> String { + format!("{}/extenddb_catalog", self.connection_string) + } + + fn generate_backend_config_section(&self) -> String { + format!( + r#"[storage.mongodb] +connection_string = "{}" +# max_connections = 50 # Max concurrent connections for data operations (default 50) +# max_catalog_connections = 20 # Max concurrent connections for catalog/management operations (default 20)"#, + self.connection_string + ) + } +} diff --git a/crates/storage-mongodb/src/catalog_store.rs b/crates/storage-mongodb/src/catalog_store.rs new file mode 100644 index 00000000..625cacb1 --- /dev/null +++ b/crates/storage-mongodb/src/catalog_store.rs @@ -0,0 +1,110 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Catalog store implementation for `MongoDB`. + +use futures::future::BoxFuture; +use mongodb::bson::doc; + +/// `MongoDB` catalog store. +pub struct MongoCatalogStore { + client: mongodb::Client, + catalog_db: mongodb::Database, + pub(crate) encryption_key: Option, +} + +impl MongoCatalogStore { + #[must_use] + pub fn new(client: mongodb::Client) -> Self { + let catalog_db = client.database("extenddb_catalog"); + Self { + client, + catalog_db, + encryption_key: None, + } + } + + #[must_use] + pub fn with_encryption_key(client: mongodb::Client, encryption_key: String) -> Self { + let catalog_db = client.database("extenddb_catalog"); + Self { + client, + catalog_db, + encryption_key: Some(encryption_key), + } + } + + /// Get a reference to the catalog database. + pub(crate) fn catalog_db(&self) -> &mongodb::Database { + &self.catalog_db + } + + /// Get a reference to the `MongoDB` client. + pub(crate) fn client(&self) -> &mongodb::Client { + &self.client + } +} + +// Implement CatalogStore supertrait +impl extenddb_storage::CatalogStore for MongoCatalogStore { + fn cached_encryption_key(&self) -> Option { + self.encryption_key.clone() + } +} + +// Implement DiagnosticsStore +impl extenddb_storage::diagnostics::DiagnosticsStore for MongoCatalogStore { + fn count_tables(&self) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + Box::pin(async { + let coll = self + .catalog_db + .collection::("tables"); + let count = coll.count_documents(doc! {}).await.map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + Ok(count as i64) + }) + } + + fn count_indexes(&self) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + Box::pin(async { + use futures::TryStreamExt; + + // Count tables that have GSIs or LSIs defined + let coll = self + .catalog_db + .collection::("tables"); + let mut cursor = coll.find(doc! {}).await.map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + + let mut index_count: i64 = 0; + while let Some(table_doc) = cursor + .try_next() + .await + .map_err(|e| extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()))? + { + if let Ok(gsis) = table_doc.get_array("global_secondary_indexes") { + index_count += gsis.len() as i64; + } + if let Ok(lsis) = table_doc.get_array("local_secondary_indexes") { + index_count += lsis.len() as i64; + } + } + Ok(index_count) + }) + } + + fn test_data_database_connection( + &self, + ) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + Box::pin(async { + // Ping the data database to verify connectivity + let data_db = self.client.database("extenddb_data"); + data_db.run_command(doc! { "ping": 1 }).await.map_err(|e| { + extenddb_storage::diagnostics::DiagError::ConnectionFailed(e.to_string()) + })?; + Ok("extenddb_data".to_string()) + }) + } +} diff --git a/crates/storage-mongodb/src/condition.rs b/crates/storage-mongodb/src/condition.rs new file mode 100644 index 00000000..d731e59d --- /dev/null +++ b/crates/storage-mongodb/src/condition.rs @@ -0,0 +1,747 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Condition expression to `MongoDB` filter compiler. +//! +//! Translates `extenddb_core::expression::Expr` AST into a `bson::Document` +//! query filter for use with `MongoDB`'s filter pushdown (findOneAndReplace, etc.). + +use bson::{Bson, Document, doc}; + +use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement}; +use extenddb_core::types::AttributeValue; +use extenddb_storage::error::StorageError; + +/// Compile a condition expression to a `MongoDB` filter document. +/// +/// The resulting filter operates on the `item_data` field of the `MongoDB` document. +/// Returns an empty document if no condition is provided. +pub fn condition_to_filter(expr: &Expr, maps: &ExpressionMaps) -> Result { + compile_expr(expr, maps) +} + +/// Resolve a path to the `MongoDB` field path within `item_data`. +/// +/// `DynamoDB` paths like `address.city` or `tags[0]` become +/// `item_data.address.city` or `item_data.tags.L.0` in `MongoDB`. +fn resolve_path_to_field( + elements: &[PathElement], + maps: &ExpressionMaps, +) -> Result { + let mut parts = vec!["item_data".to_string()]; + for elem in elements { + match elem { + PathElement::Attribute(name) => { + let resolved = if let Some(stripped) = name.strip_prefix('#') { + maps.resolve_name(stripped) + .map_err(|e| StorageError::Validation(e.to_string()))? + .to_string() + } else { + name.clone() + }; + parts.push(resolved); + } + PathElement::Index(idx) => { + // For list index access: item_data.attr.L. + parts.push("L".to_string()); + parts.push(idx.to_string()); + } + } + } + Ok(parts.join(".")) +} + +/// Convert an `AttributeValue` to a BSON value for filter comparisons. +/// +/// `DynamoDB` stores typed values like `{"S": "hello"}`, so when comparing +/// `item_data.foo.S` we need the raw string value, not the wrapped form. +/// +/// Numbers are kept as strings to match the storage format in `item_data` +/// (DynamoDB numbers are 38-digit decimals stored as their string representation). +fn av_to_bson(av: &AttributeValue) -> Bson { + match av { + AttributeValue::S(s) => Bson::String(s.clone()), + AttributeValue::N(n) => Bson::String(n.clone()), + AttributeValue::B(b) => Bson::Binary(bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: b.clone(), + }), + AttributeValue::Bool(b) => Bson::Boolean(*b), + AttributeValue::Null => Bson::Boolean(true), // NULL type stores {"NULL": true} + _ => Bson::Null, // Sets and complex types handled differently + } +} + +/// Get the type suffix for a `DynamoDB` `AttributeValue` (S, N, B, BOOL, NULL, L, M, SS, NS, BS). +fn av_type_suffix(av: &AttributeValue) -> &'static str { + match av { + AttributeValue::S(_) => "S", + AttributeValue::N(_) => "N", + AttributeValue::B(_) => "B", + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::L(_) => "L", + AttributeValue::M(_) => "M", + AttributeValue::SS(_) => "SS", + AttributeValue::NS(_) => "NS", + AttributeValue::BS(_) => "BS", + } +} + +/// Resolve a value expression (Path or Placeholder) to the field path and BSON value. +/// +/// Returns (`field_path_for_filter`, `bson_value`) or just the `bson_value` for placeholders. +enum ResolvedValue { + /// A field path in the document (e.g., "`item_data.age.N`") + Field(String), + /// A literal BSON value with its type suffix + Literal(Bson, &'static str), +} + +fn resolve_value(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Path(elements) => { + let field = resolve_path_to_field(elements, maps)?; + Ok(ResolvedValue::Field(field)) + } + Expr::Placeholder(name) => { + let av = maps + .resolve_value(name) + .map_err(|e| StorageError::Validation(e.to_string()))?; + let suffix = av_type_suffix(av); + let bson_val = av_to_bson(av); + Ok(ResolvedValue::Literal(bson_val, suffix)) + } + _ => Err(StorageError::Validation( + "Unexpected expression type in condition value position".to_string(), + )), + } +} + +/// Build a comparison filter between two expressions. +fn build_comparison( + left: &Expr, + op: CompareOp, + right: &Expr, + maps: &ExpressionMaps, +) -> Result { + let left_resolved = resolve_value(left, maps)?; + let right_resolved = resolve_value(right, maps)?; + + // Determine the field path and value for the comparison + let (field_path, value) = match (left_resolved, right_resolved) { + (ResolvedValue::Field(path), ResolvedValue::Literal(val, suffix)) => { + // field op :value -> item_data.field.TYPE op val + let typed_path = format!("{path}.{suffix}"); + (typed_path, val) + } + (ResolvedValue::Literal(val, suffix), ResolvedValue::Field(path)) => { + // :value op field -> reverse the comparison + let typed_path = format!("{path}.{suffix}"); + let reversed_op = reverse_op(op); + return build_field_comparison(&typed_path, reversed_op, val); + } + (ResolvedValue::Field(left_path), ResolvedValue::Field(right_path)) => { + // field op field -> use $expr + return build_field_vs_field_comparison(&left_path, op, &right_path); + } + (ResolvedValue::Literal(_, _), ResolvedValue::Literal(_, _)) => { + // literal op literal -> evaluate statically (unusual case) + // For simplicity, just return an empty filter (always true) + return Ok(doc! {}); + } + }; + + build_field_comparison(&field_path, op, value) +} + +fn build_field_comparison( + field: &str, + op: CompareOp, + value: Bson, +) -> Result { + let filter = match op { + CompareOp::Eq => doc! { field: value }, + CompareOp::Ne => doc! { field: { "$ne": value } }, + CompareOp::Lt => doc! { field: { "$lt": value } }, + CompareOp::Le => doc! { field: { "$lte": value } }, + CompareOp::Gt => doc! { field: { "$gt": value } }, + CompareOp::Ge => doc! { field: { "$gte": value } }, + }; + Ok(filter) +} + +fn build_field_vs_field_comparison( + left_field: &str, + op: CompareOp, + right_field: &str, +) -> Result { + let mongo_op = match op { + CompareOp::Eq => "$eq", + CompareOp::Ne => "$ne", + CompareOp::Lt => "$lt", + CompareOp::Le => "$lte", + CompareOp::Gt => "$gt", + CompareOp::Ge => "$gte", + }; + Ok(doc! { + "$expr": { + mongo_op: [format!("${left_field}"), format!("${right_field}")] + } + }) +} + +fn reverse_op(op: CompareOp) -> CompareOp { + match op { + CompareOp::Eq => CompareOp::Eq, + CompareOp::Ne => CompareOp::Ne, + CompareOp::Lt => CompareOp::Gt, + CompareOp::Le => CompareOp::Ge, + CompareOp::Gt => CompareOp::Lt, + CompareOp::Ge => CompareOp::Le, + } +} + +/// Compile an expression AST node to a `MongoDB` filter document. +fn compile_expr(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Compare { left, op, right } => build_comparison(left, *op, right, maps), + + Expr::And(left, right) => { + let left_filter = compile_expr(left, maps)?; + let right_filter = compile_expr(right, maps)?; + Ok(doc! { "$and": [left_filter, right_filter] }) + } + + Expr::Or(left, right) => { + let left_filter = compile_expr(left, maps)?; + let right_filter = compile_expr(right, maps)?; + Ok(doc! { "$or": [left_filter, right_filter] }) + } + + Expr::Not(inner) => { + let inner_filter = compile_expr(inner, maps)?; + Ok(doc! { "$nor": [inner_filter] }) + } + + Expr::Function { name, args } => compile_function(name, args, maps), + + Expr::Between { operand, low, high } => compile_between(operand, low, high, maps), + + Expr::In { operand, list } => compile_in(operand, list, maps), + + _ => Err(StorageError::Validation( + "Unsupported expression type in condition filter".to_string(), + )), + } +} + +fn compile_function( + name: &str, + args: &[Expr], + maps: &ExpressionMaps, +) -> Result { + match name.to_lowercase().as_str() { + "attribute_exists" => { + if args.len() != 1 { + return Err(StorageError::Validation( + "attribute_exists requires exactly one argument".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + Ok(doc! { &field: { "$exists": true } }) + } + + "attribute_not_exists" => { + if args.len() != 1 { + return Err(StorageError::Validation( + "attribute_not_exists requires exactly one argument".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + Ok(doc! { &field: { "$exists": false } }) + } + + "begins_with" => { + if args.len() != 2 { + return Err(StorageError::Validation( + "begins_with requires exactly two arguments".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + let prefix_val = resolve_literal(&args[1], maps)?; + match prefix_val { + AttributeValue::S(prefix) => { + let escaped = regex_escape(&prefix); + let typed_field = format!("{field}.S"); + Ok(doc! { &typed_field: { "$regex": format!("^{escaped}") } }) + } + _ => Err(StorageError::Validation( + "begins_with requires a string prefix".to_string(), + )), + } + } + + "contains" => { + if args.len() != 2 { + return Err(StorageError::Validation( + "contains requires exactly two arguments".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + let val = resolve_literal(&args[1], maps)?; + match &val { + AttributeValue::S(substr) => { + // String contains: check substring in string field OR membership in SS/L + let escaped = regex_escape(substr); + let string_field = format!("{field}.S"); + let ss_field = format!("{field}.SS"); + let list_field = format!("{field}.L"); + let bson_val = av_to_bson(&val); + let list_elem = doc! { "S": substr.as_str() }; + Ok(doc! { "$or": [ + { &string_field: { "$regex": &escaped } }, + { &ss_field: &bson_val }, + { &list_field: &list_elem }, + ] }) + } + AttributeValue::N(n) => { + // Number membership in NS or L + let ns_field = format!("{field}.NS"); + let list_field = format!("{field}.L"); + let bson_val = av_to_bson(&val); + let list_elem = doc! { "N": n.as_str() }; + Ok(doc! { "$or": [ + { &ns_field: &bson_val }, + { &list_field: &list_elem }, + ] }) + } + AttributeValue::B(b) => { + // Binary membership in BS or L + let bs_field = format!("{field}.BS"); + let list_field = format!("{field}.L"); + let bson_val = av_to_bson(&val); + let list_elem = doc! { "B": bson::Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: b.clone() } }; + Ok(doc! { "$or": [ + { &bs_field: &bson_val }, + { &list_field: &list_elem }, + ] }) + } + _ => { + // For other types, check membership in L (list) + let list_field = format!("{field}.L"); + let suffix = av_type_suffix(&val); + let list_elem = doc! { suffix: av_to_bson(&val) }; + Ok(doc! { &list_field: &list_elem }) + } + } + } + + "attribute_type" => { + if args.len() != 2 { + return Err(StorageError::Validation( + "attribute_type requires exactly two arguments".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + let type_val = resolve_literal(&args[1], maps)?; + match type_val { + AttributeValue::S(type_name) => { + let typed_field = format!("{field}.{type_name}"); + Ok(doc! { &typed_field: { "$exists": true } }) + } + _ => Err(StorageError::Validation( + "attribute_type requires a string type argument".to_string(), + )), + } + } + + "size" => { + // size() is used in comparisons, not standalone. + // This case handles it if it appears as a standalone function call, + // which shouldn't happen in well-formed expressions. + Err(StorageError::Validation( + "size() cannot be used as a standalone condition".to_string(), + )) + } + + _ => Err(StorageError::Validation(format!( + "Unsupported function in condition: {name}" + ))), + } +} + +fn compile_between( + operand: &Expr, + low: &Expr, + high: &Expr, + maps: &ExpressionMaps, +) -> Result { + let operand_resolved = resolve_value(operand, maps)?; + let low_resolved = resolve_value(low, maps)?; + let high_resolved = resolve_value(high, maps)?; + + if let ( + ResolvedValue::Field(path), + ResolvedValue::Literal(low_val, suffix), + ResolvedValue::Literal(high_val, _), + ) = (operand_resolved, low_resolved, high_resolved) + { + let typed_path = format!("{path}.{suffix}"); + Ok(doc! { &typed_path: { "$gte": low_val, "$lte": high_val } }) + } else { + // Fallback: compile as AND of two comparisons + let gte = build_comparison(operand, CompareOp::Ge, low, maps)?; + let lte = build_comparison(operand, CompareOp::Le, high, maps)?; + Ok(doc! { "$and": [gte, lte] }) + } +} + +fn compile_in( + operand: &Expr, + list: &[Expr], + maps: &ExpressionMaps, +) -> Result { + let operand_resolved = resolve_value(operand, maps)?; + + match operand_resolved { + ResolvedValue::Field(path) => { + // Collect all values, assuming they are the same type + if list.is_empty() { + // Empty IN list never matches — use $and with contradictory conditions + return Ok( + doc! { "$and": [ { "_id": { "$exists": true } }, { "_id": { "$type": "null" } } ] }, + ); + } + + let first_literal = resolve_literal(&list[0], maps)?; + let suffix = av_type_suffix(&first_literal); + let typed_path = format!("{path}.{suffix}"); + + let values: Vec = list + .iter() + .map(|expr| { + let av = resolve_literal(expr, maps)?; + Ok(av_to_bson(&av)) + }) + .collect::, StorageError>>()?; + + Ok(doc! { &typed_path: { "$in": values } }) + } + ResolvedValue::Literal(_, _) => { + // Literal IN list of fields — unusual, compile as OR + let mut or_clauses = Vec::new(); + for item in list { + let eq = build_comparison(operand, CompareOp::Eq, item, maps)?; + or_clauses.push(Bson::Document(eq)); + } + Ok(doc! { "$or": or_clauses }) + } + } +} + +fn resolve_path_from_expr(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Path(elements) => resolve_path_to_field(elements, maps), + _ => Err(StorageError::Validation( + "Expected a path expression".to_string(), + )), + } +} + +fn resolve_literal(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Validation( + "Expected a value placeholder".to_string(), + )), + } +} + +/// Escape special regex characters in a string. +fn regex_escape(s: &str) -> String { + let special = [ + '.', '^', '$', '*', '+', '?', '(', ')', '[', ']', '{', '}', '|', '\\', + ]; + let mut result = String::with_capacity(s.len()); + for c in s.chars() { + if special.contains(&c) { + result.push('\\'); + } + result.push(c); + } + result +} + +// ============================================================================ +// Unit Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::expression::ExpressionMaps; + use extenddb_core::types::AttributeValue; + use std::collections::HashMap; + + fn make_maps(names: Vec<(&str, &str)>, values: Vec<(&str, AttributeValue)>) -> ExpressionMaps { + let names_map: HashMap = names + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let values_map: HashMap = values + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(); + ExpressionMaps::new(names_map, values_map) + } + + #[test] + fn test_attribute_exists() { + let maps = make_maps(vec![], vec![]); + let expr = Expr::Function { + name: "attribute_exists".to_string(), + args: vec![Expr::Path(vec![PathElement::Attribute("foo".to_string())])], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.foo": { "$exists": true } }); + } + + #[test] + fn test_attribute_not_exists() { + let maps = make_maps(vec![], vec![]); + let expr = Expr::Function { + name: "attribute_not_exists".to_string(), + args: vec![Expr::Path(vec![PathElement::Attribute("bar".to_string())])], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.bar": { "$exists": false } }); + } + + #[test] + fn test_equality_comparison_string() { + let maps = make_maps( + vec![], + vec![(":val", AttributeValue::S("hello".to_string()))], + ); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("name".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":val".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.name.S": "hello" }); + } + + #[test] + fn test_less_than_comparison_number() { + let maps = make_maps(vec![], vec![(":min", AttributeValue::N("100".to_string()))]); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "price".to_string(), + )])), + op: CompareOp::Lt, + right: Box::new(Expr::Placeholder(":min".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.price.N": { "$lt": "100" } }); + } + + #[test] + fn test_and_condition() { + let maps = make_maps( + vec![], + vec![ + (":v1", AttributeValue::S("active".to_string())), + (":v2", AttributeValue::N("5".to_string())), + ], + ); + let expr = Expr::And( + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "status".to_string(), + )])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":v1".to_string())), + }), + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "count".to_string(), + )])), + op: CompareOp::Gt, + right: Box::new(Expr::Placeholder(":v2".to_string())), + }), + ); + let filter = condition_to_filter(&expr, &maps).unwrap(); + let expected = doc! { + "$and": [ + { "item_data.status.S": "active" }, + { "item_data.count.N": { "$gt": "5" } } + ] + }; + assert_eq!(filter, expected); + } + + #[test] + fn test_or_condition() { + let maps = make_maps( + vec![], + vec![ + (":a", AttributeValue::S("x".to_string())), + (":b", AttributeValue::S("y".to_string())), + ], + ); + let expr = Expr::Or( + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("f".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":a".to_string())), + }), + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("f".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":b".to_string())), + }), + ); + let filter = condition_to_filter(&expr, &maps).unwrap(); + let expected = doc! { + "$or": [ + { "item_data.f.S": "x" }, + { "item_data.f.S": "y" } + ] + }; + assert_eq!(filter, expected); + } + + #[test] + fn test_not_condition() { + let maps = make_maps(vec![], vec![]); + let expr = Expr::Not(Box::new(Expr::Function { + name: "attribute_exists".to_string(), + args: vec![Expr::Path(vec![PathElement::Attribute( + "deleted".to_string(), + )])], + })); + let filter = condition_to_filter(&expr, &maps).unwrap(); + let expected = doc! { + "$nor": [{ "item_data.deleted": { "$exists": true } }] + }; + assert_eq!(filter, expected); + } + + #[test] + fn test_begins_with() { + let maps = make_maps( + vec![], + vec![(":prefix", AttributeValue::S("user#".to_string()))], + ); + let expr = Expr::Function { + name: "begins_with".to_string(), + args: vec![ + Expr::Path(vec![PathElement::Attribute("sk".to_string())]), + Expr::Placeholder(":prefix".to_string()), + ], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.sk.S": { "$regex": "^user#" } }); + } + + #[test] + fn test_between() { + let maps = make_maps( + vec![], + vec![ + (":lo", AttributeValue::N("10".to_string())), + (":hi", AttributeValue::N("20".to_string())), + ], + ); + let expr = Expr::Between { + operand: Box::new(Expr::Path(vec![PathElement::Attribute("age".to_string())])), + low: Box::new(Expr::Placeholder(":lo".to_string())), + high: Box::new(Expr::Placeholder(":hi".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!( + filter, + doc! { "item_data.age.N": { "$gte": "10", "$lte": "20" } } + ); + } + + #[test] + fn test_in_condition() { + let maps = make_maps( + vec![], + vec![ + (":v1", AttributeValue::S("a".to_string())), + (":v2", AttributeValue::S("b".to_string())), + (":v3", AttributeValue::S("c".to_string())), + ], + ); + let expr = Expr::In { + operand: Box::new(Expr::Path(vec![PathElement::Attribute("x".to_string())])), + list: vec![ + Expr::Placeholder(":v1".to_string()), + Expr::Placeholder(":v2".to_string()), + Expr::Placeholder(":v3".to_string()), + ], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.x.S": { "$in": ["a", "b", "c"] } }); + } + + #[test] + fn test_name_ref_resolution() { + let maps = make_maps( + vec![("n", "status")], + vec![(":v", AttributeValue::S("active".to_string()))], + ); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("#n".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":v".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.status.S": "active" }); + } + + #[test] + fn test_ne_comparison() { + let maps = make_maps( + vec![], + vec![(":v", AttributeValue::S("deleted".to_string()))], + ); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "status".to_string(), + )])), + op: CompareOp::Ne, + right: Box::new(Expr::Placeholder(":v".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.status.S": { "$ne": "deleted" } }); + } + + #[test] + fn test_regex_escape() { + assert_eq!(regex_escape("user.name"), "user\\.name"); + assert_eq!(regex_escape("a+b"), "a\\+b"); + assert_eq!(regex_escape("normal"), "normal"); + } + + #[test] + fn test_attribute_type() { + let maps = make_maps(vec![], vec![(":t", AttributeValue::S("S".to_string()))]); + let expr = Expr::Function { + name: "attribute_type".to_string(), + args: vec![ + Expr::Path(vec![PathElement::Attribute("field".to_string())]), + Expr::Placeholder(":t".to_string()), + ], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.field.S": { "$exists": true } }); + } +} diff --git a/crates/storage-mongodb/src/config.rs b/crates/storage-mongodb/src/config.rs new file mode 100644 index 00000000..9e5a4e26 --- /dev/null +++ b/crates/storage-mongodb/src/config.rs @@ -0,0 +1,58 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Configuration for `MongoDB` storage backend. + +use serde::{Deserialize, Serialize}; + +/// `MongoDB` storage backend configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MongoStorageConfig { + /// `MongoDB` connection string (mongodb://...) + pub connection_string: String, + /// Maximum concurrent connections for data operations + #[serde(default = "default_max_connections")] + pub max_connections: u32, + /// Maximum concurrent connections for catalog/management operations + #[serde(default = "default_max_catalog_connections")] + pub max_catalog_connections: u32, +} + +fn default_max_connections() -> u32 { + 50 +} + +fn default_max_catalog_connections() -> u32 { + 20 +} + +impl extenddb_storage::config::StorageConfig for MongoStorageConfig { + fn connection_config(&self) -> &str { + &self.connection_string + } + + fn max_connections(&self) -> u32 { + self.max_connections + } + + fn max_catalog_connections(&self) -> u32 { + self.max_catalog_connections + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +impl TryFrom for MongoStorageConfig { + type Error = toml::de::Error; + + fn try_from(table: toml::Table) -> Result { + let value = toml::Value::Table(table); + value.try_into() + } +} diff --git a/crates/storage-mongodb/src/credential_store.rs b/crates/storage-mongodb/src/credential_store.rs new file mode 100644 index 00000000..e0aedfcf --- /dev/null +++ b/crates/storage-mongodb/src/credential_store.rs @@ -0,0 +1,216 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Credential store implementation for `MongoDB`. + +use mongodb::bson::{Document, doc}; + +use extenddb_auth::{CredentialStore, StoredCredential}; +use extenddb_core::error::DynamoDbError; + +/// `MongoDB` credential store for authentication. +pub struct MongoCredentialStore { + client: mongodb::Client, + encryption_key: String, +} + +impl MongoCredentialStore { + #[must_use] + pub fn new(client: mongodb::Client, encryption_key: String) -> Self { + Self { + client, + encryption_key, + } + } + + fn catalog_db(&self) -> mongodb::Database { + self.client.database("extenddb_catalog") + } + + async fn lookup_user_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let coll = self.catalog_db().collection::("access_keys"); + let doc = coll + .find_one(doc! { "access_key_id": access_key_id }) + .await + .map_err(|e| { + tracing::error!("Credential lookup failed for access key {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + let Some(key_doc) = doc else { + return Ok(None); + }; + + let encrypted = match key_doc.get_binary_generic("secret_key_encrypted") { + Ok(bytes) => bytes.clone(), + Err(_) => return Ok(None), + }; + let account_id = key_doc.get_str("account_id").unwrap_or_default().to_owned(); + let user_name = key_doc.get_str("user_name").unwrap_or_default().to_owned(); + let is_active = key_doc.get_bool("is_active").unwrap_or(true); + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("Secret key decryption failed for access key {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: user_name, + session_name: None, + is_session: false, + session_token: None, + is_active, + expires_at: None, + })) + } + + async fn lookup_session_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let coll = self.catalog_db().collection::("iam_sessions"); + let doc = coll + .find_one(doc! { "access_key_id": access_key_id }) + .await + .map_err(|e| { + tracing::error!( + "Session credential lookup failed for access key {access_key_id}: {e}" + ); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + let Some(session_doc) = doc else { + return Ok(None); + }; + + let encrypted = match session_doc.get_binary_generic("secret_key_encrypted") { + Ok(bytes) => bytes.clone(), + Err(_) => return Ok(None), + }; + let account_id = session_doc + .get_str("account_id") + .unwrap_or_default() + .to_owned(); + let role_name = session_doc + .get_str("role_name") + .unwrap_or_default() + .to_owned(); + let session_name = session_doc + .get_str("session_name") + .unwrap_or_default() + .to_owned(); + let session_token = session_doc + .get_str("session_token") + .unwrap_or_default() + .to_owned(); + + let expires_at = session_doc.get_datetime("expires_at").map_err(|_| { + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let expires_ts = time::OffsetDateTime::from_unix_timestamp_nanos( + i128::from(expires_at.timestamp_millis()) * 1_000_000, + ) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH); + + if expires_ts < time::OffsetDateTime::now_utc() { + return Err(DynamoDbError::ExpiredTokenException( + "The security token included in the request is expired".to_owned(), + )); + } + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!( + "Session secret key decryption failed for access key {access_key_id}: {e}" + ); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: role_name, + session_name: Some(session_name), + is_session: true, + session_token: Some(session_token), + is_active: true, + expires_at: Some(expires_ts), + })) + } +} + +#[async_trait::async_trait] +impl CredentialStore for MongoCredentialStore { + async fn lookup_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + if access_key_id.starts_with("AKIA") { + return self.lookup_user_credential(access_key_id).await; + } + + if access_key_id.starts_with("ASIA") { + return self.lookup_session_credential(access_key_id).await; + } + + Ok(None) + } +} + +// ── Crypto helpers ────────────────────────────────────────────────────── + +fn decrypt_secret(encrypted: &[u8], key_b64: &str, aad: &str) -> Result { + use aes_gcm::Aes256Gcm; + use aes_gcm::KeyInit; + use aes_gcm::aead::Aead; + use aes_gcm::aead::Payload; + use base64::Engine; + + if encrypted.len() < 28 { + return Err( + "ciphertext too short (need at least 12-byte nonce + 16-byte auth tag)".to_owned(), + ); + } + + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + + let key = aes_gcm::Key::::from_slice(&key_bytes); + let cipher = Aes256Gcm::new(key); + let nonce = aes_gcm::Nonce::from_slice(&encrypted[..12]); + + // Try with AAD first (CB-11 format). + let payload_with_aad = Payload { + msg: &encrypted[12..], + aad: aad.as_bytes(), + }; + if let Ok(plaintext_bytes) = cipher.decrypt(nonce, payload_with_aad) { + return String::from_utf8(plaintext_bytes) + .map_err(|e| format!("decrypted secret is not valid UTF-8: {e}")); + } + + // Fall back to without AAD (pre-CB-11 format). + tracing::debug!("Decrypting secret without AAD (pre-CB-11 format) for {aad}"); + let plaintext_bytes = cipher + .decrypt(nonce, &encrypted[12..]) + .map_err(|e| format!("decrypt: {e}"))?; + + String::from_utf8(plaintext_bytes) + .map_err(|e| format!("decrypted secret is not valid UTF-8: {e}")) +} diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs new file mode 100644 index 00000000..9f8ee45c --- /dev/null +++ b/crates/storage-mongodb/src/data/mod.rs @@ -0,0 +1,193 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Data engine helpers for the `MongoDB` backend. +//! +//! Contains document conversion, collection naming, and key extraction utilities. + +use std::collections::BTreeMap; + +use bson::{Bson, Document, doc}; + +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{composite_pk_to_text, pk_to_text, sk_info}; + +/// Returns the `MongoDB` collection name for a `DynamoDB` table. +pub fn data_collection_name(table_id: &str) -> String { + format!("_ddb_{table_id}") +} + +/// Returns the `MongoDB` collection name for a secondary index. +pub fn index_collection_name(index_id: &str) -> String { + format!("_ddb_{index_id}") +} + +/// Convert a `DynamoDB` Item to a `MongoDB` BSON document for storage. +/// +/// Document structure: `{ _id, pk, sk_s/sk_n/sk_b, item_data }` +pub fn item_to_document( + item: &Item, + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let pk_text = composite_pk_to_text(item, key_schema)?; + + // Serialize the full item as item_data + let item_json = + serde_json::to_value(item).map_err(|e| StorageError::Internal(e.to_string()))?; + let item_bson = bson::to_bson(&item_json).map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut doc = Document::new(); + + // Build the _id field + if let Some((sk_name, sk_type)) = sk_info(key_schema, attribute_definitions) { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk_text = sk_to_text(sk_value)?; + doc.insert("_id", format!("{pk_text}#{sk_text}")); + doc.insert("pk", pk_text); + + // Insert the typed sort key field + match sk_type { + ScalarAttributeType::S => { + if let AttributeValue::S(s) = sk_value { + doc.insert("sk_s", s.clone()); + } + } + ScalarAttributeType::N => { + if let AttributeValue::N(n) = sk_value { + // Store as Decimal128 for proper numeric ordering + match n.parse::() { + Ok(d) => { + doc.insert("sk_n", d); + } + Err(_) => { + // Fallback: try parsing as f64 + if let Ok(f) = n.parse::() { + doc.insert("sk_n", f); + } else { + return Err(StorageError::Internal(format!( + "Cannot convert sort key '{n}' to numeric BSON type" + ))); + } + } + } + } + } + ScalarAttributeType::B => { + if let AttributeValue::B(b) = sk_value { + doc.insert( + "sk_b", + bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: b.clone(), + }, + ); + } + } + } + } else { + // PK-only table + doc.insert("_id", pk_text.clone()); + doc.insert("pk", pk_text); + } + + doc.insert("item_data", item_bson); + Ok(doc) +} + +/// Convert a `MongoDB` document back to a `DynamoDB` Item. +pub fn document_to_item(doc: &Document) -> Result { + let item_data = doc + .get("item_data") + .ok_or_else(|| StorageError::Internal("Document missing item_data field".to_string()))?; + + let json_value: serde_json::Value = bson::from_bson(item_data.clone()) + .map_err(|e| StorageError::Internal(format!("BSON to JSON conversion error: {e}")))?; + + let item: Item = serde_json::from_value(json_value) + .map_err(|e| StorageError::Internal(format!("JSON to Item conversion error: {e}")))?; + + Ok(item) +} + +/// Convert a sort key value to text for use in the _id field. +fn sk_to_text(value: &AttributeValue) -> Result { + match value { + AttributeValue::S(s) => Ok(s.clone()), + AttributeValue::N(n) => Ok(n.clone()), + AttributeValue::B(b) => { + use base64::Engine; + Ok(base64::engine::general_purpose::STANDARD.encode(b)) + } + _ => Err(StorageError::Internal( + "sort key must be S, N, or B".to_owned(), + )), + } +} + +/// Build a primary key filter for `MongoDB` queries. +pub fn pk_filter( + key: &Item, + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let pk_text = composite_pk_to_text(key, key_schema)?; + let mut filter = doc! { "pk": &pk_text }; + + if let Some((sk_name, sk_type)) = sk_info(key_schema, attribute_definitions) { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key in key".to_owned()))?; + match sk_type { + ScalarAttributeType::S => { + if let AttributeValue::S(s) = sk_value { + filter.insert("sk_s", s.clone()); + } + } + ScalarAttributeType::N => { + if let AttributeValue::N(n) = sk_value { + match n.parse::() { + Ok(d) => { + filter.insert("sk_n", d); + } + Err(_) => { + if let Ok(f) = n.parse::() { + filter.insert("sk_n", f); + } + } + } + } + } + ScalarAttributeType::B => { + if let AttributeValue::B(b) = sk_value { + filter.insert( + "sk_b", + bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: b.clone(), + }, + ); + } + } + } + } + + Ok(filter) +} + +/// Get the sort key column name for a table. +pub fn sk_field_name( + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Option<&'static str> { + sk_info(key_schema, attribute_definitions).map(|(_, sk_type)| match sk_type { + ScalarAttributeType::S => "sk_s", + ScalarAttributeType::N => "sk_n", + ScalarAttributeType::B => "sk_b", + }) +} diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs new file mode 100644 index 00000000..e0b10646 --- /dev/null +++ b/crates/storage-mongodb/src/data_engine.rs @@ -0,0 +1,2156 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DataEngine` trait implementation for `MongoEngine`. + +use bson::{Document, doc}; +use futures::future::BoxFuture; +use mongodb::options::{FindOneAndDeleteOptions, FindOneAndReplaceOptions, ReturnDocument}; + +use extenddb_core::expression::{ + self, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, UpdateAction, +}; +use extenddb_core::types::{ + AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, StreamEventName, + StreamRecord, StreamRecordData, TableKeyInfo, extract_key, item_size_bytes, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{ + composite_pk_to_text, encode_netstring_composite, pk_to_text, sk_info, +}; +use extenddb_storage::{ + DataEngine, IdempotencyKey, ItemPairResult, QueryResult, StreamCapture, StreamEngine, + TransactGetOp, TransactWriteOp, +}; + +use crate::MongoEngine; +use crate::condition::condition_to_filter; +use crate::data::{ + data_collection_name, document_to_item, item_to_document, pk_filter, sk_field_name, +}; + +use extenddb_core::types::{AttributeDefinition, Projection, ProjectionType}; + +impl DataEngine for MongoEngine { + fn put_item( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let item = item.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.put_item_impl( + &key_info, + item, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn get_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + Box::pin(async move { self.get_item_impl(&key_info, &key).await }) + } + + fn delete_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.delete_item_impl( + &key_info, + &key, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn update_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, ItemPairResult> { + let key_info = key_info.clone(); + let key = key.clone(); + let actions = actions.to_vec(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.update_item_impl( + &key_info, + &key, + &actions, + return_old, + return_new, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn query( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + let key_info = key_info.clone(); + let key_condition = key_condition.clone(); + let maps = maps.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(std::string::ToString::to_string); + Box::pin(async move { + self.query_impl( + &key_info, + &key_condition, + &maps, + forward, + limit, + exclusive_start_key.as_ref(), + index_name.as_deref(), + ) + .await + }) + } + + fn scan( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + let key_info = key_info.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(std::string::ToString::to_string); + Box::pin(async move { + self.scan_impl( + &key_info, + limit, + exclusive_start_key.as_ref(), + segment, + total_segments, + index_name.as_deref(), + ) + .await + }) + } + + fn transact_get_items( + &self, + ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + let ops_data: Vec<_> = ops + .iter() + .map(|op| (op.key_info.clone(), op.key.clone())) + .collect(); + Box::pin(async move { self.transact_get_items_impl(&ops_data).await }) + } + + fn transact_write_items( + &self, + ops: &[TransactWriteOp<'_>], + idempotency: Option>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let ops_owned: Vec<_> = ops.iter().map(clone_transact_write_op).collect(); + let idempotency_owned = idempotency.map(|k| { + ( + k.account_id.to_owned(), + k.token.to_owned(), + k.fingerprint.to_owned(), + ) + }); + Box::pin(async move { + let idem_ref = idempotency_owned.as_ref().map(|(a, t, f)| IdempotencyKey { + account_id: a.as_str(), + token: t.as_str(), + fingerprint: f.as_str(), + }); + self.transact_write_items_impl(&ops_owned, idem_ref).await + }) + } + + fn cleanup_expired_idempotency_tokens( + &self, + max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let coll = self.data_db.collection::("idempotency_tokens"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(max_age_seconds as u64); + let cutoff_bson = mongodb::bson::DateTime::from_millis(cutoff.unix_timestamp() * 1000); + let result = coll + .delete_many(doc! { "created_at": { "$lt": cutoff_bson } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.deleted_count) + }) + } +} + +impl MongoEngine { + async fn write_stream_inline( + &self, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, + ) -> Result<(), StorageError> { + use extenddb_core::types::StreamViewType; + + let source_item = new_item.or(old_item); + let Some(source) = source_item else { + return Ok(()); + }; + + let event = match (old_item, new_item) { + (None, Some(_)) => StreamEventName::Insert, + (Some(_), Some(_)) => StreamEventName::Modify, + (Some(_), None) => StreamEventName::Remove, + (None, None) => return Ok(()), + }; + + let keys: std::collections::BTreeMap = key_info + .key_schema + .iter() + .filter_map(|ks| { + source + .get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let new_image = match capture.view_type { + StreamViewType::NewImage | StreamViewType::NewAndOldImages => new_item.cloned(), + _ => None, + }; + let old_image = match capture.view_type { + StreamViewType::OldImage | StreamViewType::NewAndOldImages => old_item.cloned(), + _ => None, + }; + + let size = source_item.map_or(0, |i| i64::try_from(item_size_bytes(i)).unwrap_or(i64::MAX)); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, b) + } + _ => String::new(), + }) + .unwrap_or_default(); + + let shard_id = self + .assign_shard(&key_info.account_id, &key_info.table_name, &pk_str) + .await?; + let seq = self.next_sequence_number(&shard_id).await?; + + let record = StreamRecord { + event_id: uuid::Uuid::new_v4().to_string(), + event_name: event, + event_version: "1.1".to_owned(), + event_source: "aws:dynamodb".to_owned(), + aws_region: capture.region.to_string(), + dynamodb: StreamRecordData { + approximate_creation_date_time: i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) + .unwrap_or(i64::MAX), + keys, + new_image, + old_image, + sequence_number: seq, + size_bytes: size, + stream_view_type: capture.view_type, + }, + user_identity: capture.user_identity.clone(), + }; + + self.write_stream_record( + &key_info.account_id, + &record, + &shard_id, + &key_info.table_name, + ) + .await + } + + async fn put_item_impl( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + let key_filter = pk_filter(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + session + .start_transaction() + .with_options(tx_options) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old_item: Option; + let return_val: Option; + + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(ref existing) = existing_doc { + let existing_item = document_to_item(existing)?; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + let _ = session.abort_transaction().await; + return Err(StorageError::ConditionFailed(Some(existing_item))); + } + let opts = FindOneAndReplaceOptions::builder() + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter, new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + old_item = old_doc.as_ref().map(document_to_item).transpose()?; + return_val = if return_old { old_item.clone() } else { None }; + } else { + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(cond, &empty, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + let _ = session.abort_transaction().await; + return Err(StorageError::ConditionFailed(None)); + } + let result = coll.insert_one(new_doc).session(&mut session).await; + if let Err(e) = result { + if e.to_string().contains("E11000") { + let _ = session.abort_transaction().await; + let winner = coll + .find_one(key_filter) + .await + .map_err(|e2| StorageError::Internal(e2.to_string()))? + .map(|d| document_to_item(&d)) + .transpose()?; + return Err(StorageError::ConditionFailed(winner)); + } + let _ = session.abort_transaction().await; + return Err(StorageError::Internal(e.to_string())); + } + old_item = None; + return_val = None; + } + } else { + let opts = FindOneAndReplaceOptions::builder() + .upsert(true) + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter, new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + old_item = old_doc.as_ref().map(document_to_item).transpose()?; + return_val = if return_old { old_item.clone() } else { None }; + } + + // Sync GSI collections within the transaction + self.sync_indexes_in_session(key_info, old_item.as_ref(), Some(&item), &mut session) + .await?; + + // Write stream record within the transaction + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + old_item.as_ref(), + Some(&item), + &mut session, + ) + .await?; + } + + session + .commit_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(return_val) + } + + async fn get_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> Result, StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let doc = coll + .find_one(filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + doc.as_ref().map(document_to_item).transpose() + } + + async fn delete_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + session + .start_transaction() + .with_options(tx_options) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let deleted_item: Option; + + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(ref existing) = existing_doc { + let existing_item = document_to_item(existing)?; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + let _ = session.abort_transaction().await; + return Err(StorageError::ConditionFailed(Some(existing_item))); + } + coll.delete_one(key_filter) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + deleted_item = Some(existing_item); + } else { + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(cond, &empty, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + let _ = session.abort_transaction().await; + return Err(StorageError::ConditionFailed(None)); + } + deleted_item = None; + } + } else { + let old_doc = coll + .find_one_and_delete(key_filter) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + deleted_item = old_doc.as_ref().map(document_to_item).transpose()?; + } + + // Sync GSI collections within the transaction + if deleted_item.is_some() { + self.sync_indexes_in_session(key_info, deleted_item.as_ref(), None, &mut session) + .await?; + } + + // Write stream record within the transaction + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + deleted_item.as_ref(), + None, + &mut session, + ) + .await?; + } + + session + .commit_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(if return_old { deleted_item } else { None }) + } + + #[allow(clippy::too_many_arguments)] + async fn update_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result<(Option, Option), StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + + // Fast path: use native MongoDB atomic operators when possible. + // This avoids transactions and retries for simple unconditional updates. + if condition.is_none() && !return_old && stream.is_none() { + if let Some(mongo_update) = self.try_build_native_update(actions, maps) { + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(ReturnDocument::After) + .build(); + let result_doc = coll + .find_one_and_update(key_filter, mongo_update) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let new_item = if return_new { + result_doc.as_ref().map(document_to_item).transpose()? + } else { + None + }; + + // Sync GSI (non-transactional but data write is atomic) + if let Some(ref doc) = result_doc { + let item = document_to_item(doc)?; + self.sync_indexes(key_info, None, Some(&item)).await?; + } + + return Ok((None, new_item)); + } + } + + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + for _attempt in 0..50 { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let current_version = existing_doc + .as_ref() + .and_then(|d| d.get_i64("_v").ok()) + .unwrap_or(0); + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc)? + } else { + key.clone() + }; + + if let Some(cond) = condition { + let eval_item = if existing_doc.is_some() { + &existing_item + } else { + &std::collections::BTreeMap::new() + }; + let passed = expression::evaluate_condition(cond, eval_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + let _ = session.abort_transaction().await; + return Err(StorageError::ConditionFailed(if existing_doc.is_some() { + Some(existing_item.clone()) + } else { + None + })); + } + } + + let need_old = return_old || stream.is_some(); + let old_item = if need_old { + Some(existing_item.clone()) + } else { + None + }; + + let mut new_item = existing_item; + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + + let mut new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let new_version = current_version + 1; + new_doc.insert("_v", new_version); + + if existing_doc.is_some() { + let mut versioned_filter = key_filter.clone(); + if current_version == 0 { + versioned_filter.insert("_v", doc! { "$not": { "$gt": 0_i64 } }); + } else { + versioned_filter.insert("_v", current_version); + } + let result = coll + .replace_one(versioned_filter, new_doc) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.matched_count == 0 { + let _ = session.abort_transaction().await; + let base_us = 50u64.saturating_mul(1u64 << _attempt.min(8)); + let jitter = rand::random_range(0..=base_us); + tokio::time::sleep(std::time::Duration::from_micros(jitter)).await; + continue; + } + } else { + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Sync GSI collections within the transaction + self.sync_indexes_in_session( + key_info, + old_item.as_ref(), + Some(&new_item), + &mut session, + ) + .await?; + + // Write stream record within the transaction + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + old_item.as_ref(), + Some(&new_item), + &mut session, + ) + .await?; + } + + session + .commit_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old_item_result = if return_old { old_item } else { None }; + let new_item_result = if return_new { Some(new_item) } else { None }; + return Ok((old_item_result, new_item_result)); + } + + Err(StorageError::Internal( + "UpdateItem: too many version conflicts, giving up".to_owned(), + )) + } + + #[allow(clippy::too_many_arguments)] + async fn query_impl( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + use futures::TryStreamExt; + + // Determine collection and effective key schema for the query target + let (coll_name, effective_key_schema) = if let Some(idx_name) = index_name { + let idx_info = self + .index_info_by_table_id_impl(&key_info.table_id, idx_name) + .await?; + ( + data_collection_name(&idx_info.index_id), + idx_info.key_schema.clone(), + ) + } else { + ( + data_collection_name(&key_info.table_id), + key_info.key_schema.clone(), + ) + }; + let coll = self.data_db.collection::(&coll_name); + + // Build the query filter — handle multi-part HASH keys + let pk_text = if key_condition.extra_pk_conditions.is_empty() { + let pk_value = resolve_key_expr(&key_condition.pk_value, maps)?; + pk_to_text(&pk_value) + .map_err(|e| StorageError::Internal(e.to_string()))? + .into_owned() + } else { + let mut parts = Vec::with_capacity(1 + key_condition.extra_pk_conditions.len()); + let first_val = resolve_key_expr(&key_condition.pk_value, maps)?; + parts.push( + pk_to_text(&first_val) + .map_err(|e| StorageError::Internal(e.to_string()))? + .into_owned(), + ); + for (_, value) in &key_condition.extra_pk_conditions { + let val = resolve_key_expr(value, maps)?; + parts.push( + pk_to_text(&val) + .map_err(|e| StorageError::Internal(e.to_string()))? + .into_owned(), + ); + } + encode_netstring_composite(&parts) + }; + + let mut filter = doc! { "pk": &pk_text }; + + // Determine sort key field using effective key schema + let sk_field = sk_field_name(&effective_key_schema, &key_info.attribute_definitions); + + // Apply sort key condition + if let Some(ref sk_cond) = key_condition.sk_condition { + if let Some(sk_f) = sk_field { + let sk_filter = build_sk_filter(sk_cond, sk_f, maps)?; + for (k, v) in sk_filter { + filter.insert(k, v); + } + } + } + + // Apply exclusive_start_key pagination + if let Some(start_key) = exclusive_start_key { + if let Some(sk_f) = sk_field { + // Get the sort key value from the start key + if let Some((sk_name, sk_type)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + if let Some(sk_val) = start_key.get(sk_name) { + let sk_bson = sk_to_bson(sk_val, sk_type)?; + if forward { + filter.insert(sk_f, doc! { "$gt": sk_bson }); + } else { + filter.insert(sk_f, doc! { "$lt": sk_bson }); + } + } + } + } + } + + // Build sort direction + let sort_direction = if forward { 1 } else { -1 }; + let sort_doc = if let Some(sk_f) = sk_field { + doc! { sk_f: sort_direction } + } else { + doc! { "pk": sort_direction } + }; + + // Apply limit (fetch one extra for pagination) + let fetch_limit = limit.map(|l| l + 1); + + let collation_opt = if sk_field == Some("sk_s") { + Some( + mongodb::options::Collation::builder() + .locale("simple".to_string()) + .build(), + ) + } else { + None + }; + + let opts = mongodb::options::FindOptions::builder() + .sort(sort_doc) + .limit(fetch_limit) + .collation(collation_opt) + .build(); + + let cursor = coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut items: Vec = docs + .iter() + .map(document_to_item) + .collect::, _>>()?; + + // Post-fetch filtering for binary begins_with (BSON Binary comparison + // sorts by length first, making $gte/$lt unreliable for prefix matching). + if let Some(ref sk_cond) = key_condition.sk_condition { + if let SortKeyCondition::BeginsWith { prefix, .. } = sk_cond { + let prefix_av = resolve_key_expr(prefix, maps)?; + if let AttributeValue::B(ref prefix_bytes) = prefix_av { + if let Some((sk_name, _)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + items.retain(|item| { + item.get(sk_name) + .and_then(|v| { + if let AttributeValue::B(b) = v { + Some(b.starts_with(prefix_bytes)) + } else { + None + } + }) + .unwrap_or(false) + }); + } + } + } + } + + // Handle pagination + let last_evaluated_key = if let Some(l) = limit { + #[allow(clippy::cast_sign_loss)] + let l_usize = l as usize; + if items.len() > l_usize { + items.truncate(l_usize); + items + .last() + .map(|item| extract_key(item, &key_info.key_schema)) + } else { + None + } + } else { + None + }; + + Ok((items, last_evaluated_key)) + } + + async fn scan_impl( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + use futures::TryStreamExt; + + let coll_name = if let Some(idx_name) = index_name { + let idx_info = self + .index_info_by_table_id_impl(&key_info.table_id, idx_name) + .await?; + data_collection_name(&idx_info.index_id) + } else { + data_collection_name(&key_info.table_id) + }; + let coll = self.data_db.collection::(&coll_name); + + let mut filter = Document::new(); + + // Apply exclusive_start_key for pagination (using _id for scan ordering) + if let Some(start_key) = exclusive_start_key { + let start_pk = composite_pk_to_text(start_key, &key_info.key_schema)?; + if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + if let Some(sk_val) = start_key.get(sk_name) { + let sk_text = match sk_val { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(b) + } + _ => return Err(StorageError::Internal("invalid sk type".to_string())), + }; + let start_id = format!("{start_pk}#{sk_text}"); + filter.insert("_id", doc! { "$gt": start_id }); + } + } else { + filter.insert("_id", doc! { "$gt": &start_pk }); + } + } + + // Parallel scan segment filtering + // segment/total_segments use CRC32 hash of pk modulo total_segments + let apply_segment_filter = segment.is_some() && total_segments.is_some(); + + // Apply limit + let fetch_limit = limit.map(|l| { + let extra = l + 1; + if apply_segment_filter { + extra * total_segments.unwrap_or(1) + } else { + extra + } + }); + + let opts = mongodb::options::FindOptions::builder() + .sort(doc! { "_id": 1 }) + .limit(fetch_limit) + .build(); + + let cursor = coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut items: Vec = Vec::new(); + for doc in &docs { + let item = document_to_item(doc)?; + + // Apply segment filter if needed + if let (Some(seg), Some(total)) = (segment, total_segments) { + let pk_text = composite_pk_to_text(&item, &key_info.key_schema)?; + let hash = crc32fast::hash(pk_text.as_bytes()); + #[allow(clippy::cast_sign_loss)] + let total_u = total as u32; + #[allow(clippy::cast_sign_loss)] + let seg_u = seg as u32; + if hash % total_u != seg_u { + continue; + } + } + + items.push(item); + + // Check if we have enough items + if let Some(l) = limit { + #[allow(clippy::cast_sign_loss)] + if items.len() > l as usize { + break; + } + } + } + + // Handle pagination + let last_evaluated_key = if let Some(l) = limit { + #[allow(clippy::cast_sign_loss)] + let l_usize = l as usize; + if items.len() > l_usize { + items.truncate(l_usize); + items + .last() + .map(|item| extract_key(item, &key_info.key_schema)) + } else { + None + } + } else { + None + }; + + Ok((items, last_evaluated_key)) + } + + // ── Native MongoDB Update (fast path) ───────────────────────────── + + fn try_build_native_update( + &self, + actions: &[UpdateAction], + maps: &ExpressionMaps, + ) -> Option { + let mut inc_doc = Document::new(); + let mut set_doc = Document::new(); + let mut unset_doc = Document::new(); + + for action in actions { + match action { + UpdateAction::Add { path, value } => { + if path.len() != 1 { + return None; + } + let attr_name = match &path[0] { + PathElement::Attribute(name) => name, + _ => return None, + }; + let val = match value { + Expr::Placeholder(name) => maps.resolve_value(name).ok()?, + _ => return None, + }; + match val { + AttributeValue::N(n) => { + let field = format!("item_data.{attr_name}.N"); + // Store numeric increment as string (matching our storage format) + // Use $inc on a helper field and reconcile, OR use a different approach. + // Actually: item_data stores N as string. We can't $inc a string. + // We need a numeric shadow field for $inc to work. + // For now, only optimize if we can parse as i64. + if let Ok(i) = n.parse::() { + // Use $inc on a numeric shadow field, then $set the string representation. + // Actually this won't work atomically in one update... + // The simplest correct approach: use $inc on item_data.attr.N + // BUT item_data.attr.N is stored as a string, not a number. + // MongoDB $inc doesn't work on strings. + // FALLBACK: we cannot use the native fast path for numeric ADD + // unless we change the storage format. Give up. + let _ = (field, i); + return None; + } + return None; + } + AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_) => { + // Set ADD — could use $addToSet but storage format is complex + return None; + } + _ => return None, + } + } + UpdateAction::Set { path, value } => { + if path.len() != 1 { + return None; + } + let attr_name = match &path[0] { + PathElement::Attribute(name) => name, + _ => return None, + }; + let val = match value { + Expr::Placeholder(name) => maps.resolve_value(name).ok()?, + _ => return None, // complex expressions (if_not_exists, list_append, arithmetic) + }; + let field = format!("item_data.{attr_name}"); + let val_json = serde_json::to_value(val).ok()?; + let val_bson = bson::to_bson(&val_json).ok()?; + set_doc.insert(field, val_bson); + } + UpdateAction::Remove { path } => { + if path.len() != 1 { + return None; + } + let attr_name = match &path[0] { + PathElement::Attribute(name) => name, + _ => return None, + }; + let field = format!("item_data.{attr_name}"); + unset_doc.insert(field, 1); + } + UpdateAction::Delete { .. } => { + return None; + } + } + } + + let mut update = Document::new(); + if !inc_doc.is_empty() { + update.insert("$inc", inc_doc); + } + if !set_doc.is_empty() { + update.insert("$set", set_doc); + } + if !unset_doc.is_empty() { + update.insert("$unset", unset_doc); + } + + if update.is_empty() { + return None; + } + + Some(update) + } + + // ── GSI Sync ────────────────────────────────────────────────────── + + async fn sync_indexes( + &self, + key_info: &TableKeyInfo, + old_item: Option<&Item>, + new_item: Option<&Item>, + ) -> Result<(), StorageError> { + use futures::TryStreamExt; + + // Fast path: skip catalog query if we know this table has no GSIs + if let Some(entry) = self.gsi_cache.get(&key_info.table_id) { + if !*entry { + return Ok(()); + } + } + + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut cursor = indexes_coll + .find(doc! { "_id.table_id": &key_info.table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut found_any = false; + while let Some(idx_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + found_any = true; + let index_id = match idx_doc.get_str("index_id") { + Ok(id) => id.to_string(), + Err(_) => continue, + }; + let idx_key_schema: Vec = match idx_doc.get("key_schema") { + Some(ks) => bson::from_bson(ks.clone()).unwrap_or_default(), + None => continue, + }; + let projection: Projection = match idx_doc.get("projection") { + Some(p) => bson::from_bson(p.clone()).unwrap_or(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + None => Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }; + + let idx_coll_name = data_collection_name(&index_id); + let idx_coll = self.data_db.collection::(&idx_coll_name); + + // Delete old index entry + if let Some(old) = old_item { + if item_has_index_keys(old, &idx_key_schema) { + let old_filter = + pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; + let _ = idx_coll.delete_one(old_filter).await; + } + } + + // Insert new index entry + if let Some(new) = new_item { + if item_has_index_keys(new, &idx_key_schema) { + let projected = + project_item(new, &idx_key_schema, &key_info.key_schema, &projection); + let idx_doc = item_to_document( + &projected, + &idx_key_schema, + &key_info.attribute_definitions, + )?; + let filter = + pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + } + + self.gsi_cache.insert(key_info.table_id.clone(), found_any); + Ok(()) + } + + async fn sync_indexes_in_session( + &self, + key_info: &TableKeyInfo, + old_item: Option<&Item>, + new_item: Option<&Item>, + session: &mut mongodb::ClientSession, + ) -> Result<(), StorageError> { + use futures::TryStreamExt; + + if let Some(entry) = self.gsi_cache.get(&key_info.table_id) { + if !*entry { + return Ok(()); + } + } + + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut cursor = indexes_coll + .find(doc! { "_id.table_id": &key_info.table_id }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut found_any = false; + while let Some(idx_doc) = cursor + .next(session) + .await + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + { + found_any = true; + let index_id = match idx_doc.get_str("index_id") { + Ok(id) => id.to_string(), + Err(_) => continue, + }; + let idx_key_schema: Vec = match idx_doc.get("key_schema") { + Some(ks) => bson::from_bson(ks.clone()).unwrap_or_default(), + None => continue, + }; + let projection: Projection = match idx_doc.get("projection") { + Some(p) => bson::from_bson(p.clone()).unwrap_or(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + None => Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }; + + let idx_coll_name = data_collection_name(&index_id); + let idx_coll = self.data_db.collection::(&idx_coll_name); + + if let Some(old) = old_item { + if item_has_index_keys(old, &idx_key_schema) { + let old_filter = + pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; + let _ = idx_coll.delete_one(old_filter).session(&mut *session).await; + } + } + + if let Some(new) = new_item { + if item_has_index_keys(new, &idx_key_schema) { + let projected = + project_item(new, &idx_key_schema, &key_info.key_schema, &projection); + let idx_doc = item_to_document( + &projected, + &idx_key_schema, + &key_info.attribute_definitions, + )?; + let filter = + pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + } + + self.gsi_cache.insert(key_info.table_id.clone(), found_any); + Ok(()) + } + + async fn write_stream_inline_in_session( + &self, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, + session: &mut mongodb::ClientSession, + ) -> Result<(), StorageError> { + use extenddb_core::types::StreamViewType; + + let source_item = new_item.or(old_item); + let Some(source) = source_item else { + return Ok(()); + }; + + let event = match (old_item, new_item) { + (None, Some(_)) => StreamEventName::Insert, + (Some(_), Some(_)) => StreamEventName::Modify, + (Some(_), None) => StreamEventName::Remove, + (None, None) => return Ok(()), + }; + + let keys: std::collections::BTreeMap = key_info + .key_schema + .iter() + .filter_map(|ks| { + source + .get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let new_image = match capture.view_type { + StreamViewType::NewImage | StreamViewType::NewAndOldImages => new_item.cloned(), + _ => None, + }; + let old_image = match capture.view_type { + StreamViewType::OldImage | StreamViewType::NewAndOldImages => old_item.cloned(), + _ => None, + }; + + let size = source_item.map_or(0, |i| i64::try_from(item_size_bytes(i)).unwrap_or(i64::MAX)); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, b) + } + _ => String::new(), + }) + .unwrap_or_default(); + + let shard_id = self + .assign_shard(&key_info.account_id, &key_info.table_name, &pk_str) + .await?; + let seq = self.next_sequence_number(&shard_id).await?; + + let record = StreamRecord { + event_id: uuid::Uuid::new_v4().to_string(), + event_name: event, + event_version: "1.1".to_owned(), + event_source: "aws:dynamodb".to_owned(), + aws_region: capture.region.to_string(), + dynamodb: StreamRecordData { + approximate_creation_date_time: i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) + .unwrap_or(i64::MAX), + keys, + new_image, + old_image, + sequence_number: seq, + size_bytes: size, + stream_view_type: capture.view_type, + }, + user_identity: capture.user_identity.clone(), + }; + + let record_json = + serde_json::to_value(&record).map_err(|e| StorageError::Internal(e.to_string()))?; + let record_bson = + bson::to_bson(&record_json).map_err(|e| StorageError::Internal(e.to_string()))?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": &key_info.account_id, "table_name": &key_info.table_name } }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Internal(format!("Table {} not found in catalog", key_info.table_name)) + })?; + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let records_coll = self.data_db.collection::("stream_records"); + records_coll + .insert_one(doc! { + "sequence_number": &record.dynamodb.sequence_number, + "shard_id": &shard_id, + "table_id": table_id, + "event_name": format!("{:?}", record.event_name), + "record_data": record_bson, + "created_at": mongodb::bson::DateTime::now(), + }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + } + + // ── Transactions ────────────────────────────────────────────────── + + async fn transact_get_items_impl( + &self, + ops: &[(TableKeyInfo, Item)], + ) -> Result>, StorageError> { + use extenddb_core::types::CancellationReason; + use extenddb_core::validation; + + // Validate key types before starting transaction + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut any_failed = false; + for (key_info, key) in ops { + match validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) { + Ok(()) => reasons.push(CancellationReason::none()), + Err(e) => { + any_failed = true; + reasons.push(CancellationReason::validation_error(e.to_string())); + } + } + } + if any_failed { + return Err(StorageError::TransactionCanceled(reasons)); + } + + // Use a MongoDB session with snapshot read concern for consistent reads + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .build(); + + session + .start_transaction() + .with_options(tx_options) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::with_capacity(ops.len()); + for (key_info, key) in ops { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let doc = coll + .find_one(filter) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let item = doc.as_ref().map(document_to_item).transpose()?; + results.push(item); + } + + session + .commit_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(results) + } + + async fn transact_write_items_impl( + &self, + ops: &[OwnedTransactWriteOp], + idempotency: Option>, + ) -> Result<(), StorageError> { + use extenddb_core::types::CancellationReason; + use extenddb_core::validation; + + // Start a MongoDB multi-document transaction + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + session + .start_transaction() + .with_options(tx_options) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Check idempotency token, scoped to the caller's account so that + // identical tokens from different accounts never collide. + if let Some(key) = idempotency { + let idem_coll = self.data_db.collection::("idempotency_tokens"); + let existing = idem_coll + .find_one(doc! { "account_id": key.account_id, "token": key.token }) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(existing_doc) = existing { + let stored_fp = existing_doc.get_str("fingerprint").unwrap_or_default(); + if stored_fp == key.fingerprint { + session + .abort_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Err(StorageError::IdempotentReplay); + } + session + .abort_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Err(StorageError::IdempotentMismatch); + } + + // Store the token + idem_coll + .insert_one(doc! { + "account_id": key.account_id, + "token": key.token, + "fingerprint": key.fingerprint, + "created_at": mongodb::bson::DateTime::now(), + }) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut any_failed = false; + + for op in ops { + let reason = self + .execute_transact_write_op_in_session(op, &mut session) + .await; + match reason { + Ok(()) => reasons.push(CancellationReason::none()), + Err(TransactOpError::Cancel(r)) => { + any_failed = true; + reasons.push(r); + } + Err(TransactOpError::Storage(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + + if any_failed { + let _ = session.abort_transaction().await; + return Err(StorageError::TransactionCanceled(reasons)); + } + + session + .commit_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + } + + async fn execute_transact_write_op_in_session( + &self, + op: &OwnedTransactWriteOp, + session: &mut mongodb::ClientSession, + ) -> Result<(), TransactOpError> { + use extenddb_core::types::CancellationReason; + use extenddb_core::validation; + + match op { + OwnedTransactWriteOp::Put { + key_info, + item, + condition, + maps, + .. + } => { + validation::validate_item_keys( + item, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(item, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + + if let Some(cond) = condition { + let existing_item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc).map_err(TransactOpError::Storage)? + } else { + Item::new() + }; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(None), + )); + } + } + + let new_doc = + item_to_document(item, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter, new_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + + Ok(()) + } + OwnedTransactWriteOp::Delete { + key_info, + key, + condition, + maps, + .. + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(|e| { + TransactOpError::Storage(StorageError::Internal(e.to_string())) + })?; + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc).map_err(TransactOpError::Storage)? + } else { + Item::new() + }; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(None), + )); + } + } + + coll.delete_one(key_filter) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + + Ok(()) + } + OwnedTransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + .. + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + + let mut item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc).map_err(TransactOpError::Storage)? + } else { + key.clone() + }; + + if let Some(cond) = condition { + let condition_item = if existing_doc.is_some() { + &item + } else { + &std::collections::BTreeMap::new() + }; + let passed = expression::evaluate_condition(cond, condition_item, maps) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(None), + )); + } + } + + expression::apply_update(actions, &mut item, maps).map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter, new_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + + Ok(()) + } + OwnedTransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + .. + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let existing_doc = coll + .find_one(key_filter) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc).map_err(TransactOpError::Storage)? + } else { + Item::new() + }; + + let passed = expression::evaluate_condition(condition, &existing_item, maps) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(None), + )); + } + + Ok(()) + } + } + } +} + +// ── Transaction helper types ────────────────────────────────────────── + +enum TransactOpError { + Cancel(extenddb_core::types::CancellationReason), + Storage(StorageError), +} + +/// Owned version of `TransactWriteOp` to allow moving into async blocks. +enum OwnedTransactWriteOp { + Put { + key_info: TableKeyInfo, + item: Item, + condition: Option, + maps: ExpressionMaps, + }, + Delete { + key_info: TableKeyInfo, + key: Item, + condition: Option, + maps: ExpressionMaps, + }, + Update { + key_info: TableKeyInfo, + key: Item, + actions: Vec, + condition: Option, + maps: ExpressionMaps, + }, + ConditionCheck { + key_info: TableKeyInfo, + key: Item, + condition: Expr, + maps: ExpressionMaps, + }, +} + +fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + .. + } => OwnedTransactWriteOp::Put { + key_info: (*key_info).clone(), + item: (*item).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + }, + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + .. + } => OwnedTransactWriteOp::Delete { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + }, + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + .. + } => OwnedTransactWriteOp::Update { + key_info: (*key_info).clone(), + key: (*key).clone(), + actions: actions.to_vec(), + condition: condition.cloned(), + maps: (*maps).clone(), + }, + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + .. + } => OwnedTransactWriteOp::ConditionCheck { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: (*condition).clone(), + maps: (*maps).clone(), + }, + } +} + +/// Resolve a key expression (Placeholder) to an `AttributeValue`. +fn resolve_key_expr(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Internal( + "expected placeholder in key condition".to_owned(), + )), + } +} + +/// Build a `MongoDB` filter for a sort key condition. +fn build_sk_filter( + sk_cond: &SortKeyCondition, + sk_field: &str, + maps: &ExpressionMaps, +) -> Result { + match sk_cond { + SortKeyCondition::Compare { op, value, .. } => { + let av = resolve_key_expr(value, maps)?; + let sk_type = infer_sk_type_from_field(sk_field); + let bson_val = sk_to_bson(&av, sk_type)?; + + let filter = match op { + extenddb_core::expression::CompareOp::Eq => doc! { sk_field: bson_val }, + extenddb_core::expression::CompareOp::Lt => doc! { sk_field: { "$lt": bson_val } }, + extenddb_core::expression::CompareOp::Le => doc! { sk_field: { "$lte": bson_val } }, + extenddb_core::expression::CompareOp::Gt => doc! { sk_field: { "$gt": bson_val } }, + extenddb_core::expression::CompareOp::Ge => doc! { sk_field: { "$gte": bson_val } }, + extenddb_core::expression::CompareOp::Ne => doc! { sk_field: { "$ne": bson_val } }, + }; + Ok(filter) + } + SortKeyCondition::Between { low, high, .. } => { + let sk_type = infer_sk_type_from_field(sk_field); + let low_av = resolve_key_expr(low, maps)?; + let high_av = resolve_key_expr(high, maps)?; + let low_bson = sk_to_bson(&low_av, sk_type)?; + let high_bson = sk_to_bson(&high_av, sk_type)?; + Ok(doc! { sk_field: { "$gte": low_bson, "$lte": high_bson } }) + } + SortKeyCondition::BeginsWith { prefix, .. } => { + let prefix_av = resolve_key_expr(prefix, maps)?; + match prefix_av { + AttributeValue::S(ref p) => { + // For begins_with on string sort keys: sk_s >= prefix AND sk_s < prefix + max_char + let upper = increment_string(p); + Ok(doc! { sk_field: { "$gte": p.as_str(), "$lt": &upper } }) + } + AttributeValue::B(ref _b) => { + // BSON Binary comparison sorts by length first, then by content. + // This means $gte/$lt range queries don't work for prefix matching + // when the prefix is shorter than the stored values. Return an empty + // filter here and let the caller do post-fetch prefix filtering. + Ok(Document::new()) + } + _ => Err(StorageError::Validation( + "begins_with requires string or binary sort key".to_string(), + )), + } + } + } +} + +/// Convert an `AttributeValue` sort key to the appropriate BSON type. +fn sk_to_bson( + av: &AttributeValue, + sk_type: ScalarAttributeType, +) -> Result { + match (sk_type, av) { + (ScalarAttributeType::S, AttributeValue::S(s)) => Ok(bson::Bson::String(s.clone())), + (ScalarAttributeType::N, AttributeValue::N(n)) => match n.parse::() { + Ok(d) => Ok(bson::Bson::Decimal128(d)), + Err(_) => { + if let Ok(f) = n.parse::() { + Ok(bson::Bson::Double(f)) + } else { + Err(StorageError::Internal(format!( + "cannot parse numeric sort key: {n}" + ))) + } + } + }, + (ScalarAttributeType::B, AttributeValue::B(b)) => Ok(bson::Bson::Binary(bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: b.clone(), + })), + _ => Err(StorageError::Internal("sort key type mismatch".to_string())), + } +} + +/// Infer the `ScalarAttributeType` from the sort key field name. +fn infer_sk_type_from_field(field: &str) -> ScalarAttributeType { + if field.ends_with("_n") { + ScalarAttributeType::N + } else if field.ends_with("_b") { + ScalarAttributeType::B + } else { + ScalarAttributeType::S + } +} + +/// Increment a string to get the exclusive upper bound for `begins_with`. +fn increment_string(s: &str) -> String { + // Append the maximum Unicode code point + let mut result = s.to_string(); + result.push(char::MAX); + result +} + +/// Increment bytes to get the exclusive upper bound for `begins_with` on binary. +fn increment_bytes(b: &[u8]) -> Vec { + let mut result = b.to_vec(); + // Increment the last byte, with carry + let mut i = result.len(); + while i > 0 { + i -= 1; + if result[i] < 255 { + result[i] += 1; + return result; + } + result[i] = 0; + } + // All bytes were 0xFF; prepend a 0x01 byte (makes it longer) + result.insert(0, 1); + result +} + +fn item_has_index_keys(item: &Item, idx_key_schema: &[KeySchemaElement]) -> bool { + idx_key_schema + .iter() + .all(|ks| item.contains_key(&ks.attribute_name)) +} + +fn project_item( + item: &Item, + idx_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + projection: &Projection, +) -> Item { + match projection.projection_type { + ProjectionType::All => item.clone(), + ProjectionType::KeysOnly => { + let mut projected = Item::new(); + for ks in idx_key_schema.iter().chain(base_key_schema.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + projected + } + ProjectionType::Include => { + let mut projected = Item::new(); + // Always include key attributes + for ks in idx_key_schema.iter().chain(base_key_schema.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + // Include non-key attributes from projection + if let Some(ref attrs) = projection.non_key_attributes { + for attr in attrs { + if let Some(v) = item.get(attr) { + projected.insert(attr.clone(), v.clone()); + } + } + } + projected + } + } +} diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs new file mode 100644 index 00000000..40ee734b --- /dev/null +++ b/crates/storage-mongodb/src/lib.rs @@ -0,0 +1,270 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MongoDB` storage backend for extenddb. +//! +//! Implements the storage traits from `extenddb-storage` using `MongoDB` +//! as the backing store. Phase 1 covers `TableEngine`, `DataEngine`, +//! Bootstrapper, and `StorageConfig` with condition filter pushdown. + +#![allow(unused)] + +mod admin_store; +mod authorization_store; +mod backup_engine; +mod bootstrapper; +mod catalog_store; +pub mod condition; +pub mod config; +mod credential_store; +mod data; +mod data_engine; +mod management_store; +mod metadata_engine; +mod operations; +mod stream_engine; +mod table_engine; +mod ttl_worker; +mod worker_store; + +pub use bootstrapper::MongoBootstrapper; +pub use catalog_store::MongoCatalogStore; +pub use config::MongoStorageConfig; +pub use credential_store::MongoCredentialStore; + +use std::sync::Arc; + +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; + +// ============================================================================ +// OperationsEngineRegistration +// ============================================================================ + +inventory::submit! { + extenddb_storage::operations::OperationsEngineRegistration { + name: "mongodb", + operations: &operations::MongoOperationsEngine, + } +} + +// ============================================================================ +// BackendRegistration +// ============================================================================ + +inventory::submit! { + extenddb_storage::bootstrapper::BackendRegistration { + name: "mongodb", + factory: |config_path, cli_args| { + Box::pin(async move { + let store = MongoBootstrapper::from_config(&config_path, &cli_args).await?; + Ok(Box::new(store) as Box) + }) + } + } +} + +// ============================================================================ +// StorageConfigRegistration +// ============================================================================ + +inventory::submit! { + extenddb_storage::config::StorageConfigRegistration { + backend: "mongodb", + deserializer: |table| { + let config: MongoStorageConfig = table.clone().try_into() + .map_err(|e: toml::de::Error| format!("Failed to parse mongodb config: {e}"))?; + Ok(Box::new(config) as Box) + }, + } +} + +// ============================================================================ +// SettingsStoreRegistration +// ============================================================================ + +inventory::submit! { + extenddb_storage::settings_store::SettingsStoreRegistration { + backend: "mongodb", + factory: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed(e.to_string()))?; + Ok(Box::new(MongoCatalogStore::new(client)) as Box) + }) + }, + } +} + +// ============================================================================ +// DiagnosticsStoreRegistration +// ============================================================================ + +inventory::submit! { + extenddb_storage::diagnostics_store::DiagnosticsStoreRegistration { + backend: "mongodb", + factory: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed(e.to_string()))?; + Ok(Box::new(MongoCatalogStore::new(client)) as Box) + }) + }, + } +} + +// ============================================================================ +// ServerComponentsRegistration +// ============================================================================ + +use extenddb_auth::BuiltinAuthProvider; +use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; +use extenddb_storage::server_components::{ + BackendError, ServerComponents, ServerComponentsRegistration, +}; + +/// Backend-specific runtime hooks for `MongoDB`. +struct MongoRuntimeHooks { + engine: Arc, +} + +#[async_trait::async_trait] +impl ServerRuntimeHooks for MongoRuntimeHooks { + async fn spawn_workers(&self, ctx: &WorkerContext) { + let storage_for_ttl = self.engine.clone(); + let metrics = ctx.metrics.clone(); + tokio::spawn(async move { ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics).await }); + tracing::info!("MongoDB backend: TTL cleanup worker spawned"); + } + + fn backend_info(&self) -> Option { + Some("mongodb".to_string()) + } +} + +inventory::submit! { + ServerComponentsRegistration { + backend: "mongodb", + factory: |config, region| { + let connection_string = config.connection_config().to_string(); + let max_connections = config.max_connections(); + let region = region.to_string(); + Box::pin(async move { + // Create MongoEngine + let engine = MongoEngine::new(&connection_string, ®ion, max_connections) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: e.to_string(), + })?; + + let engine = Arc::new(engine); + + // Create catalog store + let catalog_client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: format!("Failed to create catalog client: {e}"), + })?; + + // Load encryption key from settings collection + let catalog_db = catalog_client.database("extenddb_catalog"); + let settings_coll = catalog_db.collection::("settings"); + let enc_key = settings_coll + .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? + .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) + .unwrap_or_default(); + + let catalog_store = Arc::new( + MongoCatalogStore::with_encryption_key(catalog_client, enc_key.clone()) + ) as Arc; + + // Create credential store. The bin layer wraps this in + // CachedCredentialStore using the operator-configured TTL + // before constructing the auth provider. + let auth_client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Auth client: {e}")))?; + let cred_store: Arc = + Arc::new(MongoCredentialStore::new(auth_client, enc_key)); + + // Create runtime hooks + let runtime_hooks = Box::new(MongoRuntimeHooks { + engine: engine.clone(), + }); + + Ok(ServerComponents { + engine, + catalog_store, + credential_store: cred_store, + runtime_hooks: Some(runtime_hooks), + }) + }) + }, + } +} + +// ============================================================================ +// MongoEngine +// ============================================================================ + +/// `MongoDB` storage backend. +pub struct MongoEngine { + client: mongodb::Client, + catalog_db: mongodb::Database, + data_db: mongodb::Database, + region: String, + max_connections: u32, + /// Cache of `table_id` -> `has_gsi`. Avoids catalog queries on every write + /// for tables with no GSIs. + gsi_cache: dashmap::DashMap, +} + +impl MongoEngine { + pub async fn new( + connection_string: &str, + region: &str, + max_connections: u32, + ) -> Result { + let mut options = mongodb::options::ClientOptions::parse(connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + options.max_pool_size = Some(max_connections); + + let client = mongodb::Client::with_options(options) + .map_err(|e| StorageError::Connection(e.to_string()))?; + + let catalog_db = client.database("extenddb_catalog"); + let data_db = client.database("extenddb_data"); + + Ok(Self { + client, + catalog_db, + data_db, + region: region.to_owned(), + max_connections, + gsi_cache: dashmap::DashMap::new(), + }) + } + + /// Validate `account_id` against injection attacks. + fn validate_account_id(account_id: &str) -> Result<(), StorageError> { + if account_id.contains('$') + || account_id.contains('.') + || account_id.contains('\0') + || !account_id.is_ascii() + { + return Err(StorageError::Validation(format!( + "Invalid account_id: {account_id}" + ))); + } + Ok(()) + } +} diff --git a/crates/storage-mongodb/src/management_store.rs b/crates/storage-mongodb/src/management_store.rs new file mode 100644 index 00000000..1e3052bc --- /dev/null +++ b/crates/storage-mongodb/src/management_store.rs @@ -0,0 +1,2400 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `ManagementStore` trait implementation for `MongoDB`. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{self, Binary, DateTime as BsonDateTime, Document, doc}; +use mongodb::options::{FindOptions, UpdateOptions}; + +use extenddb_storage::management_store::{ + AccessKeyCreated, AccountDetail, AdminEntry, GroupDetail, GroupListEntry, ManagementStore, + MetricsRow, OpError, OpResult, RoleDetail, RoleListEntry, UserDetail, UserListEntry, +}; + +use crate::catalog_store::MongoCatalogStore; + +fn is_duplicate_key(e: &mongodb::error::Error) -> bool { + matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ) +} + +fn to_offset_dt(dt: BsonDateTime) -> time::OffsetDateTime { + time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(dt.timestamp_millis()) * 1_000_000) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH) +} + +fn now_bson() -> BsonDateTime { + BsonDateTime::now() +} + +// ── ManagementStore ───────────────────────────────────────────────────── + +impl ManagementStore for MongoCatalogStore { + fn create_account(&self, account_id: &str, account_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let account_name = account_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("accounts"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "account_name": &account_name, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => { + Err(OpError::AlreadyExists("Account already exists".to_owned())) + } + Err(e) => { + tracing::error!("create_account failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn delete_account(&self, account_id: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let tables_coll = self.catalog_db().collection::("tables"); + let has_tables = tables_coll + .count_documents(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("delete_account check tables: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if has_tables > 0 { + return Err(OpError::HasDependents( + "Cannot delete account with existing tables. Delete all tables first." + .to_owned(), + )); + } + + let coll = self.catalog_db().collection::("accounts"); + let result = coll + .delete_one(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("delete_account: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if result.deleted_count == 0 { + return Err(OpError::NotFound("Account not found".to_owned())); + } + Ok(()) + }) + } + + fn list_all_accounts(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("accounts"); + let opts = FindOptions::builder() + .sort(doc! { "account_id": 1 }) + .build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_all_accounts: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_all_accounts cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("account_name").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn list_all_accounts_full( + &self, + ) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("accounts"); + let opts = FindOptions::builder() + .sort(doc! { "account_id": 1 }) + .build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_all_accounts_full: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_all_accounts_full cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("account_name").ok()?.to_owned(), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn list_accounts_for( + &self, + account_id: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("accounts"); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("list_accounts_for: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_accounts_for cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("account_name").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn get_account_detail( + &self, + account_id: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("accounts"); + let acct = coll + .find_one(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("get_account_detail: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(acct_doc) = acct else { + return Ok(None); + }; + + let account_name = acct_doc + .get_str("account_name") + .unwrap_or_default() + .to_owned(); + + let users_coll = self.catalog_db().collection::("iam_users"); + let users_cursor = users_coll + .find(doc! { "account_id": &account_id }) + .with_options(FindOptions::builder().sort(doc! { "user_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_account_detail users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let users: Vec = users_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_account_detail users cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("user_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let groups_coll = self.catalog_db().collection::("iam_groups"); + let groups_cursor = groups_coll + .find(doc! { "account_id": &account_id }) + .with_options( + FindOptions::builder() + .sort(doc! { "group_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_account_detail groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let groups: Vec = groups_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_account_detail groups cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("group_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let roles_coll = self.catalog_db().collection::("iam_roles"); + let roles_cursor = roles_coll + .find(doc! { "account_id": &account_id }) + .with_options(FindOptions::builder().sort(doc! { "role_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_account_detail roles: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let roles: Vec = roles_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_account_detail roles cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("role_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + Ok(Some(AccountDetail { + account_name, + users, + groups, + roles, + })) + }) + } + + fn dashboard_counts(&self) -> BoxFuture<'_, OpResult<(i64, i64)>> { + Box::pin(async { + let accounts_coll = self.catalog_db().collection::("accounts"); + let account_count = accounts_coll.count_documents(doc! {}).await.map_err(|e| { + tracing::error!("dashboard_counts accounts: {e}"); + OpError::Internal("Database error".to_owned()) + })? as i64; + + let admins_coll = self.catalog_db().collection::("admin_users"); + let admin_count = admins_coll.count_documents(doc! {}).await.map_err(|e| { + tracing::error!("dashboard_counts admins: {e}"); + OpError::Internal("Database error".to_owned()) + })? as i64; + + Ok((account_count, admin_count)) + }) + } + + fn create_user( + &self, + account_id: &str, + user_name: &str, + password_hash: Option<&str>, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password_hash = password_hash.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let user_arn = format!("arn:aws:iam::{account_id}:user/{user_name}"); + + let mut user_doc = doc! { + "account_id": &account_id, + "user_name": &user_name, + "user_arn": &user_arn, + "created_at": now_bson(), + }; + if let Some(ref ph) = password_hash { + user_doc.insert("password_hash", ph.as_str()); + } + + let coll = self.catalog_db().collection::("iam_users"); + let result = coll.insert_one(user_doc).await; + match result { + Ok(_) => {} + Err(e) if is_duplicate_key(&e) => { + return Err(OpError::AlreadyExists("IAM user already exists".to_owned())); + } + Err(e) => { + tracing::error!("create_user failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + } + + // Seed default self-service policy. + let self_service_policy = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": format!("arn:aws:iam::{}:user/{}", account_id, user_name) + }] + }); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policy_doc = doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + "policy_name": "SelfServicePolicy", + "policy_document": bson::to_bson(&self_service_policy).unwrap_or_default(), + "created_at": now_bson(), + }; + // Use upsert to avoid errors on conflict + let filter = doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + "policy_name": "SelfServicePolicy", + }; + let opts = UpdateOptions::builder().upsert(true).build(); + if let Err(e) = policies_coll + .update_one(filter, doc! { "$setOnInsert": policy_doc }) + .with_options(opts) + .await + { + tracing::error!("seed self-service policy failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + + Ok(()) + }) + } + + fn delete_user(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let result = coll + .delete_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("delete_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("IAM user not found".to_owned())); + } + // Cascade: delete access keys, policies, group memberships + let keys_coll = self.catalog_db().collection::("access_keys"); + let _ = keys_coll + .delete_many(doc! { "account_id": &account_id, "user_name": &user_name }) + .await; + let policies_coll = self.catalog_db().collection::("iam_policies"); + let _ = policies_coll + .delete_many(doc! { "account_id": &account_id, "principal_type": "user", "principal_name": &user_name }) + .await; + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let _ = members_coll + .delete_many(doc! { "account_id": &account_id, "user_name": &user_name }) + .await; + Ok(()) + }) + } + + fn list_users(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let opts = FindOptions::builder().sort(doc! { "user_name": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_users cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("user_name").ok()?.to_owned(), + d.get_str("user_arn").ok()?.to_owned(), + d.get_str("password_hash").is_ok(), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn get_user_detail( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let exists = coll + .find_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("get_user_detail exists: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if exists.is_none() { + return Ok(None); + } + + let keys_coll = self.catalog_db().collection::("access_keys"); + let keys_cursor = keys_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options( + FindOptions::builder() + .sort(doc! { "access_key_id": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_user_detail keys: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let keys: Vec<(String, bool)> = keys_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail keys cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("access_key_id").ok()?.to_owned(), + d.get_bool("is_active").unwrap_or(true), + )) + }) + .collect(); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policies_cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .with_options( + FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_user_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let policies: Vec = policies_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("policy_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let tags_coll = self.catalog_db().collection::("iam_user_tags"); + let tags_cursor = tags_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options(FindOptions::builder().sort(doc! { "tag_key": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_user_detail tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let tags: Vec<(String, String)> = tags_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect(); + + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let groups_cursor = members_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options( + FindOptions::builder() + .sort(doc! { "group_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_user_detail groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let groups: Vec = groups_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail groups cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("group_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + Ok(Some(UserDetail { + keys, + policies, + tags, + groups, + })) + }) + } + + fn verify_iam_user_password( + &self, + account_id: &str, + user_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password = password.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let doc = coll + .find_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("verify_iam_user_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(user_doc) = doc else { + return Ok(false); + }; + + let Some(hash) = user_doc.get_str("password_hash").ok() else { + return Ok(false); + }; + + let hash = hash.to_owned(); + Ok(tokio::task::spawn_blocking(move || { + bcrypt::verify(password, &hash).unwrap_or(false) + }) + .await + .unwrap_or(false)) + }) + } + + fn change_user_password( + &self, + account_id: &str, + user_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let result = coll + .update_one( + doc! { "account_id": &account_id, "user_name": &user_name }, + doc! { "$set": { "password_hash": &password_hash } }, + ) + .await + .map_err(|e| { + tracing::error!("change_user_password failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.matched_count == 0 { + return Err(OpError::NotFound("IAM user not found".to_owned())); + } + Ok(()) + }) + } + + fn tag_user( + &self, + account_id: &str, + user_name: &str, + tags: &[(String, String)], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let tags = tags.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + for (key, value) in &tags { + let filter = doc! { + "account_id": &account_id, + "user_name": &user_name, + "tag_key": key, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "user_name": &user_name, + "tag_key": key, + "tag_value": value, + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("tag_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn untag_user( + &self, + account_id: &str, + user_name: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + for key in &tag_keys { + coll.delete_one(doc! { + "account_id": &account_id, + "user_name": &user_name, + "tag_key": key, + }) + .await + .map_err(|e| { + tracing::error!("untag_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn list_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + let opts = FindOptions::builder().sort(doc! { "tag_key": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_user_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_user_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn create_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { + let group_arn = format!("arn:aws:iam::{account_id}:group/{group_name}"); + let coll = self.catalog_db().collection::("iam_groups"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "group_name": &group_name, + "group_arn": &group_arn, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "IAM group already exists".to_owned(), + )), + Err(e) => { + tracing::error!("create_group failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn delete_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_groups"); + let result = coll + .delete_one(doc! { "account_id": &account_id, "group_name": &group_name }) + .await + .map_err(|e| { + tracing::error!("delete_group failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("IAM group not found".to_owned())); + } + Ok(()) + }) + } + + fn list_groups(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_groups"); + let opts = FindOptions::builder() + .sort(doc! { "group_name": 1 }) + .build(); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_groups cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("group_name").ok()?.to_owned(), + d.get_str("group_arn").ok()?.to_owned(), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn get_group_detail( + &self, + account_id: &str, + group_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_groups"); + let exists = coll + .find_one(doc! { "account_id": &account_id, "group_name": &group_name }) + .await + .map_err(|e| { + tracing::error!("get_group_detail exists: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if exists.is_none() { + return Ok(None); + } + + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let members_cursor = members_coll + .find(doc! { "account_id": &account_id, "group_name": &group_name }) + .with_options(FindOptions::builder().sort(doc! { "user_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_group_detail members: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let members: Vec = members_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_group_detail members cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("user_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policies_cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "group", + "principal_name": &group_name, + }) + .with_options( + FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_group_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let policies: Vec = policies_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_group_detail policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("policy_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let users_coll = self.catalog_db().collection::("iam_users"); + let all_users_cursor = users_coll + .find(doc! { "account_id": &account_id }) + .with_options(FindOptions::builder().sort(doc! { "user_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_group_detail all_users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let all_users: Vec = all_users_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_group_detail all_users cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("user_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + Ok(Some(GroupDetail { + members, + policies, + all_users, + })) + }) + } + + fn add_group_member( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_group_members"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "group_name": &group_name, + "user_name": &user_name, + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "User is already a member of this group".to_owned(), + )), + Err(e) => { + tracing::error!("add_group_member failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn remove_group_member( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_group_members"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "group_name": &group_name, + "user_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("remove_group_member failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Membership not found".to_owned())); + } + Ok(()) + }) + } + + fn create_role( + &self, + account_id: &str, + role_name: &str, + trust_policy: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let trust_policy = trust_policy.clone(); + Box::pin(async move { + let role_arn = format!("arn:aws:iam::{account_id}:role/{role_name}"); + let coll = self.catalog_db().collection::("iam_roles"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "role_name": &role_name, + "role_arn": &role_arn, + "trust_policy": bson::to_bson(&trust_policy).unwrap_or_default(), + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => { + Err(OpError::AlreadyExists("IAM role already exists".to_owned())) + } + Err(e) => { + tracing::error!("create_role failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn delete_role(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let result = coll + .delete_one(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("delete_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("IAM role not found".to_owned())); + } + Ok(()) + }) + } + + fn list_roles(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let opts = FindOptions::builder().sort(doc! { "role_name": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_roles: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_roles cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let tp_bson = d.get("trust_policy")?; + let trust_policy: serde_json::Value = bson::from_bson(tp_bson.clone()).ok()?; + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("role_name").ok()?.to_owned(), + d.get_str("role_arn").ok()?.to_owned(), + trust_policy, + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn get_role_detail( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let role_doc = coll + .find_one(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("get_role_detail role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(role_doc) = role_doc else { + return Ok(None); + }; + + let trust_policy: serde_json::Value = role_doc + .get("trust_policy") + .and_then(|b| bson::from_bson(b.clone()).ok()) + .unwrap_or(serde_json::Value::Null); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policies_cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .with_options( + FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_role_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let policies: Vec = policies_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_role_detail policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("policy_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let tags_coll = self.catalog_db().collection::("iam_role_tags"); + let tags_cursor = tags_coll + .find(doc! { "account_id": &account_id, "role_name": &role_name }) + .with_options(FindOptions::builder().sort(doc! { "tag_key": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_role_detail tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let tags: Vec<(String, String)> = tags_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_role_detail tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect(); + + Ok(Some(RoleDetail { + trust_policy, + policies, + tags, + })) + }) + } + + fn get_role_trust_policy( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let doc = coll + .find_one(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("get_role_trust_policy: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + d.get("trust_policy") + .and_then(|b| bson::from_bson(b.clone()).ok()) + })) + }) + } + + fn tag_role( + &self, + account_id: &str, + role_name: &str, + tags: &[(String, String)], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let tags = tags.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + for (key, value) in &tags { + let filter = doc! { + "account_id": &account_id, + "role_name": &role_name, + "tag_key": key, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "role_name": &role_name, + "tag_key": key, + "tag_value": value, + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("tag_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn untag_role( + &self, + account_id: &str, + role_name: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + for key in &tag_keys { + coll.delete_one(doc! { + "account_id": &account_id, + "role_name": &role_name, + "tag_key": key, + }) + .await + .map_err(|e| { + tracing::error!("untag_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn list_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + let opts = FindOptions::builder().sort(doc! { "tag_key": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": &role_name }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_role_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_role_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn put_policy( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + let policy_name = policy_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let filter = doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + "policy_name": &policy_name, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + "policy_name": &policy_name, + "policy_document": bson::to_bson(&document).unwrap_or_default(), + }, + "$setOnInsert": { + "created_at": now_bson(), + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("put_policy failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn delete_policy( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + let policy_name = policy_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + "policy_name": &policy_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_policy failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Policy not found".to_owned())); + } + Ok(()) + }) + } + + fn list_policies( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let opts = FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let policy_name = d.get_str("policy_name").ok()?.to_owned(); + let policy_document: serde_json::Value = d + .get("policy_document") + .and_then(|b| bson::from_bson(b.clone()).ok())?; + let created_at = to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()); + Some((policy_name, policy_document, created_at)) + }) + .collect()) + }) + } + + fn set_user_boundary( + &self, + account_id: &str, + user_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let filter = doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + "policy_document": bson::to_bson(&document).unwrap_or_default(), + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("set_user_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn get_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("get_user_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + d.get("policy_document") + .and_then(|b| bson::from_bson(b.clone()).ok()) + })) + }) + } + + fn delete_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_user_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Permissions boundary not set".to_owned())); + } + Ok(()) + }) + } + + fn set_role_boundary( + &self, + account_id: &str, + role_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let filter = doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + "policy_document": bson::to_bson(&document).unwrap_or_default(), + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("set_role_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn get_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("get_role_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + d.get("policy_document") + .and_then(|b| bson::from_bson(b.clone()).ok()) + })) + }) + } + + fn delete_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_role_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Permissions boundary not set".to_owned())); + } + Ok(()) + }) + } + + fn create_access_key( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + // Check user exists (MongoDB has no FK constraints) + let users_coll = self.catalog_db().collection::("iam_users"); + let user_exists = users_coll + .find_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("create_access_key user check: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if user_exists.is_none() { + return Err(OpError::NotFound("User not found".to_owned())); + } + + let enc_key = self.get_encryption_key().await?; + + let access_key_id = generate_access_key_id(); + let secret_key = generate_secret_key(); + let encrypted = encrypt_secret(&secret_key, &enc_key, &access_key_id).map_err(|e| { + tracing::error!("create_access_key encryption: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let coll = self.catalog_db().collection::("access_keys"); + coll.insert_one(doc! { + "access_key_id": &access_key_id, + "account_id": &account_id, + "user_name": &user_name, + "secret_key_encrypted": Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: encrypted }, + "is_active": true, + "created_at": now_bson(), + }) + .await + .map_err(|e| { + if is_duplicate_key(&e) { + OpError::NotFound("User not found".to_owned()) + } else { + tracing::error!("create_access_key failed: {e}"); + OpError::Internal("Database error".to_owned()) + } + })?; + + Ok(AccessKeyCreated { + access_key_id, + secret_access_key: secret_key, + }) + }) + } + + fn delete_access_key( + &self, + account_id: &str, + user_name: &str, + key_id: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let key_id = key_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("access_keys"); + let result = coll + .delete_one(doc! { + "access_key_id": &key_id, + "account_id": &account_id, + "user_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_access_key failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Access key not found".to_owned())); + } + Ok(()) + }) + } + + fn list_access_keys( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("access_keys"); + let opts = FindOptions::builder() + .sort(doc! { "created_at": 1 }) + .build(); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_access_keys: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_access_keys cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("access_key_id").ok()?.to_owned(), + d.get_bool("is_active").unwrap_or(true), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn import_access_key( + &self, + account_id: &str, + user_name: &str, + access_key_id: &str, + secret_access_key: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let access_key_id = access_key_id.to_owned(); + let secret_access_key = secret_access_key.to_owned(); + Box::pin(async move { + let enc_key = self.get_encryption_key().await?; + + let encrypted = + encrypt_secret(&secret_access_key, &enc_key, &access_key_id).map_err(|e| { + tracing::error!("import_access_key encryption: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let coll = self.catalog_db().collection::("access_keys"); + let result = coll + .insert_one(doc! { + "access_key_id": &access_key_id, + "account_id": &account_id, + "user_name": &user_name, + "secret_key_encrypted": Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: encrypted }, + "is_active": true, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "Access key ID already exists".to_owned(), + )), + Err(e) => { + tracing::error!("import_access_key failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn store_session( + &self, + session_token: &str, + access_key_id: &str, + secret_key_encrypted: &[u8], + account_id: &str, + role_name: &str, + session_name: &str, + session_tags: &Option, + session_policy: &Option, + expires_at: time::OffsetDateTime, + ) -> BoxFuture<'_, OpResult<()>> { + let session_token = session_token.to_owned(); + let access_key_id = access_key_id.to_owned(); + let secret_key_encrypted = secret_key_encrypted.to_vec(); + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + let session_tags = session_tags.clone(); + let session_policy = session_policy.clone(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_sessions"); + let expires_bson = BsonDateTime::from_millis(expires_at.unix_timestamp() * 1000); + + let mut session_doc = doc! { + "session_token": &session_token, + "access_key_id": &access_key_id, + "secret_key_encrypted": Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: secret_key_encrypted }, + "account_id": &account_id, + "role_name": &role_name, + "session_name": &session_name, + "expires_at": expires_bson, + }; + + if let Some(ref tags) = session_tags { + session_doc.insert("session_tags", bson::to_bson(tags).unwrap_or_default()); + } + if let Some(ref policy) = session_policy { + session_doc.insert("session_policy", bson::to_bson(policy).unwrap_or_default()); + } + + coll.insert_one(session_doc).await.map_err(|e| { + tracing::error!("store_session failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn fetch_caller_tags( + &self, + account_id: &str, + resource: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let resource = resource.to_owned(); + Box::pin(async move { + if let Some(user_name) = resource.strip_prefix("user/") { + let coll = self.catalog_db().collection::("iam_user_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": user_name }) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags user: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_caller_tags user cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + } else if let Some(role_name) = resource.strip_prefix("role/") { + let coll = self.catalog_db().collection::("iam_role_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": role_name }) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_caller_tags role cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + } else if let Some(rest) = resource.strip_prefix("assumed-role/") { + let role_name = rest.split('/').next().unwrap_or(""); + if role_name.is_empty() { + return Ok(Vec::new()); + } + let coll = self.catalog_db().collection::("iam_role_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": role_name }) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags assumed-role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_caller_tags assumed-role cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + } else { + Ok(Vec::new()) + } + }) + } +} + +// ── SettingsStore ─────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::SettingsStore for MongoCatalogStore { + fn get_setting(&self, key: &str) -> BoxFuture<'_, OpResult>> { + let key = key.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("settings"); + let doc = coll.find_one(doc! { "_id": &key }).await.map_err(|e| { + tracing::error!("get_setting: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned))) + }) + } + + fn set_setting(&self, key: &str, value: &str) -> BoxFuture<'_, OpResult<()>> { + let key = key.to_owned(); + let value = value.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("settings"); + let filter = doc! { "_id": &key }; + let update = doc! { "$set": { "value": &value } }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("set_setting failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn list_settings(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("settings"); + let opts = FindOptions::builder().sort(doc! { "_id": 1 }).build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_settings: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_settings cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("_id").ok()?.to_owned(), + d.get_str("value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn cached_encryption_key(&self) -> Option { + self.encryption_key.clone() + } +} + +// ── MetricsStore ──────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::MetricsStore for MongoCatalogStore { + fn insert_metrics(&self, rows: &[MetricsRow]) -> BoxFuture<'_, OpResult<()>> { + let rows = rows.to_vec(); + Box::pin(async move { + if rows.is_empty() { + return Ok(()); + } + let coll = self.catalog_db().collection::("metrics"); + let docs: Vec = rows + .into_iter() + .map(|r| { + let mut d = doc! { + "bucket": BsonDateTime::from_millis(r.bucket.unix_timestamp() * 1000), + "metric": &r.metric, + "sum": r.sum, + "count": r.count, + "min": r.min, + "max": r.max, + }; + if let Some(ref tn) = r.table_name { + d.insert("table_name", tn.as_str()); + } + if let Some(ref idx) = r.index_name { + d.insert("index_name", idx.as_str()); + } + if let Some(ref op) = r.operation { + d.insert("operation", op.as_str()); + } + d + }) + .collect(); + coll.insert_many(docs).await.map_err(|e| { + tracing::error!("insert_metrics failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn query_metrics( + &self, + start: time::OffsetDateTime, + end: time::OffsetDateTime, + table_name: Option<&str>, + metric: Option<&str>, + ) -> BoxFuture<'_, OpResult>> { + let table_name = table_name.map(std::borrow::ToOwned::to_owned); + let metric = metric.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let coll = self.catalog_db().collection::("metrics"); + let start_bson = BsonDateTime::from_millis(start.unix_timestamp() * 1000); + let end_bson = BsonDateTime::from_millis(end.unix_timestamp() * 1000); + + let mut filter = doc! { + "bucket": { "$gte": start_bson, "$lte": end_bson } + }; + if let Some(ref tn) = table_name { + filter.insert("table_name", tn.as_str()); + } + if let Some(ref m) = metric { + filter.insert("metric", m.as_str()); + } + + let cursor = coll.find(filter).await.map_err(|e| { + tracing::error!("query_metrics: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("query_metrics cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(MetricsRow { + bucket: to_offset_dt(d.get_datetime("bucket").ok()?.to_owned()), + metric: d.get_str("metric").ok()?.to_owned(), + table_name: d + .get_str("table_name") + .ok() + .map(std::borrow::ToOwned::to_owned), + index_name: d + .get_str("index_name") + .ok() + .map(std::borrow::ToOwned::to_owned), + operation: d + .get_str("operation") + .ok() + .map(std::borrow::ToOwned::to_owned), + sum: d.get_f64("sum").ok()?, + count: d + .get_i64("count") + .ok() + .or_else(|| d.get_i32("count").ok().map(i64::from))?, + min: d.get_f64("min").ok()?, + max: d.get_f64("max").ok()?, + }) + }) + .collect()) + }) + } + + fn prune_metrics(&self, retention: std::time::Duration) -> BoxFuture<'_, OpResult<()>> { + Box::pin(async move { + let coll = self.catalog_db().collection::("metrics"); + let cutoff = time::OffsetDateTime::now_utc() - retention; + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + coll.delete_many(doc! { "bucket": { "$lt": cutoff_bson } }) + .await + .map_err(|e| { + tracing::error!("prune_metrics failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } +} + +// ── RateLimitStore ────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::RateLimitStore for MongoCatalogStore { + fn count_principal_failures( + &self, + principal: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let principal = principal.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(window_seconds as u64); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + let count = coll + .count_documents(doc! { + "principal": &principal, + "attempted_at": { "$gte": cutoff_bson }, + }) + .await + .map_err(|e| { + tracing::error!("count_principal_failures: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(count as i64) + }) + } + + fn count_ip_failures( + &self, + source_ip: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let source_ip = source_ip.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(window_seconds as u64); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + let count = coll + .count_documents(doc! { + "source_ip": &source_ip, + "attempted_at": { "$gte": cutoff_bson }, + }) + .await + .map_err(|e| { + tracing::error!("count_ip_failures: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(count as i64) + }) + } + + fn record_failed_login(&self, principal: &str, source_ip: Option<&str>) -> BoxFuture<'_, ()> { + let principal = principal.to_owned(); + let source_ip = source_ip.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let mut login_doc = doc! { + "principal": &principal, + "attempted_at": now_bson(), + }; + if let Some(ref ip) = source_ip { + login_doc.insert("source_ip", ip.as_str()); + } + if let Err(e) = coll.insert_one(login_doc).await { + tracing::error!("record_failed_login: {e}"); + } + }) + } + + fn cleanup_old_attempts(&self, max_age_seconds: i64) -> BoxFuture<'_, ()> { + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(max_age_seconds as u64); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + if let Err(e) = coll + .delete_many(doc! { "attempted_at": { "$lt": cutoff_bson } }) + .await + { + tracing::error!("cleanup_old_attempts: {e}"); + } + }) + } +} + +// ── AdminStore ────────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::AdminStore for MongoCatalogStore { + fn create_admin(&self, admin_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let result = coll + .insert_one(doc! { + "_id": &admin_name, + "password_hash": &password_hash, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "Admin user already exists".to_owned(), + )), + Err(e) => { + tracing::error!("create_admin failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn list_admins(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("admin_users"); + let opts = FindOptions::builder().sort(doc! { "_id": 1 }).build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_admins: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_admins cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(AdminEntry { + admin_name: d.get_str("_id").ok()?.to_owned(), + created_at: to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + }) + }) + .collect()) + }) + } + + fn delete_admin(&self, admin_name: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let result = coll + .delete_one(doc! { "_id": &admin_name }) + .await + .map_err(|e| { + tracing::error!("delete_admin failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn change_admin_password( + &self, + admin_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let result = coll + .update_one( + doc! { "_id": &admin_name }, + doc! { "$set": { "password_hash": &password_hash } }, + ) + .await + .map_err(|e| { + tracing::error!("change_admin_password failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.matched_count == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn verify_admin_password( + &self, + admin_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult>> { + let admin_name = admin_name.to_owned(); + let password = password.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let doc = coll + .find_one(doc! { "_id": &admin_name }) + .await + .map_err(|e| { + tracing::error!("verify_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let Some(admin_doc) = doc else { + return Ok(None); + }; + let Some(hash) = admin_doc.get_str("password_hash").ok() else { + return Ok(None); + }; + let hash = hash.to_owned(); + Ok(Some( + tokio::task::spawn_blocking(move || { + bcrypt::verify(password, &hash).unwrap_or(false) + }) + .await + .unwrap_or(false), + )) + }) + } +} + +// ── Helper: encryption key retrieval ──────────────────────────────────── + +impl MongoCatalogStore { + async fn get_encryption_key(&self) -> OpResult { + if let Some(ref cached) = self.encryption_key { + return Ok(cached.clone()); + } + let coll = self.catalog_db().collection::("settings"); + let doc = coll + .find_one(doc! { "_id": "encryption_key" }) + .await + .map_err(|e| { + tracing::error!("get_encryption_key: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + doc.and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) + .ok_or_else(|| OpError::Internal("Encryption key not configured".to_owned())) + } +} + +// ── Crypto helpers ────────────────────────────────────────────────────── + +fn generate_access_key_id() -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + let mut rng = rand::rng(); + let suffix: String = (0..8) + .map(|_| CHARSET[rand::Rng::random_range(&mut rng, 0..CHARSET.len())] as char) + .collect(); + format!("AKIAEXTENDDB{suffix}") +} + +fn generate_secret_key() -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut rng = rand::rng(); + let suffix: String = (0..32) + .map(|_| CHARSET[rand::Rng::random_range(&mut rng, 0..CHARSET.len())] as char) + .collect(); + format!("extenddb{suffix}") +} + +fn encrypt_secret(plaintext: &str, key_b64: &str, aad: &str) -> Result, String> { + use aes_gcm::Aes256Gcm; + use aes_gcm::KeyInit; + use aes_gcm::aead::Aead; + use aes_gcm::aead::Payload; + use base64::Engine; + + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + + let key = aes_gcm::Key::::from_slice(&key_bytes); + let cipher = Aes256Gcm::new(key); + + let nonce_bytes: [u8; 12] = rand::random(); + let nonce = aes_gcm::Nonce::from_slice(&nonce_bytes); + + let payload = Payload { + msg: plaintext.as_bytes(), + aad: aad.as_bytes(), + }; + let ciphertext = cipher + .encrypt(nonce, payload) + .map_err(|e| format!("encrypt: {e}"))?; + + let mut result = Vec::with_capacity(12 + ciphertext.len()); + result.extend_from_slice(&nonce_bytes); + result.extend_from_slice(&ciphertext); + Ok(result) +} diff --git a/crates/storage-mongodb/src/metadata_engine.rs b/crates/storage-mongodb/src/metadata_engine.rs new file mode 100644 index 00000000..72f021ee --- /dev/null +++ b/crates/storage-mongodb/src/metadata_engine.rs @@ -0,0 +1,546 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MetadataEngine` implementation for `MongoDB`. +//! +//! Handles TTL configuration, resource tags, and table size bookkeeping. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::IndexModel; +use mongodb::bson::{Document, doc}; +use mongodb::options::IndexOptions; + +use extenddb_core::types::{Item, Tag, TimeToLiveDescription, TimeToLiveStatus}; +use extenddb_storage::MetadataEngine; +use extenddb_storage::TtlTableInfo; +use extenddb_storage::error::StorageError; + +use crate::MongoEngine; +use crate::data::{data_collection_name, document_to_item}; + +fn extract_id_fields(doc: &Document) -> (String, String) { + let id = doc.get_document("_id").ok(); + let account_id = id + .and_then(|d| d.get_str("account_id").ok()) + .unwrap_or_default() + .to_owned(); + let table_name = id + .and_then(|d| d.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + (account_id, table_name) +} + +impl MetadataEngine for MongoEngine { + fn describe_ttl( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let table_doc = coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + match table_doc.get_str("ttl_attribute") { + Ok(attr) => Ok(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some(attr.to_owned()), + }), + Err(_) => Ok(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Disabled, + attribute_name: None, + }), + } + }) + } + + fn update_ttl( + &self, + account_id: &str, + table_name: &str, + attribute_name: &str, + enabled: bool, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let attribute_name = attribute_name.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + + let ttl_val = if enabled { + mongodb::bson::Bson::String(attribute_name) + } else { + mongodb::bson::Bson::Null + }; + + let result = coll + .update_one( + doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "table_status": "ACTIVE", + }, + doc! { + "$set": { + "ttl_attribute": ttl_val, + "ttl_index_ready": false, + } + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.matched_count == 0 { + let exists = coll + .find_one( + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + return match exists { + None => Err(StorageError::TableNotFound(table_name)), + Some(_) => Err(StorageError::TableNotActive(table_name)), + }; + } + + Ok(()) + }) + } + + fn tag_resource(&self, arn: &str, tags: &[Tag]) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_string(); + let tags = tags.to_vec(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tags"); + + for tag in &tags { + coll.update_one( + doc! { "resource_arn": &arn, "tag_key": &tag.key }, + doc! { "$set": { "resource_arn": &arn, "tag_key": &tag.key, "tag_value": &tag.value } }, + ) + .upsert(true) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + }) + } + + fn untag_resource( + &self, + arn: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_string(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tags"); + + for key in &tag_keys { + coll.delete_one(doc! { "resource_arn": &arn, "tag_key": key }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + }) + } + + fn list_tags(&self, arn: &str) -> BoxFuture<'_, Result, StorageError>> { + let arn = arn.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tags"); + let mut cursor = coll + .find(doc! { "resource_arn": &arn }) + .sort(doc! { "tag_key": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut tags = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let key = doc.get_str("tag_key").unwrap_or_default().to_owned(); + let value = doc.get_str("tag_value").unwrap_or_default().to_owned(); + tags.push(Tag { key, value }); + } + Ok(tags) + }) + } + + fn tables_with_ttl( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { + "_id.account_id": &account_id, + "ttl_attribute": { "$ne": mongodb::bson::Bson::Null }, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let name = doc + .get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + let attr = doc.get_str("ttl_attribute").unwrap_or_default().to_owned(); + results.push((name, attr)); + } + Ok(results) + }) + } + + fn all_tables_with_ttl(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { + "ttl_attribute": { "$ne": mongodb::bson::Bson::Null }, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let (account_id, name) = extract_id_fields(&doc); + let attr = doc.get_str("ttl_attribute").unwrap_or_default().to_owned(); + results.push((account_id, name, attr)); + } + Ok(results) + }) + } + + fn all_tables_with_ttl_index_ready( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { + "ttl_attribute": { "$ne": mongodb::bson::Bson::Null }, + "ttl_index_ready": true, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let (account_id, name) = extract_id_fields(&doc); + let attr = doc.get_str("ttl_attribute").unwrap_or_default().to_owned(); + results.push((account_id, name, attr)); + } + Ok(results) + }) + } + + fn create_ttl_index( + &self, + account_id: &str, + table_name: &str, + ttl_attribute: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let ttl_attribute = ttl_attribute.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let id_filter = + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }; + let table_doc = tables_coll + .find_one(id_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let index_name = format!("idx_ttl_{ttl_attribute}"); + let index_key = format!("item_data.{ttl_attribute}.N"); + + let index = IndexModel::builder() + .keys(doc! { &index_key: 1 }) + .options( + IndexOptions::builder() + .name(index_name) + .sparse(true) + .build(), + ) + .build(); + + data_coll + .create_index(index) + .await + .map_err(|e| StorageError::Internal(format!("TTL index creation failed: {e}")))?; + + tables_coll + .update_one(id_filter, doc! { "$set": { "ttl_index_ready": true } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + }) + } + + fn drop_ttl_index( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let id_filter = + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }; + + // Mark index as not ready first + tables_coll + .update_one( + id_filter.clone(), + doc! { "$set": { "ttl_index_ready": false } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let table_doc = tables_coll + .find_one(id_filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let ttl_attribute = table_doc.get_str("ttl_attribute").unwrap_or_default(); + let index_name = format!("idx_ttl_{ttl_attribute}"); + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + data_coll + .drop_index(index_name) + .await + .map_err(|e| StorageError::Internal(format!("TTL index drop failed: {e}")))?; + + Ok(()) + }) + } + + fn find_expired_items_indexed( + &self, + account_id: &str, + table_name: &str, + ttl_attribute: &str, + limit: usize, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let ttl_attribute = ttl_attribute.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let now_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64; + + let ttl_field = format!("item_data.{ttl_attribute}.N"); + + // Find items where TTL attribute N value is between 1 and now (expired) + // DynamoDB stores numbers as strings in the N field + let filter = doc! { + &ttl_field: { + "$exists": true, + "$ne": mongodb::bson::Bson::Null, + } + }; + + let mut cursor = data_coll + .find(filter) + .sort(doc! { &ttl_field: 1 }) + .limit(limit as i64 * 2) // over-fetch since we filter in app + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut items = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + if items.len() >= limit { + break; + } + // Parse the TTL value and check if expired + if let Ok(item_data) = doc.get_document("item_data") { + if let Ok(ttl_obj) = item_data.get_document(&ttl_attribute) { + if let Ok(n_str) = ttl_obj.get_str("N") { + if let Ok(ttl_val) = n_str.parse::() { + if ttl_val >= 1 && ttl_val <= now_epoch { + let item = document_to_item(&doc)?; + items.push(item); + } + } + } + } + } + } + Ok(items) + }) + } + + fn refresh_table_size( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let id_filter = + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }; + let table_doc = tables_coll + .find_one(id_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let item_count = data_coll + .count_documents(doc! {}) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + as i64; + + // Approximate size via collStats + let stats_result = self + .data_db + .run_command(doc! { "collStats": &coll_name }) + .await; + + let table_size = match stats_result { + Ok(stats) => stats.get_i64("size").unwrap_or(0), + Err(_) => 0, + }; + + tables_coll + .update_one( + doc! { "_id": { "account_id": &account_id, "table_name": &table_name }, "table_status": "ACTIVE" }, + doc! { "$set": { "item_count": item_count, "table_size_bytes": table_size } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + }) + } + + fn list_active_table_names( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { "_id.account_id": &account_id, "table_status": "ACTIVE" }) + .sort(doc! { "_id.table_name": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut names = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let name = doc + .get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + names.push(name); + } + Ok(names) + }) + } + + fn all_active_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { "table_status": "ACTIVE" }) + .sort(doc! { "_id.account_id": 1, "_id.table_name": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let (account_id, name) = extract_id_fields(&doc); + results.push((account_id, name)); + } + Ok(results) + }) + } +} diff --git a/crates/storage-mongodb/src/operations.rs b/crates/storage-mongodb/src/operations.rs new file mode 100644 index 00000000..2389a8f1 --- /dev/null +++ b/crates/storage-mongodb/src/operations.rs @@ -0,0 +1,129 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MongoDB` implementation of `OperationsEngine`. + +use extenddb_storage::error::StorageError; +use extenddb_storage::operations::{ConnectionParts, OperationsEngine}; + +/// `MongoDB` operations engine for extenddb CLI commands. +pub struct MongoOperationsEngine; + +impl OperationsEngine for MongoOperationsEngine { + fn parse_connection_string(&self, s: &str) -> Result { + // Best-effort parse of `mongodb[+srv]://[user[:pass]@]host[:port][/db][?...]`. + // The mongo driver owns full URI validation at connect time; this is only + // for display and CLI-side identifier extraction. + let scheme_end = s + .find("://") + .ok_or_else(|| StorageError::Internal("connection string has no scheme".to_owned()))? + + 3; + let rest = &s[scheme_end..]; + + // Split at first `?` to drop query string. + let (authority_path, _) = rest.split_once('?').unwrap_or((rest, "")); + + // Split userinfo from host by the last `@` before the first `/`. + let path_start = authority_path.find('/').unwrap_or(authority_path.len()); + let authority = &authority_path[..path_start]; + let path = authority_path[path_start..].trim_start_matches('/'); + + let (user, password, hostport) = if let Some(at) = authority.rfind('@') { + let (userinfo, hp) = authority.split_at(at); + let hp = &hp[1..]; + let (u, p) = userinfo + .split_once(':') + .map_or((userinfo, ""), |(u, p)| (u, p)); + (u.to_owned(), p.to_owned(), hp) + } else { + (String::new(), String::new(), authority) + }; + + // hostport may be a comma-separated list for replica sets — take the first + // seed. Port defaults to 27017 (matches mongo driver default). + let first_host = hostport.split(',').next().unwrap_or(hostport); + let (host, port) = if let Some((h, p)) = first_host.rsplit_once(':') { + (h.to_owned(), p.parse::().unwrap_or(27017)) + } else { + (first_host.to_owned(), 27017) + }; + + Ok(ConnectionParts { + host, + port, + user, + password, + database: path.to_owned(), + }) + } + + fn redact_connection_string(&self, s: &str) -> String { + // Redact password from mongodb[+srv]://user:password@host[:port]/... + let Some(scheme_end) = s.find("://") else { + return s.to_owned(); + }; + let after_scheme = scheme_end + 3; + // Only consider `@` before the first `?`. + let query_start = s[after_scheme..] + .find('?') + .map_or(s.len(), |q| after_scheme + q); + let Some(at) = s[after_scheme..query_start].rfind('@') else { + return s.to_owned(); + }; + let at_idx = after_scheme + at; + let userinfo = &s[after_scheme..at_idx]; + let Some(colon) = userinfo.find(':') else { + return s.to_owned(); + }; + let user = &userinfo[..colon]; + format!("{}{user}:***{}", &s[..after_scheme], &s[at_idx..]) + } + + fn validate_identifier(&self, name: &str, label: &str) -> Result<(), StorageError> { + // MongoDB collection/database identifier constraints: + // - no `$` prefix reserved for operators + // - no `.` (used as path separator inside documents) + // - no `\0` (null byte) + // - no non-ASCII + if name.contains('$') { + return Err(StorageError::Internal(format!( + "{label} must not contain '$'" + ))); + } + if name.contains('.') { + return Err(StorageError::Internal(format!( + "{label} must not contain '.'" + ))); + } + if name.contains('\0') { + return Err(StorageError::Internal(format!( + "{label} must not contain null bytes" + ))); + } + if !name.is_ascii() { + return Err(StorageError::Internal(format!( + "{label} must contain only ASCII characters" + ))); + } + Ok(()) + } + + fn catalog_version(&self) -> String { + // Mongo catalog version — matches the constant enforced by + // MongoBootstrapper::expected_catalog_version. + "0.0.2".to_owned() + } + + fn is_sensitive_key(&self, key: &str) -> bool { + let lower = key.to_lowercase(); + [ + "connection_string", + "password", + "secret", + "token", + "encryption_key", + ] + .iter() + .any(|pattern| lower.contains(pattern)) + } +} diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs new file mode 100644 index 00000000..98044bae --- /dev/null +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -0,0 +1,526 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `StreamEngine` trait implementation for `MongoEngine`. +//! +//! `DynamoDB` Streams are implemented using `MongoDB`'s own stream record storage. +//! Stream records are written to a `stream_records` collection in the data database, +//! grouped by shard. Shards are stored in `stream_shards` in the data database. +//! This approach uses the same storage model as the `PostgreSQL` backend rather than +//! `MongoDB` Change Streams, to maintain behavioral parity (explicit sequence numbers, +//! shard assignment, retention cleanup). + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::DateTime as BsonDateTime; +use mongodb::bson::{self, Document, doc}; +use mongodb::options::FindOptions; + +use extenddb_core::types::{ + DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, + StreamSummary, StreamViewType, +}; +use extenddb_storage::StreamEngine; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{parse_stream_arn, stream_arn}; +use extenddb_storage::{StreamListResult, StreamRecordsResult}; + +use crate::MongoEngine; + +const SHARDS_PER_STREAM: u32 = 4; + +impl MongoEngine { + /// Initialize stream shards for a table. Only creates shard documents; + /// the caller is responsible for setting `stream_label` on the table doc. + pub(crate) async fn init_stream_shards( + &self, + table_name: &str, + table_id: &str, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + for i in 0..SHARDS_PER_STREAM { + let shard_id = format!("shardId-{table_name}-{i:012}"); + let start_seq = format!("{:021}", 0); + shards_coll + .insert_one(doc! { + "shard_id": &shard_id, + "table_id": table_id, + "starting_sequence_number": &start_seq, + "created_at": BsonDateTime::now(), + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } +} + +impl StreamEngine for MongoEngine { + fn write_stream_record( + &self, + account_id: &str, + record: &StreamRecord, + shard_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_owned(); + let record = record.clone(); + let shard_id = shard_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let record_json = + serde_json::to_value(&record).map_err(|e| StorageError::Internal(e.to_string()))?; + let record_bson = + bson::to_bson(&record_json).map_err(|e| StorageError::Internal(e.to_string()))?; + + // Look up table_id + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Internal(format!("Table {table_name} not found in catalog")) + })?; + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let records_coll = self.data_db.collection::("stream_records"); + records_coll + .insert_one(doc! { + "sequence_number": &record.dynamodb.sequence_number, + "shard_id": &shard_id, + "table_id": table_id, + "event_name": format!("{:?}", record.event_name), + "record_data": record_bson, + "created_at": BsonDateTime::now(), + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + }) + } + + fn get_stream_records( + &self, + shard_id: &str, + after_sequence: Option<&str>, + limit: i64, + ) -> BoxFuture<'_, StreamRecordsResult> { + let shard_id = shard_id.to_owned(); + let after_sequence = after_sequence.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let records_coll = self.data_db.collection::("stream_records"); + + let filter = if let Some(ref after) = after_sequence { + doc! { + "shard_id": &shard_id, + "sequence_number": { "$gt": after }, + } + } else { + doc! { "shard_id": &shard_id } + }; + + let opts = FindOptions::builder() + .sort(doc! { "sequence_number": 1 }) + .limit(limit) + .build(); + + let cursor = records_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let records: Vec = docs + .into_iter() + .map(|d| { + let record_bson = d + .get("record_data") + .ok_or_else(|| StorageError::Internal("Missing record_data".to_owned()))?; + let json_val: serde_json::Value = bson::from_bson(record_bson.clone()) + .map_err(|e| StorageError::Internal(e.to_string()))?; + serde_json::from_value(json_val) + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect::, _>>()?; + + let last_seq = records.last().map(|r| r.dynamodb.sequence_number.clone()); + Ok((records, last_seq)) + }) + } + + fn describe_stream( + &self, + account_id: &str, + input: &DescribeStreamInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let stream_arn_val = input.stream_arn.clone(); + let limit = input.limit; + let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); + Box::pin(async move { + let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "stream_label": &stream_label, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn_val} not found." + )) + })?; + + let key_schema = table_doc + .get("key_schema") + .and_then(|b| bson::from_bson(b.clone()).ok()) + .ok_or_else(|| StorageError::Internal("Missing key_schema".to_owned()))?; + + let stream_view_type = table_doc + .get("stream_specification") + .and_then(|b| { + let json: serde_json::Value = bson::from_bson(b.clone()).ok()?; + json.get("StreamViewType") + .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) + }) + .unwrap_or(StreamViewType::KeysOnly); + + let table_status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let limit = limit.unwrap_or(100); + let shards_coll = self.data_db.collection::("stream_shards"); + + let filter = if let Some(ref start) = exclusive_start_shard_id { + doc! { + "table_id": table_id, + "shard_id": { "$gt": start }, + } + } else { + doc! { "table_id": table_id } + }; + + let opts = FindOptions::builder() + .sort(doc! { "shard_id": 1 }) + .limit(limit + 1) + .build(); + + let cursor = shards_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let shard_docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + #[allow(clippy::cast_sign_loss)] + let limit_usize = limit as usize; + let last_shard = if shard_docs.len() > limit_usize { + shard_docs.get(limit_usize - 1).and_then(|d| { + d.get_str("shard_id") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + } else { + None + }; + + let shards: Vec = shard_docs + .into_iter() + .take(limit_usize) + .filter_map(|d| { + Some(Shard { + shard_id: d.get_str("shard_id").ok()?.to_owned(), + parent_shard_id: d + .get_str("parent_shard_id") + .ok() + .map(std::borrow::ToOwned::to_owned), + sequence_number_range: SequenceNumberRange { + starting_sequence_number: d + .get_str("starting_sequence_number") + .ok()? + .to_owned(), + ending_sequence_number: d + .get_str("ending_sequence_number") + .ok() + .map(std::borrow::ToOwned::to_owned), + }, + }) + }) + .collect(); + + let stream_status = if table_status == "DELETING" { + StreamStatus::Disabling + } else { + StreamStatus::Enabled + }; + + Ok(StreamDescription { + stream_arn: stream_arn_val, + stream_label, + stream_status, + stream_view_type, + table_name, + key_schema, + shards, + last_evaluated_shard_id: last_shard, + }) + }) + } + + fn list_streams( + &self, + account_id: &str, + table_name: Option<&str>, + limit: i64, + exclusive_start_stream_arn: Option<&str>, + ) -> BoxFuture<'_, StreamListResult> { + let account_id = account_id.to_owned(); + let table_name = table_name.map(std::borrow::ToOwned::to_owned); + let exclusive_start_stream_arn = + exclusive_start_stream_arn.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + + let mut filter = doc! { + "_id.account_id": &account_id, + "stream_label": { "$ne": null }, + }; + + if let Some(ref tn) = table_name { + filter.insert("_id.table_name", tn.as_str()); + } + + if let Some(ref start_arn) = exclusive_start_stream_arn { + let (start_table, start_label) = parse_stream_arn(start_arn)?; + if table_name.is_some() { + filter.insert("stream_label", doc! { "$gt": &start_label }); + } else { + filter.insert( + "$or", + bson::bson!([ + { "_id.table_name": { "$gt": &start_table } }, + { "_id.table_name": &start_table, "stream_label": { "$gt": &start_label } } + ]), + ); + } + } + + let opts = FindOptions::builder() + .sort(doc! { "_id.table_name": 1, "stream_label": 1 }) + .limit(limit + 1) + .build(); + + let cursor = tables_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + #[allow(clippy::cast_sign_loss)] + let limit_usize = limit as usize; + + let summaries: Vec = docs + .iter() + .take(limit_usize) + .filter_map(|d| { + let id = d.get_document("_id").ok()?; + let tn = id.get_str("table_name").ok()?; + let label = d.get_str("stream_label").ok()?; + Some(StreamSummary { + stream_arn: stream_arn(&self.region, &account_id, tn, label), + stream_label: label.to_owned(), + table_name: tn.to_owned(), + }) + }) + .collect(); + + let last_arn = if docs.len() > limit_usize { + summaries.last().map(|s| s.stream_arn.clone()) + } else { + None + }; + + Ok((summaries, last_arn)) + }) + } + + fn cleanup_expired_stream_records( + &self, + retention_hours: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let records_coll = self.data_db.collection::("stream_records"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(retention_hours as u64 * 3600); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + let result = records_coll + .delete_many(doc! { "created_at": { "$lt": cutoff_bson } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.deleted_count) + }) + } + + fn assign_shard( + &self, + account_id: &str, + table_name: &str, + partition_key: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + let partition_key = partition_key.to_owned(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::Internal(format!("Table {table_name} not found")))?; + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let shards_coll = self.data_db.collection::("stream_shards"); + let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); + let cursor = shards_coll + .find(doc! { "table_id": table_id }) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if shard_docs.is_empty() { + return Err(StorageError::Internal(format!( + "No stream shards for table {table_name}" + ))); + } + + let shard_ids: Vec<&str> = shard_docs + .iter() + .filter_map(|d| d.get_str("shard_id").ok()) + .collect(); + + let hash = crc32fast::hash(partition_key.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shard_ids.len(); + Ok(shard_ids[idx].to_owned()) + }) + } + + fn next_sequence_number(&self, _shard_id: &str) -> BoxFuture<'_, Result> { + Box::pin(async move { + // Use atomic findAndModify on a sequence counter document + let counters_coll = self.data_db.collection::("counters"); + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(mongodb::options::ReturnDocument::After) + .build(); + let doc = counters_coll + .find_one_and_update( + doc! { "_id": "stream_seq" }, + doc! { "$inc": { "value": 1_i64 } }, + ) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Internal("Failed to generate sequence number".to_owned()) + })?; + + let seq_val = doc.get_i64("value").unwrap_or(1); + Ok(format!("{seq_val:021}")) + }) + } + + fn validate_shard( + &self, + account_id: &str, + stream_arn_val: &str, + shard_id: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_owned(); + let stream_arn_val = stream_arn_val.to_owned(); + let shard_id = shard_id.to_owned(); + Box::pin(async move { + let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "stream_label": &stream_label, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let Some(table_doc) = table_doc else { + return Err(StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn_val} not found." + ))); + }; + + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let shards_coll = self.data_db.collection::("stream_shards"); + let exists = shards_coll + .find_one(doc! { "shard_id": &shard_id, "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn_val} not found." + ))); + } + Ok(()) + }) + } + + fn latest_sequence_number( + &self, + shard_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let shard_id = shard_id.to_owned(); + Box::pin(async move { + let records_coll = self.data_db.collection::("stream_records"); + let opts = FindOptions::builder() + .sort(doc! { "sequence_number": -1 }) + .limit(1) + .build(); + let cursor = records_coll + .find(doc! { "shard_id": &shard_id }) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(docs.first().and_then(|d| { + d.get_str("sequence_number") + .ok() + .map(std::borrow::ToOwned::to_owned) + })) + }) + } +} diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs new file mode 100644 index 00000000..1e48b36e --- /dev/null +++ b/crates/storage-mongodb/src/table_engine.rs @@ -0,0 +1,1038 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `TableEngine` trait implementation for `MongoEngine`. + +use bson::{Document, doc}; +use futures::future::BoxFuture; +use mongodb::IndexModel; +use mongodb::options::{Collation, CollationStrength, IndexOptions}; + +use extenddb_core::types::{ + BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, DescribeTableInput, + GsiDescription, IndexInfo, IndexType, KeyType, ListTablesInput, ListTablesOutput, + LsiDescription, ProvisionedThroughputDescription, ScalarAttributeType, TableDescription, + TableKeyInfo, TableStatus, UpdateTableInput, +}; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{index_arn, sk_info, stream_arn, table_arn}; + +use crate::MongoEngine; +use crate::data::data_collection_name; + +impl TableEngine for MongoEngine { + fn create_table( + &self, + account_id: &str, + input: CreateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.create_table_impl(&account_id, input).await }) + } + + fn delete_table( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.delete_table_impl(&account_id, input).await }) + } + + fn describe_table( + &self, + account_id: &str, + input: DescribeTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { + self.describe_table_impl(&account_id, &input.table_name) + .await + }) + } + + fn list_tables( + &self, + account_id: &str, + input: ListTablesInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.list_tables_impl(&account_id, input).await }) + } + + fn update_table( + &self, + account_id: &str, + input: UpdateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.update_table_impl(&account_id, input).await }) + } + + fn table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { self.table_key_info_impl(&account_id, &table_name).await }) + } + + fn index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let index_name = index_name.to_string(); + Box::pin(async move { + self.index_info_impl(&account_id, &table_name, &index_name) + .await + }) + } + + fn index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let table_id = table_id.to_string(); + let index_name = index_name.to_string(); + Box::pin(async move { + self.index_info_by_table_id_impl(&table_id, &index_name) + .await + }) + } +} + +impl MongoEngine { + async fn create_table_impl( + &self, + account_id: &str, + input: CreateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + let table_id = uuid::Uuid::new_v4().to_string(); + let table_arn_val = table_arn(&self.region, account_id, &input.table_name); + let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); + let deletion_protection = input.deletion_protection_enabled.unwrap_or(false); + + let now = time::OffsetDateTime::now_utc(); + let creation_epoch = now.unix_timestamp() as f64; + + // Build the table metadata document + let key_schema_bson = + bson::to_bson(&input.key_schema).map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs_bson = bson::to_bson(&input.attribute_definitions) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + let pt_bson = input + .provisioned_throughput + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_bson = input + .stream_specification + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Compute stream label early so it's stored in the table document + let stream_label_opt = if input + .stream_specification + .as_ref() + .is_some_and(|ss| ss.stream_enabled) + { + Some( + now.format(&time::format_description::well_known::Iso8601::DEFAULT) + .unwrap_or_else(|_| "unknown".to_string()), + ) + } else { + None + }; + let stream_label_bson = stream_label_opt + .as_ref() + .map_or(bson::Bson::Null, |l| bson::Bson::String(l.clone())); + + let table_doc = doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name }, + "key_schema": key_schema_bson, + "attribute_definitions": attr_defs_bson, + "billing_mode": billing_str, + "provisioned_throughput": pt_bson.unwrap_or(bson::Bson::Null), + "stream_specification": stream_bson.unwrap_or(bson::Bson::Null), + "table_status": "ACTIVE", + "creation_date_time": bson::DateTime::from_millis((creation_epoch * 1000.0) as i64), + "table_size_bytes": 0_i64, + "item_count": 0_i64, + "table_arn": &table_arn_val, + "table_id": &table_id, + "deletion_protection_enabled": deletion_protection, + "ttl_attribute": bson::Bson::Null, + "stream_label": stream_label_bson, + }; + + let tables_coll = self.catalog_db.collection::("tables"); + tables_coll.insert_one(table_doc).await.map_err(|e| { + if e.to_string().contains("E11000") { + StorageError::TableAlreadyExists(input.table_name.clone()) + } else { + StorageError::Internal(e.to_string()) + } + })?; + + // Create the data collection with appropriate indexes + let coll_name = data_collection_name(&table_id); + self.data_db + .create_collection(&coll_name) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let data_coll = self.data_db.collection::(&coll_name); + + // Create index based on sort key type + if let Some((_, sk_type)) = sk_info(&input.key_schema, &input.attribute_definitions) { + let sk_field = match sk_type { + ScalarAttributeType::S => "sk_s", + ScalarAttributeType::N => "sk_n", + ScalarAttributeType::B => "sk_b", + }; + let index_keys = doc! { "pk": 1, sk_field: 1 }; + let mut index_opts = IndexOptions::builder().unique(true).build(); + // Use simple collation for string sort keys (byte-order) + if sk_type == ScalarAttributeType::S { + index_opts.collation = + Some(Collation::builder().locale("simple".to_string()).build()); + } + let index = IndexModel::builder() + .keys(index_keys) + .options(index_opts) + .build(); + data_coll + .create_index(index) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + // PK-only index + let index = IndexModel::builder() + .keys(doc! { "pk": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(); + data_coll + .create_index(index) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Initialize stream shards if streaming is enabled + if stream_label_opt.is_some() { + self.init_stream_shards(&input.table_name, &table_id) + .await?; + } + + // Handle GSI creation + let gsi_descriptions = if let Some(ref gsis) = input.global_secondary_indexes { + let mut descs = Vec::new(); + for gsi in gsis { + let index_id = uuid::Uuid::new_v4().to_string(); + let index_arn_val = + index_arn(&self.region, account_id, &input.table_name, &gsi.index_name); + + // Store index metadata in catalog + let key_schema_bson = bson::to_bson(&gsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection_bson = bson::to_bson(&gsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_pt_bson = gsi + .provisioned_throughput + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_doc = doc! { + "_id": { "table_id": &table_id, "index_name": &gsi.index_name }, + "index_id": &index_id, + "index_type": "GSI", + "key_schema": key_schema_bson, + "projection": projection_bson, + "index_status": "ACTIVE", + "provisioned_throughput": index_pt_bson.unwrap_or(bson::Bson::Null), + }; + + self.catalog_db + .collection::("indexes") + .insert_one(index_doc) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + descs.push(GsiDescription { + index_name: gsi.index_name.clone(), + key_schema: gsi.key_schema.clone(), + projection: gsi.projection.clone(), + index_status: "ACTIVE".to_string(), + provisioned_throughput: gsi.provisioned_throughput.as_ref().map(|pt| { + ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + } + }), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn_val, + }); + } + Some(descs) + } else { + None + }; + + // Handle LSI creation + let lsi_descriptions = if let Some(ref lsis) = input.local_secondary_indexes { + let mut descs = Vec::new(); + for lsi in lsis { + let index_id = uuid::Uuid::new_v4().to_string(); + let index_arn_val = + index_arn(&self.region, account_id, &input.table_name, &lsi.index_name); + + let key_schema_bson = bson::to_bson(&lsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection_bson = bson::to_bson(&lsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_doc = doc! { + "_id": { "table_id": &table_id, "index_name": &lsi.index_name }, + "index_id": &index_id, + "index_type": "LSI", + "key_schema": key_schema_bson, + "projection": projection_bson, + "index_status": "ACTIVE", + "provisioned_throughput": bson::Bson::Null, + }; + + self.catalog_db + .collection::("indexes") + .insert_one(index_doc) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + descs.push(LsiDescription { + index_name: lsi.index_name.clone(), + key_schema: lsi.key_schema.clone(), + projection: lsi.projection.clone(), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn_val, + }); + } + Some(descs) + } else { + None + }; + + // Build stream ARN from pre-computed label + let stream_arn_opt = stream_label_opt + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &input.table_name, label)); + + let pt_desc = match &input.provisioned_throughput { + Some(pt) => ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + None => ProvisionedThroughputDescription { + read_capacity_units: 0, + write_capacity_units: 0, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + }; + + let billing_summary = if billing_mode == BillingMode::PayPerRequest { + Some(BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_epoch), + }) + } else { + None + }; + + // Store initial tags if provided + if let Some(ref tags) = input.tags { + let tags_coll = self.catalog_db.collection::("tags"); + for tag in tags { + tags_coll + .update_one( + doc! { "resource_arn": &table_arn_val, "tag_key": &tag.key }, + doc! { "$set": { "resource_arn": &table_arn_val, "tag_key": &tag.key, "tag_value": &tag.value } }, + ) + .upsert(true) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + Ok(TableDescription { + table_name: input.table_name, + key_schema: input.key_schema, + attribute_definitions: input.attribute_definitions, + table_status: TableStatus::Active, + creation_date_time: creation_epoch, + table_size_bytes: 0, + item_count: 0, + table_arn: table_arn_val, + table_id, + provisioned_throughput: pt_desc, + billing_mode_summary: billing_summary, + global_secondary_indexes: gsi_descriptions, + local_secondary_indexes: lsi_descriptions, + stream_specification: input.stream_specification, + latest_stream_arn: stream_arn_opt, + latest_stream_label: stream_label_opt, + deletion_protection_enabled: deletion_protection, + sse_description: None, + table_class_summary: None, + on_demand_throughput: None, + }) + } + + async fn delete_table_impl( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + // Fetch the table first + let desc = self + .describe_table_impl(account_id, &input.table_name) + .await?; + + // Check deletion protection + if desc.deletion_protection_enabled { + return Err(StorageError::DeletionProtected(input.table_name.clone())); + } + + // Mark as DELETING + let tables_coll = self.catalog_db.collection::("tables"); + tables_coll + .update_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + doc! { "$set": { "table_status": "DELETING" } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Drop the data collection + let coll_name = data_collection_name(&desc.table_id); + self.data_db + .collection::(&coll_name) + .drop() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Delete index entries + self.catalog_db + .collection::("indexes") + .delete_many(doc! { "_id.table_id": &desc.table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.gsi_cache.remove(&desc.table_id); + + // Delete the table metadata + tables_coll + .delete_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(TableDescription { + table_status: TableStatus::Deleting, + ..desc + }) + } + + pub(crate) async fn describe_table_impl( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + Self::validate_account_id(account_id)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.to_string()))?; + + self.doc_to_table_description(&table_doc).await + } + + async fn list_tables_impl( + &self, + account_id: &str, + input: ListTablesInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + use futures::TryStreamExt; + + let limit = i64::from(input.limit.unwrap_or(100)); + let tables_coll = self.catalog_db.collection::("tables"); + + let mut filter = doc! { "_id.account_id": account_id }; + if let Some(ref start) = input.exclusive_start_table_name { + filter.insert("_id.table_name", doc! { "$gt": start }); + } + + let opts = mongodb::options::FindOptions::builder() + .sort(doc! { "_id.table_name": 1 }) + .limit(limit + 1) + .projection(doc! { "_id.table_name": 1 }) + .build(); + + let cursor = tables_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let names: Vec = docs + .iter() + .filter_map(|d| { + d.get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .map(std::string::ToString::to_string) + }) + .collect(); + + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let limit_usize = limit as usize; + + if names.len() > limit_usize { + Ok(ListTablesOutput { + last_evaluated_table_name: Some(names[limit_usize - 1].clone()), + table_names: names[..limit_usize].to_vec(), + }) + } else { + Ok(ListTablesOutput { + table_names: names, + last_evaluated_table_name: None, + }) + } + } + + async fn update_table_impl( + &self, + account_id: &str, + input: UpdateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + let tables_coll = self.catalog_db.collection::("tables"); + + // Build update document + let mut update_doc = Document::new(); + + if let Some(billing_mode) = &input.billing_mode { + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + update_doc.insert("billing_mode", billing_str); + } + + if let Some(pt) = &input.provisioned_throughput { + let pt_bson = bson::to_bson(pt).map_err(|e| StorageError::Internal(e.to_string()))?; + update_doc.insert("provisioned_throughput", pt_bson); + } + + if let Some(dp) = input.deletion_protection_enabled { + update_doc.insert("deletion_protection_enabled", dp); + } + + if let Some(ss) = &input.stream_specification { + let ss_bson = bson::to_bson(ss).map_err(|e| StorageError::Internal(e.to_string()))?; + update_doc.insert("stream_specification", ss_bson); + if ss.stream_enabled { + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + let label = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Iso8601::DEFAULT) + .unwrap_or_else(|_| "unknown".to_string()); + update_doc.insert("stream_label", &label); + self.init_stream_shards(&input.table_name, table_id).await?; + } + } + + if !update_doc.is_empty() { + tables_coll + .update_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + doc! { "$set": &update_doc }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Handle GSI updates + if let Some(gsi_updates) = &input.global_secondary_index_updates { + for update in gsi_updates { + if let Some(create) = &update.create { + // Fetch table_id + let desc = self + .describe_table_impl(account_id, &input.table_name) + .await?; + let index_id = uuid::Uuid::new_v4().to_string(); + + let key_schema_bson = bson::to_bson(&create.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection_bson = bson::to_bson(&create.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let pt_bson = create + .provisioned_throughput + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_doc = doc! { + "_id": { "table_id": &desc.table_id, "index_name": &create.index_name }, + "index_id": &index_id, + "index_type": "GSI", + "key_schema": key_schema_bson, + "projection": projection_bson, + "index_status": "ACTIVE", + "provisioned_throughput": pt_bson.unwrap_or(bson::Bson::Null), + }; + + self.catalog_db + .collection::("indexes") + .insert_one(index_doc) + .await + .map_err(|e| { + if e.to_string().contains("E11000") { + StorageError::IndexAlreadyExists(create.index_name.clone()) + } else { + StorageError::Internal(e.to_string()) + } + })?; + + self.gsi_cache.insert(desc.table_id.clone(), true); + } + + if let Some(delete) = &update.delete { + let desc = self + .describe_table_impl(account_id, &input.table_name) + .await?; + let result = self.catalog_db.collection::("indexes") + .delete_one(doc! { "_id": { "table_id": &desc.table_id, "index_name": &delete.index_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.deleted_count == 0 { + return Err(StorageError::IndexNotFound(delete.index_name.clone())); + } + + // Invalidate cache — may still have other GSIs + self.gsi_cache.remove(&desc.table_id); + } + } + } + + self.describe_table_impl(account_id, &input.table_name) + .await + } + + pub(crate) async fn table_key_info_impl( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + Self::validate_account_id(account_id)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.to_string()))?; + + let status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); + if status != "ACTIVE" { + return Err(StorageError::TableNotActive(table_name.to_string())); + } + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))? + .to_string(); + + let key_schema_bson = table_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("key_schema parse error: {e}")))?; + + let attr_defs_bson = table_doc + .get("attribute_definitions") + .ok_or_else(|| StorageError::Internal("missing attribute_definitions".to_string()))?; + let attribute_definitions: Vec = + bson::from_bson(attr_defs_bson.clone()) + .map_err(|e| StorageError::Internal(format!("attr_defs parse error: {e}")))?; + + let stream_spec_bson = table_doc.get("stream_specification"); + let stream_specification = stream_spec_bson.and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + + // Check for LSIs + let indexes_coll = self.catalog_db.collection::("indexes"); + let has_lsi = indexes_coll + .count_documents(doc! { "_id.table_id": &table_id, "index_type": "LSI" }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + > 0; + + Ok(TableKeyInfo { + table_name: table_name.to_string(), + account_id: account_id.to_string(), + table_id, + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + has_lsi, + stream_specification, + }) + } + + async fn index_info_impl( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> Result { + // First, get the table_id + let key_info = self.table_key_info_impl(account_id, table_name).await?; + self.index_info_by_table_id_impl(&key_info.table_id, index_name) + .await + } + + pub(crate) async fn index_info_by_table_id_impl( + &self, + table_id: &str, + index_name: &str, + ) -> Result { + let indexes_coll = self.catalog_db.collection::("indexes"); + let index_doc = indexes_coll + .find_one(doc! { "_id": { "table_id": table_id, "index_name": index_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::IndexNotFound(index_name.to_string()))?; + + let index_id = index_doc + .get_str("index_id") + .map_err(|_| StorageError::Internal("missing index_id".to_string()))? + .to_string(); + let index_type_str = index_doc + .get_str("index_type") + .map_err(|_| StorageError::Internal("missing index_type".to_string()))?; + let index_type = match index_type_str { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + _ => { + return Err(StorageError::Internal(format!( + "unknown index type: {index_type_str}" + ))); + } + }; + + let key_schema_bson = index_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema in index".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index key_schema parse: {e}")))?; + + let projection_bson = index_doc + .get("projection") + .ok_or_else(|| StorageError::Internal("missing projection in index".to_string()))?; + let projection: extenddb_core::types::Projection = bson::from_bson(projection_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index projection parse: {e}")))?; + + Ok(IndexInfo { + index_name: index_name.to_string(), + index_id, + index_type, + key_schema, + projection, + }) + } + + /// Convert a catalog table document to a `TableDescription`. + async fn doc_to_table_description( + &self, + doc: &Document, + ) -> Result { + let id_doc = doc + .get_document("_id") + .map_err(|_| StorageError::Internal("missing _id".to_string()))?; + let table_name = id_doc + .get_str("table_name") + .map_err(|_| StorageError::Internal("missing table_name".to_string()))? + .to_string(); + let account_id = id_doc + .get_str("account_id") + .map_err(|_| StorageError::Internal("missing account_id".to_string()))?; + + let table_id = doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))? + .to_string(); + let table_arn_val = doc + .get_str("table_arn") + .map_err(|_| StorageError::Internal("missing table_arn".to_string()))? + .to_string(); + + let status_str = doc.get_str("table_status").unwrap_or("ACTIVE"); + let table_status = match status_str { + "CREATING" => TableStatus::Creating, + "ACTIVE" => TableStatus::Active, + "DELETING" => TableStatus::Deleting, + "UPDATING" => TableStatus::Updating, + _ => TableStatus::Active, + }; + + let creation_dt = doc + .get_datetime("creation_date_time") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); + + let table_size_bytes = doc.get_i64("table_size_bytes").unwrap_or(0); + let item_count = doc.get_i64("item_count").unwrap_or(0); + let deletion_protection = doc.get_bool("deletion_protection_enabled").unwrap_or(false); + + let key_schema_bson = doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("key_schema: {e}")))?; + + let attr_defs_bson = doc + .get("attribute_definitions") + .ok_or_else(|| StorageError::Internal("missing attribute_definitions".to_string()))?; + let attribute_definitions: Vec = + bson::from_bson(attr_defs_bson.clone()) + .map_err(|e| StorageError::Internal(format!("attr_defs: {e}")))?; + + let stream_specification = doc.get("stream_specification").and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + + let billing_str = doc.get_str("billing_mode").unwrap_or("PROVISIONED"); + let billing_mode = match billing_str { + "PAY_PER_REQUEST" => BillingMode::PayPerRequest, + _ => BillingMode::Provisioned, + }; + + let pt_desc = doc + .get("provisioned_throughput") + .and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson::(b.clone()).ok() + } + }) + .map_or( + ProvisionedThroughputDescription { + read_capacity_units: 0, + write_capacity_units: 0, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + |pt| ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + ); + + let billing_summary = if billing_mode == BillingMode::PayPerRequest { + Some(BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_dt), + }) + } else { + None + }; + + // Fetch indexes + let indexes_coll = self.catalog_db.collection::("indexes"); + use futures::TryStreamExt; + let index_cursor = indexes_coll + .find(doc! { "_id.table_id": &table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_docs: Vec = index_cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut gsis = Vec::new(); + let mut lsis = Vec::new(); + + for idx_doc in &index_docs { + let idx_id_doc = idx_doc + .get_document("_id") + .map_err(|_| StorageError::Internal("missing index _id".to_string()))?; + let idx_name = idx_id_doc + .get_str("index_name") + .map_err(|_| StorageError::Internal("missing index_name".to_string()))? + .to_string(); + let idx_type = idx_doc.get_str("index_type").unwrap_or("GSI"); + + let idx_ks_bson = idx_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing index key_schema".to_string()))?; + let idx_key_schema: Vec = + bson::from_bson(idx_ks_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index key_schema: {e}")))?; + + let idx_proj_bson = idx_doc + .get("projection") + .ok_or_else(|| StorageError::Internal("missing index projection".to_string()))?; + let idx_projection: extenddb_core::types::Projection = + bson::from_bson(idx_proj_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index projection: {e}")))?; + + let idx_arn = index_arn(&self.region, account_id, &table_name, &idx_name); + + match idx_type { + "GSI" => { + let idx_pt = idx_doc + .get("provisioned_throughput") + .and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson::( + b.clone(), + ) + .ok() + } + }) + .map(|pt| ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }); + + gsis.push(GsiDescription { + index_name: idx_name, + key_schema: idx_key_schema, + projection: idx_projection, + index_status: idx_doc + .get_str("index_status") + .unwrap_or("ACTIVE") + .to_string(), + provisioned_throughput: idx_pt, + index_size_bytes: 0, + item_count: 0, + index_arn: idx_arn, + }); + } + "LSI" => { + lsis.push(LsiDescription { + index_name: idx_name, + key_schema: idx_key_schema, + projection: idx_projection, + index_size_bytes: 0, + item_count: 0, + index_arn: idx_arn, + }); + } + _ => {} + } + } + + // Stream info + let stream_label = doc + .get_str("stream_label") + .ok() + .map(std::string::ToString::to_string); + let stream_arn_opt = stream_label + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &table_name, label)); + + Ok(TableDescription { + table_name, + key_schema, + attribute_definitions, + table_status, + creation_date_time: creation_dt, + table_size_bytes, + item_count, + table_arn: table_arn_val, + table_id, + provisioned_throughput: pt_desc, + billing_mode_summary: billing_summary, + global_secondary_indexes: if gsis.is_empty() { None } else { Some(gsis) }, + local_secondary_indexes: if lsis.is_empty() { None } else { Some(lsis) }, + stream_specification, + latest_stream_arn: stream_arn_opt, + latest_stream_label: stream_label, + deletion_protection_enabled: deletion_protection, + sse_description: None, + table_class_summary: None, + on_demand_throughput: None, + }) + } +} diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs new file mode 100644 index 00000000..9a09ed5b --- /dev/null +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -0,0 +1,208 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! TTL cleanup background worker for `MongoDB`. + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::metrics::MetricsCollector; +use extenddb_core::types::UserIdentity; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, MetadataEngine, TableEngine}; + +use crate::MongoEngine; + +const SCAN_INTERVAL: Duration = Duration::from_secs(60); +const BATCH_SIZE: usize = 100; + +pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { + let region_arc: Arc = Arc::from(storage.region.as_str()); + + loop { + tokio::time::sleep(SCAN_INTERVAL).await; + retry_pending_indexes(&storage).await; + sweep_expired_items(&storage, &metrics, ®ion_arc).await; + } +} + +async fn retry_pending_indexes(storage: &MongoEngine) { + let Ok(pending) = MetadataEngine::all_tables_with_ttl(storage).await else { + return; + }; + let Ok(ready) = MetadataEngine::all_tables_with_ttl_index_ready(storage).await else { + return; + }; + let ready_set: std::collections::HashSet<(&str, &str)> = ready + .iter() + .map(|(a, t, _)| (a.as_str(), t.as_str())) + .collect(); + for (account_id, table_name, ttl_attr) in &pending { + if !ready_set.contains(&(account_id.as_str(), table_name.as_str())) { + if let Err(e) = + MetadataEngine::create_ttl_index(storage, account_id, table_name, ttl_attr).await + { + tracing::debug!("TTL worker: index creation retry failed for {table_name}: {e}"); + } else { + tracing::info!("TTL worker: index created for {table_name}"); + } + } + } +} + +async fn sweep_expired_items(storage: &MongoEngine, metrics: &MetricsCollector, region: &Arc) { + let ttl_identity = UserIdentity { + identity_type: "Service".to_owned(), + principal_id: "dynamodb.amazonaws.com".to_owned(), + }; + + let tables = match MetadataEngine::all_tables_with_ttl_index_ready(storage).await { + Ok(t) => t, + Err(e) => { + tracing::warn!("TTL worker: failed to list tables: {e}"); + return; + } + }; + + let now_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + for (account_id, table_name, ttl_attribute) in &tables { + let items = match MetadataEngine::find_expired_items_indexed( + storage, + account_id, + table_name, + ttl_attribute, + BATCH_SIZE, + ) + .await + { + Ok(items) => items, + Err(e) => { + tracing::warn!("TTL worker: find expired failed for {table_name}: {e}"); + continue; + } + }; + + if items.is_empty() { + continue; + } + + let key_info = match TableEngine::table_key_info(storage, account_id, table_name).await { + Ok(ki) => ki, + Err(e) => { + tracing::warn!("TTL worker: key info failed for {table_name}: {e}"); + continue; + } + }; + + let view_type = stream_view_type(&key_info); + let (condition_expr, maps) = build_ttl_condition(ttl_attribute, now_epoch); + + let mut deleted = 0usize; + for item in &items { + let staleness = item + .get(ttl_attribute.as_str()) + .and_then(|av| { + if let extenddb_core::types::AttributeValue::N(n) = av { + n.parse::().ok() + } else { + None + } + }) + .map(|ttl_val| now_epoch.saturating_sub(ttl_val)); + + let key: extenddb_core::types::Item = key_info + .key_schema + .iter() + .filter_map(|ks| { + item.get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let return_old = view_type.is_some(); + let stream = view_type.map(|vt| extenddb_storage::StreamCapture { + view_type: vt, + user_identity: Some(ttl_identity.clone()), + region: region.clone(), + }); + match DataEngine::delete_item( + storage, + &key_info, + &key, + return_old, + Some(&condition_expr), + &maps, + stream.as_ref(), + ) + .await + { + Err(StorageError::ConditionFailed(_)) => {} + Err(e) => { + tracing::warn!("TTL worker: delete failed for {table_name}: {e}"); + } + Ok(_old_item) => { + deleted += 1; + metrics.record_ttl_deletion(table_name); + if let Some(s) = staleness { + #[allow(clippy::cast_precision_loss)] + metrics.record_ttl_staleness(table_name, s as f64); + } + } + } + } + + if deleted > 0 { + tracing::info!("TTL worker: deleted {deleted} expired items from {table_name}"); + } + } +} + +fn stream_view_type( + key_info: &extenddb_core::types::TableKeyInfo, +) -> Option { + key_info.stream_specification.as_ref().and_then(|spec| { + if spec.stream_enabled { + spec.stream_view_type + } else { + None + } + }) +} + +fn build_ttl_condition( + ttl_attribute: &str, + now_epoch: u64, +) -> ( + extenddb_core::expression::Expr, + extenddb_core::expression::ExpressionMaps, +) { + use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement}; + use std::collections::HashMap; + + let ttl_path = vec![PathElement::Attribute("#ttl".to_owned())]; + let condition_expr = Expr::And( + Box::new(Expr::Function { + name: "attribute_exists".to_owned(), + args: vec![Expr::Path(ttl_path.clone())], + }), + Box::new(Expr::Compare { + left: Box::new(Expr::Path(ttl_path)), + op: CompareOp::Le, + right: Box::new(Expr::Placeholder("now".to_owned())), + }), + ); + + let mut names = HashMap::new(); + names.insert("ttl".to_owned(), ttl_attribute.to_owned()); + let mut values = HashMap::new(); + values.insert( + "now".to_owned(), + extenddb_core::types::AttributeValue::N(now_epoch.to_string()), + ); + + (condition_expr, ExpressionMaps::new(names, values)) +} diff --git a/crates/storage-mongodb/src/worker_store.rs b/crates/storage-mongodb/src/worker_store.rs new file mode 100644 index 00000000..6cb7d3a9 --- /dev/null +++ b/crates/storage-mongodb/src/worker_store.rs @@ -0,0 +1,148 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `WorkerStore` implementation for `MongoDB`. +//! +//! Processes control-plane state transitions (CREATING → ACTIVE, DELETING → deleted) +//! as a background safety net for incomplete operations. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{Document, doc}; + +use extenddb_storage::WorkerStore; +use extenddb_storage::error::StorageError; + +use crate::MongoEngine; +use crate::data::data_collection_name; + +impl WorkerStore for MongoEngine { + fn process_control_plane_transitions( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let mut transitions = Vec::new(); + + let tables_coll = self.catalog_db.collection::("tables"); + let now = mongodb::bson::DateTime::now(); + + // CREATING → ACTIVE: find tables stuck in CREATING whose transition time has passed + let creating_filter = doc! { + "table_status": "CREATING", + "status_transition_at": { "$lte": now }, + }; + let mut cursor = tables_coll + .find(creating_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + while let Some(table_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let table_name = table_doc + .get_str("table_name") + .unwrap_or_default() + .to_owned(); + let account_id = table_doc.get_str("account_id").unwrap_or_default(); + + tables_coll + .update_one( + doc! { + "account_id": account_id, + "table_name": &table_name, + "table_status": "CREATING", + }, + doc! { + "$set": { "table_status": "ACTIVE" }, + "$unset": { "status_transition_at": "" }, + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + transitions.push((table_name, "CREATING → active")); + } + + // DELETING → deleted: find tables stuck in DELETING whose transition time has passed + let deleting_filter = doc! { + "table_status": "DELETING", + "status_transition_at": { "$lte": now }, + }; + let mut cursor = tables_coll + .find(deleting_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + while let Some(table_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let table_name = table_doc + .get_str("table_name") + .unwrap_or_default() + .to_owned(); + let account_id = table_doc + .get_str("account_id") + .unwrap_or_default() + .to_owned(); + let table_id = table_doc.get_str("table_id").unwrap_or_default().to_owned(); + let table_arn = table_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + + // Drop the data collection + let coll_name = data_collection_name(&table_id); + let _ = self.data_db.collection::(&coll_name).drop().await; + + // Drop index collections + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut idx_cursor = indexes_coll + .find(doc! { "_id.table_id": &table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + while let Some(idx_doc) = idx_cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + if let Ok(index_id) = idx_doc.get_str("index_id") { + let idx_coll_name = data_collection_name(index_id); + let _ = self + .data_db + .collection::(&idx_coll_name) + .drop() + .await; + } + } + + // Delete index catalog entries + indexes_coll + .delete_many(doc! { "_id.table_id": &table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Delete tags for this resource + self.catalog_db + .collection::("tags") + .delete_many(doc! { "resource_arn": &table_arn }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Delete the table catalog entry + tables_coll + .delete_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + transitions.push((table_name, "DELETING → deleted")); + } + + Ok(transitions) + }) + } +} From f64cc358f40cfad2ae92e3df40fc04802e189d72 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Fri, 5 Jun 2026 21:08:27 -0700 Subject: [PATCH 03/83] chore: add MongoDB test configuration and runner updates - extenddb-mongo.toml: integration test config for MongoDB backend using ~/.extenddb/tls paths (portable) and enforce_reserved_keywords=true - extenddb.sample.toml: add [storage.mongodb] section - devtools/run-tests: export EXTENDDB_CONFIG; only set EXTENDDB_TEST_PG_CONNECTION_STRING for postgres URLs --- extenddb-mongo.toml | 32 ++++++++++++++++++++++++++++++++ extenddb.sample.toml | 7 ++++++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 extenddb-mongo.toml diff --git a/extenddb-mongo.toml b/extenddb-mongo.toml new file mode 100644 index 00000000..e3423540 --- /dev/null +++ b/extenddb-mongo.toml @@ -0,0 +1,32 @@ +# ExtendDB config for MongoDB backend (integration testing) + +[server] +bind_addr = "127.0.0.1" +port = 8100 +region = "us-east-1" + +[server.tls] +cert_path = "~/.extenddb/tls/cert.pem" +key_path = "~/.extenddb/tls/key.pem" + +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true" + +[limits] +enforce_reserved_keywords = true + +[auth] +provider = "builtin" + +[logging] +level = "debug" +format = "pretty" + +[import] +paths = ["/private/tmp", "/tmp"] + +[export] +paths = ["/private/tmp", "/tmp"] diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 2c3ba608..d533ac47 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -29,7 +29,7 @@ # run_dir = "~/.extenddb/run" # Directory for PID file (~ is expanded to $HOME) [storage] -# backend = "postgres" # Storage backend (only "postgres" supported) +# backend = "postgres" # Storage backend: "postgres" or "mongodb" [storage.postgres] # Connection string points to the CATALOG database. @@ -52,6 +52,11 @@ # DynamoDB request makes concurrent authz queries # — size this to match expected concurrency. +[storage.mongodb] +# MongoDB connection string. Requires a replica set (even single-node). +# connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +# max_pool_size = 20 # Maximum concurrent connections to MongoDB. + [auth] # provider = "builtin" # Auth provider: # "builtin" — SigV4 verification with local credential From 1024010af19728c9da94832fb1d9e2c634233537 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Fri, 5 Jun 2026 21:08:36 -0700 Subject: [PATCH 04/83] docs: add MongoDB backend documentation - docs/local-mongodb-setup.md: MongoDB installation and replica set setup - docs/getting-started.md: add MongoDB build/init instructions - AGENTS.md: update architecture, prerequisites, pitfalls for MongoDB --- AGENTS.md | 36 +++++-- docs/getting-started.md | 21 +++- docs/local-mongodb-setup.md | 191 ++++++++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 13 deletions(-) create mode 100644 docs/local-mongodb-setup.md diff --git a/AGENTS.md b/AGENTS.md index ee53cbea..bdf97ff7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ by AWS engineers. It is not a fork of DynamoDB and contains no DynamoDB source c protocol: any AWS SDK, CLI, or tool that works with DynamoDB works with ExtendDB, unchanged. - **Language:** Rust (edition 2024, MSRV 1.88+) -- **Storage backend:** PostgreSQL 14+ +- **Storage backends:** PostgreSQL 14+ (default), MongoDB 6.0+ (feature flag `mongodb`) - **Architecture:** Async (tokio), trait-based storage abstraction - **Authentication:** Mandatory SigV4 with built-in IAM (users, groups, roles, policies) - **TLS:** Mandatory (self-signed cert generated by default) @@ -51,7 +51,8 @@ extenddb/ The `TableEngine` trait in `crates/storage/src/lib.rs` defines the storage interface. All storage backends implement this trait: -- **Current:** `storage-postgres` (PostgreSQL) +- `storage-postgres` (PostgreSQL) — default backend +- `storage-mongodb` (MongoDB) — feature flag `mongodb` The trait uses RPITIT (return-position impl Trait in traits) for async methods — no `#[async_trait]` macro. @@ -66,12 +67,14 @@ extenddb (bin) │ ├─> extenddb-core (pure sync, no async) │ └─> extenddb-storage (trait definitions) ├─> extenddb-auth - └─> extenddb-storage-postgres + ├─> extenddb-storage-postgres (feature: postgres) + └─> extenddb-storage-mongodb (feature: mongodb) ``` - **extenddb-core:** Pure synchronous Rust. No async, no I/O. Types, validation, expression parsing. - **extenddb-storage:** Trait definitions only. No implementation. - **extenddb-storage-postgres:** Concrete PostgreSQL implementation. +- **extenddb-storage-mongodb:** Concrete MongoDB implementation. - **extenddb-engine:** Operation handlers that call storage traits. - **extenddb-server:** HTTP server, management API, web console. - **extenddb-auth:** SigV4 signature verification, IAM policy evaluation. @@ -82,13 +85,22 @@ extenddb (bin) ### Prerequisites - Rust 1.88+ (`rustup update`) -- PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) +- Storage backend (one of): + - PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) + - MongoDB 6.0+ with replica set (see `docs/local-mongodb-setup.md`) - Python 3.10+ for tests (`python3 -m venv ~/venvs/extenddb-venv && source ~/venvs/extenddb-venv/bin/activate && pip install -r requirements.txt`) ### Build ```bash +# PostgreSQL backend (default) cargo build --release + +# MongoDB backend +cargo build --release --features mongodb + +# Both backends +cargo build --release --features postgres,mongodb ``` Binary: `target/release/extenddb` @@ -109,11 +121,15 @@ cargo clippy --all-targets -- -D warnings ### Initialize (first time only) ```bash +# PostgreSQL (default) ./target/release/extenddb init --config extenddb.toml + +# MongoDB +./target/release/extenddb init --backend mongodb --config extenddb.toml ``` This creates: -- PostgreSQL databases (`extenddb_catalog`, `extenddb_account_`) +- Databases (`extenddb_catalog`, `extenddb_data`) - Admin user credentials (printed to stdout — save the password!) - Self-signed TLS certificate at `~/.extenddb/tls/cert.pem` - Config file `extenddb.toml` @@ -357,6 +373,9 @@ Expression parsing lives in `crates/core/src/expression/`. This is pure sync Rus | Differences from DynamoDB | `docs/differences-from-dynamodb.md` | Behavioral differences | | Troubleshooting | `docs/troubleshooting.md` | Common errors and solutions | | Storage Component Design | `docs/design/04-component-storage.md` | Storage trait design | +| MongoDB Design | `docs/design/13-storage-mongodb.md` | MongoDB backend architecture | +| Local PostgreSQL Setup | `docs/local-postgres-setup.md` | PostgreSQL installation and config | +| Local MongoDB Setup | `docs/local-mongodb-setup.md` | MongoDB installation and config | | Testing Design | `docs/design/09-testing.md` | Test strategy and infrastructure | | Extending Storage | `12-backend-plugin-architecture.md` | Guide to implementing storage backends | @@ -448,9 +467,10 @@ Activate when the user asks about installing, configuring, running, or debugging 1. **TLS is mandatory** — server refuses to start without it. Use `AWS_CA_BUNDLE` for self-signed certs. 2. **Auth is mandatory** — all requests must be SigV4-signed. Use `extenddb manage` or web console to create credentials. 3. **Account isolation** — all operations are scoped to `account_id`. Different accounts can have tables with the same name. -4. **PostgreSQL must be running** — `extenddb init` and `extenddb serve` require a running PostgreSQL instance. -5. **Python venv** — activate the venv before running tests: `source ~/venvs/extenddb-venv/bin/activate` -6. **Test credentials** — run `devtools/provision-test-credentials` before pytest to create test users and keys. +4. **Storage backend must be running** — `extenddb init` and `extenddb serve` require a running PostgreSQL or MongoDB instance. +5. **MongoDB requires a replica set** — even single-node MongoDB must be configured with `--replSet` for transactions and streams. +6. **Python venv** — activate the venv before running tests: `source ~/venvs/extenddb-venv/bin/activate` +7. **Test credentials** — run `devtools/provision-test-credentials` before pytest to create test users and keys. ## Getting Help diff --git a/docs/getting-started.md b/docs/getting-started.md index 369c9ae7..7139fd35 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -29,7 +29,9 @@ software on your behalf. After the script completes, continue from ## Prerequisites -- PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) +- **Storage backend** (one of): + - PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) + - MongoDB 6.0+ with replica set (see `docs/local-mongodb-setup.md`) - Rust toolchain (1.88+) - AWS CLI v2 (for testing) - Python 3.10+ with virtual environment (see [Python Environment Setup](../README.md#python-environment-setup) in the README) @@ -37,7 +39,14 @@ software on your behalf. After the script completes, continue from ## 1. Build extenddb ```bash +# PostgreSQL backend (default) cargo build --release + +# MongoDB backend +cargo build --release --features mongodb + +# Both backends +cargo build --release --features postgres,mongodb ``` The binary is at `target/release/extenddb`. @@ -47,15 +56,17 @@ The binary is at `target/release/extenddb`. Run `extenddb init` to create the catalog and data databases: ```bash +# PostgreSQL (default) ./target/release/extenddb init + +# MongoDB +./target/release/extenddb init --backend mongodb ``` This will: -- Create a `extenddb` PostgreSQL user (if it doesn't exist) - Create the `extenddb_catalog` database (catalog metadata) -- Create the `extenddb` database (user item data) -- Run schema migrations -- Generate an AES-256-GCM encryption key (for future access key storage) +- Create the data database (user item data) +- Generate an AES-256-GCM encryption key (for access key storage) - Create a default account and print the account ID - Create an `admin` user and print the credentials once - Generate a self-signed TLS certificate at `~/.extenddb/tls/` diff --git a/docs/local-mongodb-setup.md b/docs/local-mongodb-setup.md new file mode 100644 index 00000000..b3e6d6a6 --- /dev/null +++ b/docs/local-mongodb-setup.md @@ -0,0 +1,191 @@ +# Local MongoDB Setup + +## Prerequisites + +- MongoDB 6.0+ (for multi-document transactions) +- A replica set configuration (required even for single-node deployments) + +## Installation + +### macOS (Homebrew) + +```bash +brew tap mongodb/brew +brew install mongodb-community@7.0 +``` + +### Linux (Ubuntu/Debian) + +```bash +curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | \ + sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor +echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] \ + https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | \ + sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list +sudo apt-get update && sudo apt-get install -y mongodb-org +``` + +### Docker (recommended for development) + +```bash +docker run -d --name extenddb-mongo \ + -p 27017:27017 \ + mongo:7 --replSet rs0 +``` + +## Replica Set Initialization + +MongoDB must run as a replica set for transactions and Change Streams. + +### Single-node replica set (development) + +```bash +# If using Docker: +docker exec extenddb-mongo mongosh --quiet --eval "rs.initiate()" + +# If using a local install: +mongosh --eval "rs.initiate()" +``` + +Wait a few seconds for the replica set to elect a primary, then verify: + +```bash +mongosh --eval "rs.status().ok" +# Should output: 1 +``` + +### Homebrew (macOS) with replica set + +Edit the MongoDB config to add replica set: + +```bash +# Find the config file +brew --prefix mongodb-community@7.0 +# Usually: /opt/homebrew/etc/mongod.conf +``` + +Add to `mongod.conf`: +```yaml +replication: + replSetName: rs0 +``` + +Restart and initiate: +```bash +brew services restart mongodb-community@7.0 +mongosh --eval "rs.initiate()" +``` + +## Connection Details + +| Setting | Value | +|---------|-------| +| Host | `localhost` | +| Port | `27017` | +| Replica set | `rs0` | +| Connection string | `mongodb://localhost:27017/?replicaSet=rs0` | + +For Docker on a non-default port: +``` +mongodb://localhost:27018/?replicaSet=rs0&directConnection=true +``` + +## Building with MongoDB Support + +The MongoDB backend is behind a feature flag: + +```bash +cargo build --release --features mongodb +``` + +To build with both backends: +```bash +cargo build --release --features postgres,mongodb +``` + +## Initializing ExtendDB with MongoDB + +```bash +./target/release/extenddb init --backend mongodb --config extenddb.toml +``` + +This creates: +- `extenddb_catalog` database (table metadata, IAM, settings) +- `extenddb_data` database (per-table item collections) +- Admin user credentials (printed to stdout) +- Self-signed TLS certificate at `~/.extenddb/tls/cert.pem` +- Config file `extenddb.toml` + +The generated config will contain: +```toml +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +``` + +## Starting the Server + +```bash +./target/release/extenddb serve --config extenddb.toml +``` + +## Config Mapping + +```toml +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +# max_pool_size = 20 +``` + +Or via environment variable: +```bash +export EXTENDDB__STORAGE__MONGODB__CONNECTION_STRING="mongodb://localhost:27017/?replicaSet=rs0" +``` + +## Verifying the Connection + +```bash +# Health check +curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:8000/health + +# List tables (should return empty) +aws dynamodb list-tables \ + --endpoint-url https://127.0.0.1:8000 \ + --region us-east-1 +``` + +## Differences from PostgreSQL Backend + +- **Replica set required:** Even single-node MongoDB must be configured as a replica set. +- **DynamoDB Streams:** Inline record writes with atomic sequence numbers. +- **GSI propagation:** Synchronous inline updates during data operations. +- **Concurrency model:** Optimistic versioning with retry on conflict (vs row-level locking in PostgreSQL). + +## Stopping + +```bash +# ExtendDB +./target/release/extenddb stop --config extenddb.toml + +# Docker MongoDB +docker stop extenddb-mongo + +# Homebrew MongoDB +brew services stop mongodb-community@7.0 +``` + +--- + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is a trademark +of Amazon.com, Inc. From 8bf6c55cef64892a3494b76390e583c4d22a4125 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:44:17 -0700 Subject: [PATCH 05/83] fix(mongodb): emit DynamoDB-compliant event names in stream records stream_engine.rs and data_engine.rs wrote the shadow event_name column via format!("{:?}", record.event_name), producing "Insert" / "Modify" / "Remove". DynamoDB Streams' wire contract is uppercase: "INSERT" / "MODIFY" / "REMOVE". Add event_name_ddb_str() in stream_engine.rs to map StreamEventName to its wire-format string, and use it at both call sites. Unit test asserts each enum variant maps to the expected uppercase string. --- crates/storage-mongodb/src/data_engine.rs | 2 +- crates/storage-mongodb/src/stream_engine.rs | 30 ++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index e0b10646..816a3eca 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1459,7 +1459,7 @@ impl MongoEngine { "sequence_number": &record.dynamodb.sequence_number, "shard_id": &shard_id, "table_id": table_id, - "event_name": format!("{:?}", record.event_name), + "event_name": crate::stream_engine::event_name_ddb_str(record.event_name), "record_data": record_bson, "created_at": mongodb::bson::DateTime::now(), }) diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index 98044bae..ed62080c 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -17,8 +17,8 @@ use mongodb::bson::{self, Document, doc}; use mongodb::options::FindOptions; use extenddb_core::types::{ - DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, - StreamSummary, StreamViewType, + DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamEventName, + StreamRecord, StreamStatus, StreamSummary, StreamViewType, }; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; @@ -29,6 +29,18 @@ use crate::MongoEngine; const SHARDS_PER_STREAM: u32 = 4; +/// Map a `StreamEventName` to its DynamoDB wire-format string. +/// +/// DynamoDB Streams records use uppercase event names (`INSERT`, `MODIFY`, `REMOVE`). +/// The enum's `Debug` output is Rust-cased (`Insert`, ...) so we must not use that. +pub(crate) fn event_name_ddb_str(name: StreamEventName) -> &'static str { + match name { + StreamEventName::Insert => "INSERT", + StreamEventName::Modify => "MODIFY", + StreamEventName::Remove => "REMOVE", + } +} + impl MongoEngine { /// Initialize stream shards for a table. Only creates shard documents; /// the caller is responsible for setting `stream_label` on the table doc. @@ -90,7 +102,7 @@ impl StreamEngine for MongoEngine { "sequence_number": &record.dynamodb.sequence_number, "shard_id": &shard_id, "table_id": table_id, - "event_name": format!("{:?}", record.event_name), + "event_name": event_name_ddb_str(record.event_name), "record_data": record_bson, "created_at": BsonDateTime::now(), }) @@ -524,3 +536,15 @@ impl StreamEngine for MongoEngine { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn event_name_maps_to_ddb_wire_format() { + assert_eq!(event_name_ddb_str(StreamEventName::Insert), "INSERT"); + assert_eq!(event_name_ddb_str(StreamEventName::Modify), "MODIFY"); + assert_eq!(event_name_ddb_str(StreamEventName::Remove), "REMOVE"); + } +} From 3f63a3d22dc73dbda8f006a6e9060d758cf17d26 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:44:32 -0700 Subject: [PATCH 06/83] fix(mongodb): reject BETWEEN with low > high on query key path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DynamoDB rejects a KeyConditionExpression sk BETWEEN :lo AND :hi with :lo > :hi as ValidationException. The engine layer's condition evaluator does this check for filter/condition expressions, but the KeyConditionExpression path in Query goes through the storage backend's sort-key filter builder, which was emitting $gte lo, $lte hi with no check — matching zero documents without an error. Add the check in build_sk_filter. Comparison is done in the AttributeValue domain before Decimal128/f64 conversion. Numeric comparison uses f64 for ordering only; values that would lose Decimal128 precision are rejected downstream in sk_to_bson. Unit tests cover S, N, and B ordering. --- crates/storage-mongodb/src/data_engine.rs | 79 +++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 816a3eca..f451b583 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -2020,6 +2020,11 @@ fn build_sk_filter( let sk_type = infer_sk_type_from_field(sk_field); let low_av = resolve_key_expr(low, maps)?; let high_av = resolve_key_expr(high, maps)?; + if sk_between_low_gt_high(&low_av, &high_av) { + return Err(StorageError::Validation( + "Invalid KeyConditionExpression: The BETWEEN operator requires upper bound to be greater than or equal to lower bound".to_owned(), + )); + } let low_bson = sk_to_bson(&low_av, sk_type)?; let high_bson = sk_to_bson(&high_av, sk_type)?; Ok(doc! { sk_field: { "$gte": low_bson, "$lte": high_bson } }) @@ -2048,6 +2053,31 @@ fn build_sk_filter( } /// Convert an `AttributeValue` sort key to the appropriate BSON type. +/// Return true when a sort-key BETWEEN's low bound is strictly greater than its high bound. +/// +/// DynamoDB rejects this at the wire layer with a ValidationException; the storage +/// backend must reject it too, since the engine layer only validates BETWEEN for +/// filter/condition expressions, not for KeyConditionExpression's sort-key path. +/// +/// The comparison is done in the source AttributeValue domain so it happens before +/// any Decimal128/f64 conversion that could mask ordering. Strings are compared +/// lexicographically (matching DynamoDB), numbers via f64 (adequate for ordering — +/// values exceeding Decimal128 range are rejected downstream in `sk_to_bson`), and +/// binary bytewise. +fn sk_between_low_gt_high(low: &AttributeValue, high: &AttributeValue) -> bool { + match (low, high) { + (AttributeValue::S(l), AttributeValue::S(h)) => l > h, + (AttributeValue::N(l), AttributeValue::N(h)) => { + match (l.parse::(), h.parse::()) { + (Ok(lf), Ok(hf)) => lf > hf, + _ => false, // downstream sk_to_bson will surface the parse error + } + } + (AttributeValue::B(l), AttributeValue::B(h)) => l > h, + _ => false, // type mismatch — downstream sk_to_bson will surface it + } +} + fn sk_to_bson( av: &AttributeValue, sk_type: ScalarAttributeType, @@ -2154,3 +2184,52 @@ fn project_item( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn between_low_gt_high_string() { + assert!(sk_between_low_gt_high( + &AttributeValue::S("z".into()), + &AttributeValue::S("a".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::S("a".into()), + &AttributeValue::S("z".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::S("m".into()), + &AttributeValue::S("m".into()) + )); + } + + #[test] + fn between_low_gt_high_number() { + assert!(sk_between_low_gt_high( + &AttributeValue::N("100".into()), + &AttributeValue::N("50".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::N("50".into()), + &AttributeValue::N("100".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::N("42".into()), + &AttributeValue::N("42".into()) + )); + } + + #[test] + fn between_low_gt_high_binary() { + assert!(sk_between_low_gt_high( + &AttributeValue::B(vec![0xff]), + &AttributeValue::B(vec![0x00]) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::B(vec![0x00]), + &AttributeValue::B(vec![0xff]) + )); + } +} From 1dd4ce2c56923fd5e88437375dcaa4f8554d3bf6 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 17:03:06 -0700 Subject: [PATCH 07/83] fix(mongodb): populate CancellationReason.Item on condition-check failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously all four TransactWriteItems condition-failure sites in data_engine.rs (Put, Delete, Update, ConditionCheck) called condition_check_failed_with_item(None), discarding the pre-existing item that had already been loaded into scope. Callers that set ReturnValuesOnConditionCheckFailure=ALL_OLD on their transact op therefore got a CancellationReason with no Item field — inconsistent with DynamoDB, which returns the failing item under that flag. Thread return_values_on_ccf from TransactWriteOp through the mongo crate's OwnedTransactWriteOp, and add a small helper ccf_return_item that gates inclusion on both (a) ALL_OLD requested and (b) the item actually existed. Preserves DDB's guarantee that missing items never manifest as CancellationReason.Item. Adds unit tests for the helper covering all three code paths. --- crates/storage-mongodb/src/data_engine.rs | 119 +++++++++++++++++----- 1 file changed, 94 insertions(+), 25 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index f451b583..fdf9b7a2 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -11,8 +11,9 @@ use extenddb_core::expression::{ self, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, UpdateAction, }; use extenddb_core::types::{ - AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, StreamEventName, - StreamRecord, StreamRecordData, TableKeyInfo, extract_key, item_size_bytes, + AttributeValue, Item, KeySchemaElement, KeyType, ReturnValuesOnConditionCheckFailure, + ScalarAttributeType, StreamEventName, StreamRecord, StreamRecordData, TableKeyInfo, + extract_key, item_size_bytes, }; use extenddb_storage::error::StorageError; use extenddb_storage::util::{ @@ -1654,6 +1655,7 @@ impl MongoEngine { item, condition, maps, + return_values_on_ccf, .. } => { validation::validate_item_keys( @@ -1679,11 +1681,12 @@ impl MongoEngine { if let Some(cond) = condition { let existing_item = if let Some(doc) = existing_doc.as_ref() { - document_to_item(doc).map_err(TransactOpError::Storage)? + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) } else { - Item::new() + None }; - let passed = expression::evaluate_condition(cond, &existing_item, maps) + let for_eval = existing_item.clone().unwrap_or_default(); + let passed = expression::evaluate_condition(cond, &for_eval, maps) .map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error( e.to_string(), @@ -1691,7 +1694,10 @@ impl MongoEngine { })?; if !passed { return Err(TransactOpError::Cancel( - CancellationReason::condition_check_failed_with_item(None), + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), )); } } @@ -1716,6 +1722,7 @@ impl MongoEngine { key, condition, maps, + return_values_on_ccf, .. } => { validation::validate_key_only( @@ -1743,11 +1750,12 @@ impl MongoEngine { })?; let existing_item = if let Some(doc) = existing_doc.as_ref() { - document_to_item(doc).map_err(TransactOpError::Storage)? + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) } else { - Item::new() + None }; - let passed = expression::evaluate_condition(cond, &existing_item, maps) + let for_eval = existing_item.clone().unwrap_or_default(); + let passed = expression::evaluate_condition(cond, &for_eval, maps) .map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error( e.to_string(), @@ -1755,7 +1763,10 @@ impl MongoEngine { })?; if !passed { return Err(TransactOpError::Cancel( - CancellationReason::condition_check_failed_with_item(None), + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), )); } } @@ -1773,6 +1784,7 @@ impl MongoEngine { actions, condition, maps, + return_values_on_ccf, .. } => { validation::validate_key_only( @@ -1796,18 +1808,17 @@ impl MongoEngine { .await .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; - let mut item = if let Some(doc) = existing_doc.as_ref() { - document_to_item(doc).map_err(TransactOpError::Storage)? + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) } else { - key.clone() + None }; + let mut item = existing_item.clone().unwrap_or_else(|| key.clone()); + if let Some(cond) = condition { - let condition_item = if existing_doc.is_some() { - &item - } else { - &std::collections::BTreeMap::new() - }; + let empty = std::collections::BTreeMap::new(); + let condition_item = if existing_item.is_some() { &item } else { &empty }; let passed = expression::evaluate_condition(cond, condition_item, maps) .map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error( @@ -1816,7 +1827,10 @@ impl MongoEngine { })?; if !passed { return Err(TransactOpError::Cancel( - CancellationReason::condition_check_failed_with_item(None), + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), )); } } @@ -1845,7 +1859,7 @@ impl MongoEngine { key, condition, maps, - .. + return_values_on_ccf, } => { validation::validate_key_only( key, @@ -1869,18 +1883,22 @@ impl MongoEngine { .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; let existing_item = if let Some(doc) = existing_doc.as_ref() { - document_to_item(doc).map_err(TransactOpError::Storage)? + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) } else { - Item::new() + None }; - let passed = expression::evaluate_condition(condition, &existing_item, maps) + let for_eval = existing_item.clone().unwrap_or_default(); + let passed = expression::evaluate_condition(condition, &for_eval, maps) .map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; if !passed { return Err(TransactOpError::Cancel( - CancellationReason::condition_check_failed_with_item(None), + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), )); } @@ -1897,6 +1915,23 @@ enum TransactOpError { Storage(StorageError), } +/// Choose the `Item` value to include in a `CancellationReason` when a +/// condition check fails inside `TransactWriteItems`. +/// +/// Per DynamoDB's contract, the pre-existing item is returned only when the +/// caller requested `ReturnValuesOnConditionCheckFailure = ALL_OLD` AND the +/// item existed at the time of the check. In all other cases the field is +/// omitted (returned as `None`). +fn ccf_return_item( + rv: ReturnValuesOnConditionCheckFailure, + existing: Option<&Item>, +) -> Option { + match rv { + ReturnValuesOnConditionCheckFailure::AllOld => existing.cloned(), + ReturnValuesOnConditionCheckFailure::None => None, + } +} + /// Owned version of `TransactWriteOp` to allow moving into async blocks. enum OwnedTransactWriteOp { Put { @@ -1904,12 +1939,14 @@ enum OwnedTransactWriteOp { item: Item, condition: Option, maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, }, Delete { key_info: TableKeyInfo, key: Item, condition: Option, maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, }, Update { key_info: TableKeyInfo, @@ -1917,12 +1954,14 @@ enum OwnedTransactWriteOp { actions: Vec, condition: Option, maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, }, ConditionCheck { key_info: TableKeyInfo, key: Item, condition: Expr, maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, }, } @@ -1933,24 +1972,28 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { item, condition, maps, + return_values_on_ccf, .. } => OwnedTransactWriteOp::Put { key_info: (*key_info).clone(), item: (*item).clone(), condition: condition.cloned(), maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, }, TransactWriteOp::Delete { key_info, key, condition, maps, + return_values_on_ccf, .. } => OwnedTransactWriteOp::Delete { key_info: (*key_info).clone(), key: (*key).clone(), condition: condition.cloned(), maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, }, TransactWriteOp::Update { key_info, @@ -1958,6 +2001,7 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { actions, condition, maps, + return_values_on_ccf, .. } => OwnedTransactWriteOp::Update { key_info: (*key_info).clone(), @@ -1965,18 +2009,20 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { actions: actions.to_vec(), condition: condition.cloned(), maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, }, TransactWriteOp::ConditionCheck { key_info, key, condition, maps, - .. + return_values_on_ccf, } => OwnedTransactWriteOp::ConditionCheck { key_info: (*key_info).clone(), key: (*key).clone(), condition: (*condition).clone(), maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, }, } } @@ -2221,6 +2267,29 @@ mod tests { )); } + #[test] + fn ccf_return_item_all_old_with_existing() { + let mut item = Item::new(); + item.insert("a".to_string(), AttributeValue::S("1".to_string())); + let returned = + ccf_return_item(ReturnValuesOnConditionCheckFailure::AllOld, Some(&item)); + assert_eq!(returned, Some(item)); + } + + #[test] + fn ccf_return_item_all_old_without_existing() { + let returned = ccf_return_item(ReturnValuesOnConditionCheckFailure::AllOld, None); + assert_eq!(returned, None); + } + + #[test] + fn ccf_return_item_none_never_returns() { + let mut item = Item::new(); + item.insert("a".to_string(), AttributeValue::S("1".to_string())); + let returned = ccf_return_item(ReturnValuesOnConditionCheckFailure::None, Some(&item)); + assert_eq!(returned, None); + } + #[test] fn between_low_gt_high_binary() { assert!(sk_between_low_gt_high( From b93891cac872874578010ad242eda3aba23365bf Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 17:08:02 -0700 Subject: [PATCH 08/83] fix(mongodb): reject numeric key values that exceed Decimal128 precision The mongo backend stores numeric partition/sort keys as BSON Decimal128 for correct numeric ordering. Decimal128 supports 34 significant decimal digits; DynamoDB supports up to 38. Previously the write path (data/mod.rs::item_to_document), key-filter path (data/mod.rs::pk_filter), and sort-key comparison path (data_engine.rs::sk_to_bson) all fell back to f64 on Decimal128 parse failure. f64 has ~15 digits of precision, so values in the 35-38 digit range were silently truncated, breaking numeric ordering guarantees on sort keys (e.g. Query with ScanIndexForward could return items in an order that disagrees with the callers numeric interpretation). Reject values that exceed Decimal128 precision at all three sites with a ValidationException explaining the limit. Document as a MongoDB-backend-specific behavioral difference in docs/differences-from-dynamodb.md. Numbers in non-key attribute positions are unaffected: item_data stores the DynamoDB number string verbatim inside the {"N": ...} tag and is never numerically compared by the backend. Adds unit tests for the write path and pk_filter path at the 34-digit boundary and beyond. --- crates/storage-mongodb/src/data/mod.rs | 120 +++++++++++++++++----- crates/storage-mongodb/src/data_engine.rs | 20 ++-- docs/differences-from-dynamodb.md | 3 +- 3 files changed, 104 insertions(+), 39 deletions(-) diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index 9f8ee45c..c6ec6a6c 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -60,22 +60,18 @@ pub fn item_to_document( } ScalarAttributeType::N => { if let AttributeValue::N(n) = sk_value { - // Store as Decimal128 for proper numeric ordering - match n.parse::() { - Ok(d) => { - doc.insert("sk_n", d); - } - Err(_) => { - // Fallback: try parsing as f64 - if let Ok(f) = n.parse::() { - doc.insert("sk_n", f); - } else { - return Err(StorageError::Internal(format!( - "Cannot convert sort key '{n}' to numeric BSON type" - ))); - } - } - } + // Store as Decimal128 for correct numeric ordering. + // Values that exceed Decimal128's 34 significant digits (or + // any parse failure) are rejected rather than downcasting to + // f64, which would silently lose precision and can produce + // incorrect ordering. DynamoDB supports up to 38 digits; this + // limitation is documented in docs/differences-from-dynamodb.md. + let d = n.parse::().map_err(|_| { + StorageError::Validation(format!( + "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + })?; + doc.insert("sk_n", d); } } ScalarAttributeType::B => { @@ -151,16 +147,12 @@ pub fn pk_filter( } ScalarAttributeType::N => { if let AttributeValue::N(n) = sk_value { - match n.parse::() { - Ok(d) => { - filter.insert("sk_n", d); - } - Err(_) => { - if let Ok(f) = n.parse::() { - filter.insert("sk_n", f); - } - } - } + let d = n.parse::().map_err(|_| { + StorageError::Validation(format!( + "Numeric key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + })?; + filter.insert("sk_n", d); } } ScalarAttributeType::B => { @@ -191,3 +183,79 @@ pub fn sk_field_name( ScalarAttributeType::B => "sk_b", }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn schema_pk_str_sk_num() -> (Vec, Vec) { + ( + vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ], + vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ], + ) + } + + #[test] + fn item_to_document_rejects_numeric_sort_key_exceeding_decimal128() { + let (schema, attrs) = schema_pk_str_sk_num(); + // 35 significant digits — exceeds Decimal128's 34-digit precision. + let over_precision = "1".to_owned() + &"2".repeat(34); + assert_eq!(over_precision.chars().filter(|c| c.is_ascii_digit()).count(), 35); + + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); + item.insert("sk".to_owned(), AttributeValue::N(over_precision.clone())); + + let err = item_to_document(&item, &schema, &attrs).unwrap_err(); + match err { + StorageError::Validation(msg) => { + assert!(msg.contains(&over_precision)); + assert!(msg.contains("Decimal128")); + } + other => panic!("expected Validation error, got {other:?}"), + } + } + + #[test] + fn item_to_document_accepts_numeric_sort_key_at_decimal128_boundary() { + let (schema, attrs) = schema_pk_str_sk_num(); + // 34 significant digits — at the Decimal128 boundary. + let at_boundary = "1".repeat(34); + + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); + item.insert("sk".to_owned(), AttributeValue::N(at_boundary)); + + assert!(item_to_document(&item, &schema, &attrs).is_ok()); + } + + #[test] + fn pk_filter_rejects_numeric_sort_key_exceeding_decimal128() { + let (schema, attrs) = schema_pk_str_sk_num(); + let over_precision = "1".to_owned() + &"2".repeat(34); + let mut key = Item::new(); + key.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); + key.insert("sk".to_owned(), AttributeValue::N(over_precision)); + + let err = pk_filter(&key, &schema, &attrs).unwrap_err(); + assert!(matches!(err, StorageError::Validation(_))); + } +} diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index fdf9b7a2..f2a323ce 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -2130,18 +2130,14 @@ fn sk_to_bson( ) -> Result { match (sk_type, av) { (ScalarAttributeType::S, AttributeValue::S(s)) => Ok(bson::Bson::String(s.clone())), - (ScalarAttributeType::N, AttributeValue::N(n)) => match n.parse::() { - Ok(d) => Ok(bson::Bson::Decimal128(d)), - Err(_) => { - if let Ok(f) = n.parse::() { - Ok(bson::Bson::Double(f)) - } else { - Err(StorageError::Internal(format!( - "cannot parse numeric sort key: {n}" - ))) - } - } - }, + (ScalarAttributeType::N, AttributeValue::N(n)) => n + .parse::() + .map(bson::Bson::Decimal128) + .map_err(|_| { + StorageError::Validation(format!( + "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + }), (ScalarAttributeType::B, AttributeValue::B(b)) => Ok(bson::Bson::Binary(bson::Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: b.clone(), diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 045484a6..f92f4222 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -8,10 +8,11 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| -| Storage backend | Proprietary distributed storage | PostgreSQL | +| Storage backend | Proprietary distributed storage | PostgreSQL (default) or MongoDB (feature flag) | | Global Tables | CreateGlobalTable, replication | Not implemented (returns UnknownOperationException) | | DAX (Accelerator) | In-memory caching layer | Not applicable | | PartiQL | ExecuteStatement, BatchExecuteStatement | Not implemented (returns UnknownOperationException) | +| Numeric precision on partition/sort keys (MongoDB backend only) | 38 significant digits | 34 significant digits (BSON Decimal128). Values that exceed this precision are rejected at write and query time with a ValidationException rather than silently downcast. PostgreSQL backend supports the full 38 digits. | ## Authentication and Authorization (AWS IAM/STS auth surface used by DynamoDB) From 3d172ea10f7526dc669ee7f677b5e59d557c9669 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 17:09:28 -0700 Subject: [PATCH 09/83] chore(mongodb): rustfmt after contract fixes Long function bodies and lint-boundary formatting picked up by cargo fmt after the preceding four fix commits. No behavior change. --- crates/storage-mongodb/src/data/mod.rs | 8 +++++++- crates/storage-mongodb/src/data_engine.rs | 21 ++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index c6ec6a6c..5d8d0750 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -218,7 +218,13 @@ mod tests { let (schema, attrs) = schema_pk_str_sk_num(); // 35 significant digits — exceeds Decimal128's 34-digit precision. let over_precision = "1".to_owned() + &"2".repeat(34); - assert_eq!(over_precision.chars().filter(|c| c.is_ascii_digit()).count(), 35); + assert_eq!( + over_precision + .chars() + .filter(|c| c.is_ascii_digit()) + .count(), + 35 + ); let mut item = Item::new(); item.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index f2a323ce..5ccb53ca 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1686,8 +1686,8 @@ impl MongoEngine { None }; let for_eval = existing_item.clone().unwrap_or_default(); - let passed = expression::evaluate_condition(cond, &for_eval, maps) - .map_err(|e| { + let passed = + expression::evaluate_condition(cond, &for_eval, maps).map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error( e.to_string(), )) @@ -1755,8 +1755,8 @@ impl MongoEngine { None }; let for_eval = existing_item.clone().unwrap_or_default(); - let passed = expression::evaluate_condition(cond, &for_eval, maps) - .map_err(|e| { + let passed = + expression::evaluate_condition(cond, &for_eval, maps).map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error( e.to_string(), )) @@ -1818,7 +1818,11 @@ impl MongoEngine { if let Some(cond) = condition { let empty = std::collections::BTreeMap::new(); - let condition_item = if existing_item.is_some() { &item } else { &empty }; + let condition_item = if existing_item.is_some() { + &item + } else { + &empty + }; let passed = expression::evaluate_condition(cond, condition_item, maps) .map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error( @@ -1889,8 +1893,8 @@ impl MongoEngine { }; let for_eval = existing_item.clone().unwrap_or_default(); - let passed = expression::evaluate_condition(condition, &for_eval, maps) - .map_err(|e| { + let passed = + expression::evaluate_condition(condition, &for_eval, maps).map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; if !passed { @@ -2267,8 +2271,7 @@ mod tests { fn ccf_return_item_all_old_with_existing() { let mut item = Item::new(); item.insert("a".to_string(), AttributeValue::S("1".to_string())); - let returned = - ccf_return_item(ReturnValuesOnConditionCheckFailure::AllOld, Some(&item)); + let returned = ccf_return_item(ReturnValuesOnConditionCheckFailure::AllOld, Some(&item)); assert_eq!(returned, Some(item)); } From 3ba36a6ade0301fbbf25f97ab240f545caebfc1d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:44:32 -0700 Subject: [PATCH 10/83] chore(mongodb): resolve clippy collapsible_match warning in data_engine.rs Nested `if let Some(ref sk_cond) = key_condition.sk_condition` around `if let SortKeyCondition::BeginsWith { .. } = sk_cond` collapsed into a single pattern. Behavior identical. --- crates/storage-mongodb/src/data_engine.rs | 36 +++++++++++------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 5ccb53ca..1fb522b5 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -905,25 +905,23 @@ impl MongoEngine { // Post-fetch filtering for binary begins_with (BSON Binary comparison // sorts by length first, making $gte/$lt unreliable for prefix matching). - if let Some(ref sk_cond) = key_condition.sk_condition { - if let SortKeyCondition::BeginsWith { prefix, .. } = sk_cond { - let prefix_av = resolve_key_expr(prefix, maps)?; - if let AttributeValue::B(ref prefix_bytes) = prefix_av { - if let Some((sk_name, _)) = - sk_info(&effective_key_schema, &key_info.attribute_definitions) - { - items.retain(|item| { - item.get(sk_name) - .and_then(|v| { - if let AttributeValue::B(b) = v { - Some(b.starts_with(prefix_bytes)) - } else { - None - } - }) - .unwrap_or(false) - }); - } + if let Some(SortKeyCondition::BeginsWith { prefix, .. }) = &key_condition.sk_condition { + let prefix_av = resolve_key_expr(prefix, maps)?; + if let AttributeValue::B(ref prefix_bytes) = prefix_av { + if let Some((sk_name, _)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + items.retain(|item| { + item.get(sk_name) + .and_then(|v| { + if let AttributeValue::B(b) = v { + Some(b.starts_with(prefix_bytes)) + } else { + None + } + }) + .unwrap_or(false) + }); } } } From 164851399313c6fd33449f77278f9efffd2a9f6d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 12:19:27 -0700 Subject: [PATCH 11/83] chore(mongodb): collapse nested if/if let per clippy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream pinned a stricter Rust toolchain in 6c59a25, whose clippy is stricter about the collapsible_if and collapsible_match lints. 16 sites in the original mongo backend contribution now trip these lints: - authorization_store.rs (1 site) - data_engine.rs (11 sites) - metadata_engine.rs (4 sites) Mechanical fix — every site is 'if outer { if inner { ... } }' collapsed to 'if outer && inner { ... }', or the equivalent 'if let' pattern. Applied via 'cargo clippy --fix -p extenddb-storage-mongodb', followed by 'cargo fmt --all' to re-align the resulting blocks. Behavior unchanged. 24 unit tests still pass. --- .../src/authorization_store.rs | 30 +-- crates/storage-mongodb/src/data_engine.rs | 230 +++++++++--------- crates/storage-mongodb/src/metadata_engine.rs | 20 +- 3 files changed, 135 insertions(+), 145 deletions(-) diff --git a/crates/storage-mongodb/src/authorization_store.rs b/crates/storage-mongodb/src/authorization_store.rs index 415249d9..412c6f65 100644 --- a/crates/storage-mongodb/src/authorization_store.rs +++ b/crates/storage-mongodb/src/authorization_store.rs @@ -240,22 +240,22 @@ impl AuthorizationStore for MongoCatalogStore { }); let mut session_tags = Vec::new(); - if let Some(tags_bson) = session_doc.get("session_tags") { - if let Ok(tags_val) = bson::from_bson::(tags_bson.clone()) { - if let Some(arr) = tags_val.as_array() { - for tag in arr { - if let (Some(k), Some(v)) = ( - tag.get("Key").and_then(|k| k.as_str()), - tag.get("Value").and_then(|v| v.as_str()), - ) { - session_tags.push((k.to_owned(), v.to_owned())); - } + if let Some(tags_bson) = session_doc.get("session_tags") + && let Ok(tags_val) = bson::from_bson::(tags_bson.clone()) + { + if let Some(arr) = tags_val.as_array() { + for tag in arr { + if let (Some(k), Some(v)) = ( + tag.get("Key").and_then(|k| k.as_str()), + tag.get("Value").and_then(|v| v.as_str()), + ) { + session_tags.push((k.to_owned(), v.to_owned())); } - } else if let Some(obj) = tags_val.as_object() { - for (k, v) in obj { - if let Some(v_str) = v.as_str() { - session_tags.push((k.clone(), v_str.to_owned())); - } + } + } else if let Some(obj) = tags_val.as_object() { + for (k, v) in obj { + if let Some(v_str) = v.as_str() { + session_tags.push((k.clone(), v_str.to_owned())); } } } diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 1fb522b5..ba1bc0af 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -600,32 +600,34 @@ impl MongoEngine { // Fast path: use native MongoDB atomic operators when possible. // This avoids transactions and retries for simple unconditional updates. - if condition.is_none() && !return_old && stream.is_none() { - if let Some(mongo_update) = self.try_build_native_update(actions, maps) { - let opts = mongodb::options::FindOneAndUpdateOptions::builder() - .upsert(true) - .return_document(ReturnDocument::After) - .build(); - let result_doc = coll - .find_one_and_update(key_filter, mongo_update) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let new_item = if return_new { - result_doc.as_ref().map(document_to_item).transpose()? - } else { - None - }; + if condition.is_none() + && !return_old + && stream.is_none() + && let Some(mongo_update) = self.try_build_native_update(actions, maps) + { + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(ReturnDocument::After) + .build(); + let result_doc = coll + .find_one_and_update(key_filter, mongo_update) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; - // Sync GSI (non-transactional but data write is atomic) - if let Some(ref doc) = result_doc { - let item = document_to_item(doc)?; - self.sync_indexes(key_info, None, Some(&item)).await?; - } + let new_item = if return_new { + result_doc.as_ref().map(document_to_item).transpose()? + } else { + None + }; - return Ok((None, new_item)); + // Sync GSI (non-transactional but data write is atomic) + if let Some(ref doc) = result_doc { + let item = document_to_item(doc)?; + self.sync_indexes(key_info, None, Some(&item)).await?; } + + return Ok((None, new_item)); } let mut session = self @@ -832,30 +834,29 @@ impl MongoEngine { let sk_field = sk_field_name(&effective_key_schema, &key_info.attribute_definitions); // Apply sort key condition - if let Some(ref sk_cond) = key_condition.sk_condition { - if let Some(sk_f) = sk_field { - let sk_filter = build_sk_filter(sk_cond, sk_f, maps)?; - for (k, v) in sk_filter { - filter.insert(k, v); - } + if let Some(ref sk_cond) = key_condition.sk_condition + && let Some(sk_f) = sk_field + { + let sk_filter = build_sk_filter(sk_cond, sk_f, maps)?; + for (k, v) in sk_filter { + filter.insert(k, v); } } // Apply exclusive_start_key pagination - if let Some(start_key) = exclusive_start_key { - if let Some(sk_f) = sk_field { - // Get the sort key value from the start key - if let Some((sk_name, sk_type)) = - sk_info(&effective_key_schema, &key_info.attribute_definitions) - { - if let Some(sk_val) = start_key.get(sk_name) { - let sk_bson = sk_to_bson(sk_val, sk_type)?; - if forward { - filter.insert(sk_f, doc! { "$gt": sk_bson }); - } else { - filter.insert(sk_f, doc! { "$lt": sk_bson }); - } - } + if let Some(start_key) = exclusive_start_key + && let Some(sk_f) = sk_field + { + // Get the sort key value from the start key + if let Some((sk_name, sk_type)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + && let Some(sk_val) = start_key.get(sk_name) + { + let sk_bson = sk_to_bson(sk_val, sk_type)?; + if forward { + filter.insert(sk_f, doc! { "$gt": sk_bson }); + } else { + filter.insert(sk_f, doc! { "$lt": sk_bson }); } } } @@ -907,22 +908,21 @@ impl MongoEngine { // sorts by length first, making $gte/$lt unreliable for prefix matching). if let Some(SortKeyCondition::BeginsWith { prefix, .. }) = &key_condition.sk_condition { let prefix_av = resolve_key_expr(prefix, maps)?; - if let AttributeValue::B(ref prefix_bytes) = prefix_av { - if let Some((sk_name, _)) = + if let AttributeValue::B(ref prefix_bytes) = prefix_av + && let Some((sk_name, _)) = sk_info(&effective_key_schema, &key_info.attribute_definitions) - { - items.retain(|item| { - item.get(sk_name) - .and_then(|v| { - if let AttributeValue::B(b) = v { - Some(b.starts_with(prefix_bytes)) - } else { - None - } - }) - .unwrap_or(false) - }); - } + { + items.retain(|item| { + item.get(sk_name) + .and_then(|v| { + if let AttributeValue::B(b) = v { + Some(b.starts_with(prefix_bytes)) + } else { + None + } + }) + .unwrap_or(false) + }); } } @@ -1185,10 +1185,10 @@ impl MongoEngine { use futures::TryStreamExt; // Fast path: skip catalog query if we know this table has no GSIs - if let Some(entry) = self.gsi_cache.get(&key_info.table_id) { - if !*entry { - return Ok(()); - } + if let Some(entry) = self.gsi_cache.get(&key_info.table_id) + && !*entry + { + return Ok(()); } let indexes_coll = self.catalog_db.collection::("indexes"); @@ -1227,35 +1227,31 @@ impl MongoEngine { let idx_coll = self.data_db.collection::(&idx_coll_name); // Delete old index entry - if let Some(old) = old_item { - if item_has_index_keys(old, &idx_key_schema) { - let old_filter = - pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; - let _ = idx_coll.delete_one(old_filter).await; - } + if let Some(old) = old_item + && item_has_index_keys(old, &idx_key_schema) + { + let old_filter = pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; + let _ = idx_coll.delete_one(old_filter).await; } // Insert new index entry - if let Some(new) = new_item { - if item_has_index_keys(new, &idx_key_schema) { - let projected = - project_item(new, &idx_key_schema, &key_info.key_schema, &projection); - let idx_doc = item_to_document( - &projected, - &idx_key_schema, - &key_info.attribute_definitions, - )?; - let filter = - pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; - let opts = mongodb::options::ReplaceOptions::builder() - .upsert(true) - .build(); - idx_coll - .replace_one(filter, idx_doc) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } + if let Some(new) = new_item + && item_has_index_keys(new, &idx_key_schema) + { + let projected = + project_item(new, &idx_key_schema, &key_info.key_schema, &projection); + let idx_doc = + item_to_document(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let filter = + pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; } } @@ -1272,10 +1268,10 @@ impl MongoEngine { ) -> Result<(), StorageError> { use futures::TryStreamExt; - if let Some(entry) = self.gsi_cache.get(&key_info.table_id) { - if !*entry { - return Ok(()); - } + if let Some(entry) = self.gsi_cache.get(&key_info.table_id) + && !*entry + { + return Ok(()); } let indexes_coll = self.catalog_db.collection::("indexes"); @@ -1315,35 +1311,31 @@ impl MongoEngine { let idx_coll_name = data_collection_name(&index_id); let idx_coll = self.data_db.collection::(&idx_coll_name); - if let Some(old) = old_item { - if item_has_index_keys(old, &idx_key_schema) { - let old_filter = - pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; - let _ = idx_coll.delete_one(old_filter).session(&mut *session).await; - } + if let Some(old) = old_item + && item_has_index_keys(old, &idx_key_schema) + { + let old_filter = pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; + let _ = idx_coll.delete_one(old_filter).session(&mut *session).await; } - if let Some(new) = new_item { - if item_has_index_keys(new, &idx_key_schema) { - let projected = - project_item(new, &idx_key_schema, &key_info.key_schema, &projection); - let idx_doc = item_to_document( - &projected, - &idx_key_schema, - &key_info.attribute_definitions, - )?; - let filter = - pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; - let opts = mongodb::options::ReplaceOptions::builder() - .upsert(true) - .build(); - idx_coll - .replace_one(filter, idx_doc) - .with_options(opts) - .session(&mut *session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } + if let Some(new) = new_item + && item_has_index_keys(new, &idx_key_schema) + { + let projected = + project_item(new, &idx_key_schema, &key_info.key_schema, &projection); + let idx_doc = + item_to_document(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let filter = + pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; } } diff --git a/crates/storage-mongodb/src/metadata_engine.rs b/crates/storage-mongodb/src/metadata_engine.rs index 72f021ee..d3606cbe 100644 --- a/crates/storage-mongodb/src/metadata_engine.rs +++ b/crates/storage-mongodb/src/metadata_engine.rs @@ -421,17 +421,15 @@ impl MetadataEngine for MongoEngine { break; } // Parse the TTL value and check if expired - if let Ok(item_data) = doc.get_document("item_data") { - if let Ok(ttl_obj) = item_data.get_document(&ttl_attribute) { - if let Ok(n_str) = ttl_obj.get_str("N") { - if let Ok(ttl_val) = n_str.parse::() { - if ttl_val >= 1 && ttl_val <= now_epoch { - let item = document_to_item(&doc)?; - items.push(item); - } - } - } - } + if let Ok(item_data) = doc.get_document("item_data") + && let Ok(ttl_obj) = item_data.get_document(&ttl_attribute) + && let Ok(n_str) = ttl_obj.get_str("N") + && let Ok(ttl_val) = n_str.parse::() + && ttl_val >= 1 + && ttl_val <= now_epoch + { + let item = document_to_item(&doc)?; + items.push(item); } } Ok(items) From 7dcfd4917e66adfd852e2f8ad4d02c3353c6d197 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:46:41 -0700 Subject: [PATCH 12/83] fix(mongodb): per-shard sequence number counter stream_engine.rs::next_sequence_number used a single global counter document (_id: "stream_seq") for every shard of every table. The shard_id argument was accepted but ignored. latest_sequence_number filters by shard_id when reading, so writer and reader disagreed on the sequence-number space: a write to shard B advanced the counter shard A reads back, producing non-contiguous sequence numbers on shard A's GetRecords pages. DynamoDB Streams' contract is that sequence numbers are strictly monotonic within a shard and independent across shards. Key the counter document by shard_id (stream_seq:) so each shard gets its own atomic counter. The 21-digit zero-padded encoding of the sequence number is preserved. --- crates/storage-mongodb/src/stream_engine.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index ed62080c..cb9b51c2 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -439,17 +439,26 @@ impl StreamEngine for MongoEngine { }) } - fn next_sequence_number(&self, _shard_id: &str) -> BoxFuture<'_, Result> { + fn next_sequence_number(&self, shard_id: &str) -> BoxFuture<'_, Result> { + let shard_id = shard_id.to_owned(); Box::pin(async move { - // Use atomic findAndModify on a sequence counter document + // Per-shard atomic counter. DynamoDB Streams' contract is that + // sequence numbers are strictly monotonic *within a shard* and + // independent *across shards*. A single global counter would + // couple the sequence spaces of unrelated shards — a writer + // pushing records into shard B would advance the counter shard A + // reads back, producing non-contiguous sequence numbers on + // shard A's GetRecords pages. Keying the counter document by + // shard_id preserves the per-shard monotonicity guarantee. let counters_coll = self.data_db.collection::("counters"); let opts = mongodb::options::FindOneAndUpdateOptions::builder() .upsert(true) .return_document(mongodb::options::ReturnDocument::After) .build(); + let counter_id = format!("stream_seq:{shard_id}"); let doc = counters_coll .find_one_and_update( - doc! { "_id": "stream_seq" }, + doc! { "_id": counter_id }, doc! { "$inc": { "value": 1_i64 } }, ) .with_options(opts) From 50fa8ee074fdcc4eeb08c771274c10ec9c546b7d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:46:41 -0700 Subject: [PATCH 13/83] fix(mongodb): TTL-invalidate GSI cache entries for multi-instance safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MongoEngine::gsi_cache is a per-process DashMap. In a multi-instance deployment sharing a MongoDB catalog, an admin adding a GSI via instance A does not invalidate instance B's cache. Instance B continues to see the cached false and silently skips index updates on subsequent writes — the base table and the newly-added GSI drift out of sync until the cache is reset by a restart. Change the cache value from bool to (bool, Instant) and reject entries older than GSI_CACHE_TTL (60 seconds) on read. When a stale entry is observed, the caller falls through to the catalog query, which returns the authoritative state and refreshes the cache. Introduce three MongoEngine helpers (gsi_cache_get_fresh, gsi_cache_set, gsi_cache_invalidate) so callers don't reach into the DashMap directly. Update the five existing call sites (data_engine.rs sync_indexes and sync_indexes_in_session; table_engine.rs DropTable, CreateGSI, DeleteGSI) to use them. --- crates/storage-mongodb/src/data_engine.rs | 16 ++++---- crates/storage-mongodb/src/lib.rs | 45 ++++++++++++++++++++-- crates/storage-mongodb/src/table_engine.rs | 6 +-- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index ba1bc0af..d5b7f087 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1184,10 +1184,10 @@ impl MongoEngine { ) -> Result<(), StorageError> { use futures::TryStreamExt; - // Fast path: skip catalog query if we know this table has no GSIs - if let Some(entry) = self.gsi_cache.get(&key_info.table_id) - && !*entry - { + // Fast path: skip catalog query if we know this table has no GSIs. + // The cache entry is valid for GSI_CACHE_TTL, giving eventual + // convergence when a GSI is added on another ExtendDB instance. + if let Some(false) = self.gsi_cache_get_fresh(&key_info.table_id) { return Ok(()); } @@ -1255,7 +1255,7 @@ impl MongoEngine { } } - self.gsi_cache.insert(key_info.table_id.clone(), found_any); + self.gsi_cache_set(&key_info.table_id, found_any); Ok(()) } @@ -1268,9 +1268,7 @@ impl MongoEngine { ) -> Result<(), StorageError> { use futures::TryStreamExt; - if let Some(entry) = self.gsi_cache.get(&key_info.table_id) - && !*entry - { + if let Some(false) = self.gsi_cache_get_fresh(&key_info.table_id) { return Ok(()); } @@ -1339,7 +1337,7 @@ impl MongoEngine { } } - self.gsi_cache.insert(key_info.table_id.clone(), found_any); + self.gsi_cache_set(&key_info.table_id, found_any); Ok(()) } diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 40ee734b..45b36898 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -215,6 +215,16 @@ inventory::submit! { // MongoEngine // ============================================================================ +/// TTL for entries in [`MongoEngine::gsi_cache`]. +/// +/// The GSI cache is per-process. When multiple ExtendDB instances share a +/// catalog, an admin creating or dropping a GSI on instance A does not +/// invalidate instance B's cache. Bounding cache entries by wall-clock age +/// gives eventual convergence at a small cost (one catalog `find` per table +/// per TTL window), which is far cheaper than the cost of silently skipping +/// index updates on tables where GSIs were added out-of-band. +const GSI_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + /// `MongoDB` storage backend. pub struct MongoEngine { client: mongodb::Client, @@ -222,9 +232,12 @@ pub struct MongoEngine { data_db: mongodb::Database, region: String, max_connections: u32, - /// Cache of `table_id` -> `has_gsi`. Avoids catalog queries on every write - /// for tables with no GSIs. - gsi_cache: dashmap::DashMap, + /// Cache of `table_id` -> (`has_gsi`, insertion time). Avoids catalog + /// queries on every write for tables with no GSIs. Entries older than + /// [`GSI_CACHE_TTL`] are treated as misses and re-read from the catalog, + /// so GSI additions/removals on other ExtendDB instances converge within + /// the TTL window. + gsi_cache: dashmap::DashMap, } impl MongoEngine { @@ -254,6 +267,32 @@ impl MongoEngine { }) } + /// Look up a fresh GSI-cache entry for `table_id`. + /// + /// Returns `Some(has_gsi)` when a cache entry exists and is younger than + /// [`GSI_CACHE_TTL`], `None` otherwise (either no entry or expired). + /// Callers that get `None` must fall back to reading the catalog. + pub(crate) fn gsi_cache_get_fresh(&self, table_id: &str) -> Option { + let entry = self.gsi_cache.get(table_id)?; + let (has_gsi, inserted) = *entry; + if inserted.elapsed() <= GSI_CACHE_TTL { + Some(has_gsi) + } else { + None + } + } + + /// Record a fresh GSI-cache observation for `table_id`. + pub(crate) fn gsi_cache_set(&self, table_id: &str, has_gsi: bool) { + self.gsi_cache + .insert(table_id.to_owned(), (has_gsi, std::time::Instant::now())); + } + + /// Remove a GSI-cache entry (e.g., on GSI drop or table delete). + pub(crate) fn gsi_cache_invalidate(&self, table_id: &str) { + self.gsi_cache.remove(table_id); + } + /// Validate `account_id` against injection attacks. fn validate_account_id(account_id: &str) -> Result<(), StorageError> { if account_id.contains('$') diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 1e48b36e..633037ce 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -455,7 +455,7 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - self.gsi_cache.remove(&desc.table_id); + self.gsi_cache_invalidate(&desc.table_id); // Delete the table metadata tables_coll @@ -650,7 +650,7 @@ impl MongoEngine { } })?; - self.gsi_cache.insert(desc.table_id.clone(), true); + self.gsi_cache_set(&desc.table_id, true); } if let Some(delete) = &update.delete { @@ -667,7 +667,7 @@ impl MongoEngine { } // Invalidate cache — may still have other GSIs - self.gsi_cache.remove(&desc.table_id); + self.gsi_cache_invalidate(&desc.table_id); } } } From ea8de08de2c8dc0732f5643cf41dbae123bad2a4 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 19:51:26 -0700 Subject: [PATCH 14/83] fix(mongodb): query tables collection by _id compound key in backup engine The `tables` catalog collection uses a compound `_id`: `{ account_id, table_name }`. Four query sites in backup_engine.rs used flat `account_id` / `table_name` fields at the top level, which never match the actual document shape: - create_backup: `find_one` for the source table - restore_table_from_backup: `update_one` to mark the restored table ACTIVE and set item_count - describe_continuous_backups: `find_one` to check table exists - update_continuous_backups: `find_one` to check table exists The queries silently return no matches. create_backup returns ResourceNotFoundException even when the table exists; restore_table_from_backup leaves the restored table stuck in CREATING with item_count=0; describe/update_continuous_backups return TableNotFound for every table. Fix by using the correct `_id` shape. The `continuous_backups` collection uses ObjectId-based `_id` with `account_id` and `table_name` as flat fields, so those queries (lines 490 and 542) are correct as-is and were not changed. No changes to the `tables` collection schema or to code outside backup_engine.rs. Reproduced with a scripted CreateTable + CreateBackup and verified the fix restores backup creation. --- crates/storage-mongodb/src/backup_engine.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index 05cdc1b6..bf4e6268 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -52,8 +52,7 @@ impl BackupEngine for MongoEngine { let tables_coll = self.catalog_db.collection::("tables"); let table_doc = tables_coll .find_one(doc! { - "account_id": &account_id, - "table_name": &table_name, + "_id": { "account_id": &account_id, "table_name": &table_name }, "table_status": "ACTIVE", }) .await @@ -458,7 +457,7 @@ impl BackupEngine for MongoEngine { let tables_coll = self.catalog_db.collection::("tables"); tables_coll .update_one( - doc! { "account_id": &account_id, "table_name": &target_table_name }, + doc! { "_id": { "account_id": &account_id, "table_name": &target_table_name } }, doc! { "$set": { "item_count": item_count, "table_status": "ACTIVE" } }, ) .await @@ -478,7 +477,7 @@ impl BackupEngine for MongoEngine { Box::pin(async move { let tables_coll = self.catalog_db.collection::("tables"); let exists = tables_coll - .find_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -529,7 +528,7 @@ impl BackupEngine for MongoEngine { Box::pin(async move { let tables_coll = self.catalog_db.collection::("tables"); let exists = tables_coll - .find_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) .await .map_err(|e| StorageError::Internal(e.to_string()))?; From 3523b3f045e4d45799dfe447365f8881fd7dce9d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:46:42 -0700 Subject: [PATCH 15/83] refactor(mongodb): use server-side $out for backup and restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-item cursor-and-insert loop in create_backup and restore_table_from_backup with MongoDB's aggregation $out stage. Item data now flows directly between collections in the server; the driver-side traffic drops from one round-trip per item to a fixed overhead for the aggregation and a subsequent count_documents. The schema for backups changes from a shared backup_items collection keyed by backup_arn to one collection per backup named _backup_ where backup_id is a UUID stored in the backup metadata. Item documents in the backup collection are exact copies of the source data-collection documents — no transformation. Impact: - CreateBackup: previously O(N) driver round-trips for N items; now one aggregate call plus one count_documents. - RestoreTableFromBackup: same reduction. - DeleteBackup: previously delete_many with a filter over N shared items; now a single drop_collection. - List/Describe: unchanged; both operate on the backups metadata collection. Old backups (per-item docs in backup_items) are no longer restorable. If backward compatibility becomes necessary, a compatibility read path can check for backup_id on the metadata: absent means old-scheme, fall back to the previous cursor loop. Each active backup adds one WiredTiger file to MongoDB; docs/local-mongodb-setup.md covers the ulimit -n requirement. --- crates/storage-mongodb/src/backup_engine.rs | 173 ++++++++++---------- 1 file changed, 90 insertions(+), 83 deletions(-) diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index bf4e6268..c75f22f7 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -3,10 +3,16 @@ //! `BackupEngine` implementation for `MongoDB`. //! -//! Backups are stored as documents in a `backups` collection (metadata) and -//! a `backup_items` collection (snapshotted items). Uses `$out`-style cloning -//! approach: read all items from the data collection and bulk-insert into -//! the backup items collection tagged with `backup_arn`. +//! Backups are stored as one MongoDB collection per backup, plus a `backups` +//! metadata collection in the catalog. `CreateBackup` uses MongoDB's +//! server-side aggregation `$out` stage to clone the source data collection +//! into `_backup_` in the data database — no per-item traffic +//! between the driver and the server. `RestoreTableFromBackup` uses the same +//! stage in reverse. `DeleteBackup` drops the collection. +//! +//! Backup metadata carries a `backup_id` UUID; the collection name is derived +//! from that id so the `backup_arn` (which contains slashes and colons) never +//! appears in a collection name. use futures::TryStreamExt; use futures::future::BoxFuture; @@ -30,6 +36,15 @@ fn epoch_millis() -> u128 { .as_millis() } +/// Return the MongoDB collection name that holds items for a given backup. +/// +/// The collection lives in the data database. The name is derived from the +/// backup's UUID so it is safe for MongoDB (no colons, slashes, or dots) and +/// bounded in length regardless of how long the source `backup_arn` is. +fn backup_collection_name(backup_id: &str) -> String { + format!("_backup_{backup_id}") +} + #[allow(clippy::cast_precision_loss)] fn now_epoch_secs() -> f64 { std::time::SystemTime::now() @@ -87,53 +102,45 @@ impl BackupEngine for MongoEngine { region = self.region, ts = epoch_millis() ); - - // Snapshot items from the data collection - let coll_name = data_collection_name(&table_id); - let data_coll = self.data_db.collection::(&coll_name); - - let mut cursor = data_coll - .find(doc! {}) + let backup_id = uuid::Uuid::new_v4().to_string(); + + // Snapshot items from the data collection using a server-side + // `$out` aggregation. Items are copied directly between + // collections in MongoDB — no per-item traffic to the driver. + let src_coll_name = data_collection_name(&table_id); + let dst_coll_name = backup_collection_name(&backup_id); + let data_coll = self.data_db.collection::(&src_coll_name); + + let pipeline = vec![doc! { "$out": &dst_coll_name }]; + let out_cursor = data_coll + .aggregate(pipeline) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // `$out` writes to the target collection and returns an empty + // cursor; consume it to ensure the stage has fully completed + // before we count. + let _drained: Vec = out_cursor + .try_collect() .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let backup_items_coll = self.catalog_db.collection::("backup_items"); - let mut actual_count: i64 = 0; - - while let Some(item_doc) = cursor - .try_next() + let dst_coll = self.data_db.collection::(&dst_coll_name); + let actual_count = dst_coll + .count_documents(doc! {}) .await .map_err(|e| StorageError::Internal(e.to_string()))? - { - let mut backup_doc = Document::new(); - backup_doc.insert("backup_arn", &backup_arn); - backup_doc.insert( - "item_data", - item_doc - .get("item_data") - .cloned() - .unwrap_or(mongodb::bson::Bson::Null), - ); - backup_doc.insert("pk", item_doc.get_str("pk").unwrap_or_default()); - if let Ok(sk) = item_doc.get_str("sk_s") { - backup_doc.insert("sk", sk); - } else if let Some(sk_n) = item_doc.get("sk_n") { - backup_doc.insert("sk_n", sk_n.clone()); - } - - backup_items_coll - .insert_one(backup_doc) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - actual_count += 1; - } + as i64; let created_at = now_epoch_secs(); - // Store backup metadata + // Store backup metadata. `backup_id` is what maps to the + // physical collection; `backup_arn` remains the caller-visible + // handle and stays the `_id` for compatibility with existing + // describe/list callers. let backups_coll = self.catalog_db.collection::("backups"); let backup_meta = doc! { "_id": &backup_arn, + "backup_id": &backup_id, "backup_name": &backup_name, "backup_status": "AVAILABLE", "backup_type": "USER", @@ -314,15 +321,28 @@ impl BackupEngine for MongoEngine { Box::pin(async move { let desc = self.describe_backup(&backup_arn).await?; - // Delete backup items - let backup_items_coll = self.catalog_db.collection::("backup_items"); - backup_items_coll - .delete_many(doc! { "backup_arn": &backup_arn }) + // Look up the physical collection name from metadata. + let backups_coll = self.catalog_db.collection::("backups"); + let meta = backups_coll + .find_one(doc! { "_id": &backup_arn }) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + // Drop the backup collection. If backup_id is absent (e.g., a + // pre-`$out` backup on an old catalog) we skip — nothing to drop + // at the collection level in that case. + if let Ok(backup_id) = meta.get_str("backup_id") { + let coll_name = backup_collection_name(backup_id); + let coll = self.data_db.collection::(&coll_name); + coll.drop() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } // Mark backup as deleted - let backups_coll = self.catalog_db.collection::("backups"); backups_coll .update_one( doc! { "_id": &backup_arn }, @@ -409,49 +429,36 @@ impl BackupEngine for MongoEngine { let desc = self.create_table(&account_id, create_input).await?; - // Restore items from backup - let backup_items_coll = self.catalog_db.collection::("backup_items"); - let mut cursor = backup_items_coll - .find(doc! { "backup_arn": &backup_arn }) + // Restore items from the backup collection using server-side `$out`. + // The backup collection was written by `create_backup` in the same + // document shape as the source data collection, so this is a + // direct clone — no per-item transformation needed. + let backup_id = backup_doc + .get_str("backup_id") + .map_err(|_| { + StorageError::Internal("backup metadata missing backup_id".to_string()) + })? + .to_owned(); + let src_coll_name = backup_collection_name(&backup_id); + let src_coll = self.data_db.collection::(&src_coll_name); + let new_coll_name = data_collection_name(&desc.table_id); + + let pipeline = vec![doc! { "$out": &new_coll_name }]; + let out_cursor = src_coll + .aggregate(pipeline) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let _drained: Vec = out_cursor + .try_collect() .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let new_coll_name = data_collection_name(&desc.table_id); let new_data_coll = self.data_db.collection::(&new_coll_name); - - let mut item_count: i64 = 0; - while let Some(backup_item) = cursor - .try_next() + let item_count = new_data_coll + .count_documents(doc! {}) .await .map_err(|e| StorageError::Internal(e.to_string()))? - { - // Re-insert using the original document structure - let mut restore_doc = Document::new(); - if let Some(pk) = backup_item.get("pk") { - restore_doc.insert("pk", pk.clone()); - } - if let Some(item_data) = backup_item.get("item_data") { - restore_doc.insert("item_data", item_data.clone()); - } - if let Ok(sk) = backup_item.get_str("sk") { - restore_doc.insert("sk_s", sk); - let pk_str = backup_item.get_str("pk").unwrap_or_default(); - restore_doc.insert("_id", format!("{pk_str}#{sk}")); - } else if let Some(sk_n) = backup_item.get("sk_n") { - restore_doc.insert("sk_n", sk_n.clone()); - let pk_str = backup_item.get_str("pk").unwrap_or_default(); - restore_doc.insert("_id", format!("{pk_str}#{sk_n}")); - } else { - let pk_str = backup_item.get_str("pk").unwrap_or_default(); - restore_doc.insert("_id", pk_str); - } - - new_data_coll - .insert_one(restore_doc) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - item_count += 1; - } + as i64; // Update item count and mark table ACTIVE let tables_coll = self.catalog_db.collection::("tables"); From d67f35d4e58752c949ff3c1e4dc15e9d94aa171e Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:46:42 -0700 Subject: [PATCH 16/83] test(mongodb): pure-Rust BSON filter interpreter for pushdown parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a MongoDB filter interpreter in crates/storage-mongodb/tests/common/mod.rs that evaluates the subset of operators the condition compiler in condition.rs can emit ($eq, $ne, $lt, $lte, $gt, $gte, $exists, $in, $regex, $type, $expr, $and, $or, $nor, dotted-path traversal, implicit array-match). Not a general-purpose MongoDB query engine — the interpreter is bounded to what the compiler produces; any operator the compiler emits that this file does not handle triggers a panic rather than silently passing. Interpreter details: - Distinguishes FieldValue::Present from FieldValue::Missing so $exists: false matches truly-absent fields but not fields present with value null. - Numeric comparison via a bson_cmp helper that unifies Int32, Int64, and Double. - Regex matcher supports the two patterns the compiler emits: anchored prefix (^prefix, from begins_with) and unanchored substring (from contains on string fields). Unescapes the compiler's regex_escape output. - $expr supports only the field-vs-field shape the compiler emits. 18 self-tests exercise the interpreter's shape: scalar equality (present and absent), $exists true/false, implicit array-match on sets, lexicographic ordering, $and/$or/$nor composition, $regex prefix, $in membership, $ne, the empty-IN sentinel filter the compiler emits for empty IN() clauses, nested map paths, and missing-intermediate-path short-circuit. The wrapper file interpreter_selftests.rs is required because cargo's integration-test model treats each file under tests/ as a separate binary; common/mod.rs on its own does not get compiled without a consuming binary. --- crates/storage-mongodb/tests/common/mod.rs | 507 ++++++++++++++++++ .../tests/interpreter_selftests.rs | 13 + 2 files changed, 520 insertions(+) create mode 100644 crates/storage-mongodb/tests/common/mod.rs create mode 100644 crates/storage-mongodb/tests/interpreter_selftests.rs diff --git a/crates/storage-mongodb/tests/common/mod.rs b/crates/storage-mongodb/tests/common/mod.rs new file mode 100644 index 00000000..a5073418 --- /dev/null +++ b/crates/storage-mongodb/tests/common/mod.rs @@ -0,0 +1,507 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Pure-Rust MongoDB filter interpreter for pushdown-parity tests. +//! +//! Evaluates the subset of MongoDB filter operators emitted by +//! `crates/storage-mongodb/src/condition.rs::condition_to_filter` against a +//! BSON document. Used to differentially test the compiled-filter path +//! against the in-Rust DDB expression evaluator without needing a live +//! MongoDB. +//! +//! This is deliberately not a general-purpose MongoDB query engine — it +//! implements only the operators the compiler can emit. Adding a new +//! operator to the compiler requires adding it here. + +use bson::{Bson, Document}; + +/// Evaluate a MongoDB filter document against a BSON document. +/// +/// Returns `true` if the document matches all clauses in the filter, +/// `false` otherwise. The root document representing the filter is +/// interpreted as an implicit `$and` of its clauses (matching MongoDB's +/// top-level query semantics). +pub fn eval_filter(filter: &Document, doc: &Document) -> bool { + filter.iter().all(|(key, val)| eval_clause(key, val, doc)) +} + +/// Evaluate a single (key, value) clause at the top level of a filter. +/// +/// Handles: +/// - `$and` / `$or` / `$nor` — logical operators over an array of subfilters +/// - `$expr` — expression-based comparison, only for field-vs-field per compiler +/// - `: ` — either a scalar equality or an operator document +fn eval_clause(key: &str, val: &Bson, doc: &Document) -> bool { + match key { + "$and" => as_array(val) + .iter() + .all(|sub| as_doc(sub).is_some_and(|d| eval_filter(d, doc))), + "$or" => as_array(val) + .iter() + .any(|sub| as_doc(sub).is_some_and(|d| eval_filter(d, doc))), + "$nor" => !as_array(val) + .iter() + .any(|sub| as_doc(sub).is_some_and(|d| eval_filter(d, doc))), + "$expr" => eval_expr(val, doc), + _ => { + // : — walk the dotted path, then evaluate the + // predicate against the value found. + let field_val = walk_path(doc, key); + eval_predicate(val, &field_val) + } + } +} + +/// Evaluate a predicate against the value found at the field path. +/// +/// The predicate is either a scalar (implicit `$eq`) or an operator +/// document containing `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, +/// `$exists`, `$in`, `$regex`, or `$type`. +/// +/// If the field is a BSON array and the predicate is a scalar, MongoDB +/// matches if any element of the array equals the scalar (implicit +/// array-match). We support that shape because the compiler relies on it +/// for `contains(SS_field, :s)` and similar. +fn eval_predicate(pred: &Bson, field: &FieldValue<'_>) -> bool { + match pred { + Bson::Document(pred_doc) => { + // Operator document: every operator must match. MongoDB's + // behavior with multiple operator keys in one doc is that they + // are ANDed together. + pred_doc.iter().all(|(op, arg)| match op.as_str() { + "$eq" => eq_or_array_match(field, arg), + "$ne" => !eq_or_array_match(field, arg), + "$lt" => cmp_field(field, arg, |a, b| { + bson_cmp(a, b) == std::cmp::Ordering::Less + }), + "$lte" => cmp_field(field, arg, |a, b| { + matches!( + bson_cmp(a, b), + std::cmp::Ordering::Less | std::cmp::Ordering::Equal + ) + }), + "$gt" => cmp_field(field, arg, |a, b| { + bson_cmp(a, b) == std::cmp::Ordering::Greater + }), + "$gte" => cmp_field(field, arg, |a, b| { + matches!( + bson_cmp(a, b), + std::cmp::Ordering::Greater | std::cmp::Ordering::Equal + ) + }), + "$exists" => { + let want = as_bool(arg).unwrap_or(true); + field.is_present() == want + } + "$in" => { + let arr = as_array(arg); + match field { + FieldValue::Present(Bson::Array(field_arr)) => field_arr.iter().any(|f| { + arr.iter() + .any(|a| bson_cmp(f, a) == std::cmp::Ordering::Equal) + }), + FieldValue::Present(v) => arr + .iter() + .any(|a| bson_cmp(v, a) == std::cmp::Ordering::Equal), + FieldValue::Missing => false, + } + } + "$regex" => match arg { + Bson::String(pattern) => match field { + FieldValue::Present(Bson::String(s)) => regex_match(pattern, s), + FieldValue::Present(Bson::Array(arr)) => arr + .iter() + .any(|v| matches!(v, Bson::String(s) if regex_match(pattern, s))), + _ => false, + }, + _ => false, + }, + "$type" => match arg { + Bson::String(t) => matches!( + (t.as_str(), field), + ("null", FieldValue::Present(Bson::Null)) + | ("string", FieldValue::Present(Bson::String(_))) + | ("bool", FieldValue::Present(Bson::Boolean(_))) + | ("array", FieldValue::Present(Bson::Array(_))) + | ("object", FieldValue::Present(Bson::Document(_))) + ), + _ => false, + }, + // If the compiler ever emits an operator we don't recognize, + // fail loudly rather than silently pass. + other => panic!("bson filter interpreter: unknown operator {other}"), + }) + } + // Scalar predicate: implicit equality (or array-match). + _ => eq_or_array_match(field, pred), + } +} + +/// Field-vs-field expressions via `$expr`. The compiler only emits +/// `{"$expr": {"$op": ["$left_field", "$right_field"]}}` shapes. +fn eval_expr(val: &Bson, doc: &Document) -> bool { + let Some(expr_doc) = as_doc(val) else { + return false; + }; + for (op, args) in expr_doc { + let arr = as_array(args); + if arr.len() != 2 { + return false; + } + let (Some(lhs_ref), Some(rhs_ref)) = (as_field_ref(&arr[0]), as_field_ref(&arr[1])) else { + return false; + }; + let lhs = walk_path(doc, lhs_ref); + let rhs = walk_path(doc, rhs_ref); + let (FieldValue::Present(l), FieldValue::Present(r)) = (&lhs, &rhs) else { + return false; + }; + let ord = bson_cmp(l, r); + let matched = match op.as_str() { + "$eq" => ord == std::cmp::Ordering::Equal, + "$ne" => ord != std::cmp::Ordering::Equal, + "$lt" => ord == std::cmp::Ordering::Less, + "$lte" => matches!(ord, std::cmp::Ordering::Less | std::cmp::Ordering::Equal), + "$gt" => ord == std::cmp::Ordering::Greater, + "$gte" => matches!(ord, std::cmp::Ordering::Greater | std::cmp::Ordering::Equal), + other => panic!("bson filter interpreter: unknown $expr operator {other}"), + }; + if !matched { + return false; + } + } + true +} + +/// Whether a field's value at a dotted path is Present or Missing. +/// +/// Distinguishing these is required for `$exists` semantics: MongoDB's +/// `{$exists: false}` only matches documents where the field genuinely +/// isn't in the document, not documents where the field is present with +/// value `null`. +#[derive(Debug)] +pub enum FieldValue<'a> { + Present(&'a Bson), + Missing, +} + +impl<'a> FieldValue<'a> { + fn is_present(&self) -> bool { + matches!(self, FieldValue::Present(_)) + } +} + +/// Walk a dotted path like "item_data.address.city" through nested docs +/// (and arrays, when a path component is a numeric index). +pub fn walk_path<'a>(doc: &'a Document, path: &str) -> FieldValue<'a> { + let parts: Vec<&str> = path.split('.').collect(); + if parts.is_empty() { + return FieldValue::Missing; + } + let mut cur: &Bson = match doc.get(parts[0]) { + Some(v) => v, + None => return FieldValue::Missing, + }; + for part in &parts[1..] { + cur = match cur { + Bson::Document(d) => match d.get(*part) { + Some(v) => v, + None => return FieldValue::Missing, + }, + Bson::Array(a) => match part.parse::() { + Ok(idx) => match a.get(idx) { + Some(v) => v, + None => return FieldValue::Missing, + }, + Err(_) => return FieldValue::Missing, + }, + _ => return FieldValue::Missing, + }; + } + FieldValue::Present(cur) +} + +fn eq_or_array_match(field: &FieldValue<'_>, target: &Bson) -> bool { + match field { + FieldValue::Present(Bson::Array(arr)) => arr + .iter() + .any(|v| bson_cmp(v, target) == std::cmp::Ordering::Equal), + FieldValue::Present(v) => bson_cmp(v, target) == std::cmp::Ordering::Equal, + FieldValue::Missing => matches!(target, Bson::Null), + } +} + +fn cmp_field(field: &FieldValue<'_>, target: &Bson, pred: F) -> bool +where + F: Fn(&Bson, &Bson) -> bool, +{ + match field { + FieldValue::Present(v) => pred(v, target), + FieldValue::Missing => false, + } +} + +/// Compare two BSON values with MongoDB's total-ordering rules. +/// +/// This is a small subset — enough for the compiler's operator surface. +/// Numeric types are unified (Int32/Int64/Double/Decimal128 compare by +/// numeric value). Types that don't compare (Document vs. String) return +/// Equal by fallback because the compiler never emits comparisons between +/// mixed types in practice; if a proptest run generates one, the parity +/// check will surface the mismatch. +fn bson_cmp(a: &Bson, b: &Bson) -> std::cmp::Ordering { + use std::cmp::Ordering; + match (a, b) { + (Bson::String(x), Bson::String(y)) => x.cmp(y), + (Bson::Boolean(x), Bson::Boolean(y)) => x.cmp(y), + (Bson::Int32(x), Bson::Int32(y)) => x.cmp(y), + (Bson::Int64(x), Bson::Int64(y)) => x.cmp(y), + (Bson::Int32(x), Bson::Int64(y)) => (*x as i64).cmp(y), + (Bson::Int64(x), Bson::Int32(y)) => x.cmp(&(*y as i64)), + (Bson::Double(x), Bson::Double(y)) => x.partial_cmp(y).unwrap_or(Ordering::Equal), + (Bson::Binary(x), Bson::Binary(y)) => x.bytes.cmp(&y.bytes), + (Bson::Null, Bson::Null) => Ordering::Equal, + (Bson::Array(x), Bson::Array(y)) => { + // Elementwise; documents-of-arrays don't come up in the compiler + // output, but arrays-of-primitives can when a set field is + // compared to another set. + for (xi, yi) in x.iter().zip(y.iter()) { + match bson_cmp(xi, yi) { + Ordering::Equal => continue, + other => return other, + } + } + x.len().cmp(&y.len()) + } + (Bson::Document(x), Bson::Document(y)) => { + // Compare field-by-field in insertion order — matches how BSON + // documents are serialized and how our compiler's `$type` + // comparisons treat them. + for ((xk, xv), (yk, yv)) in x.iter().zip(y.iter()) { + match xk.cmp(yk) { + Ordering::Equal => match bson_cmp(xv, yv) { + Ordering::Equal => continue, + other => return other, + }, + other => return other, + } + } + x.len().cmp(&y.len()) + } + // Mismatched types: fall back to "not equal" ordering. The compiler + // shouldn't produce these — if a proptest generates one, the + // parity harness will report the divergence. + _ => Ordering::Equal, + } +} + +fn as_array(val: &Bson) -> Vec { + match val { + Bson::Array(a) => a.clone(), + _ => Vec::new(), + } +} + +fn as_doc(val: &Bson) -> Option<&Document> { + match val { + Bson::Document(d) => Some(d), + _ => None, + } +} + +fn as_bool(val: &Bson) -> Option { + match val { + Bson::Boolean(b) => Some(*b), + _ => None, + } +} + +/// Convert a `"$fieldname"` string to the field name it refers to, or +/// return None for anything else. Used only for `$expr` argument parsing. +fn as_field_ref(val: &Bson) -> Option<&str> { + match val { + Bson::String(s) => s.strip_prefix('$'), + _ => None, + } +} + +/// Minimal regex matcher — the compiler only emits `^prefix` and plain +/// substring patterns via `regex_escape`, so we don't need a full regex +/// engine. Anchors and escaped literals only. +fn regex_match(pattern: &str, s: &str) -> bool { + // Handle "^prefix" — anchored prefix match. + if let Some(prefix) = pattern.strip_prefix('^') { + // The compiler regex-escapes the prefix, so we treat it as a + // literal string here. Any regex metacharacter present means the + // compiler already escaped it. + let unescaped = unescape_regex(prefix); + return s.starts_with(&unescaped); + } + // Unanchored — substring match. Same treatment. + let unescaped = unescape_regex(pattern); + s.contains(&unescaped) +} + +/// Reverse the `regex_escape` transformation in `condition.rs`. Since our +/// compiler only escapes standard regex metacharacters with backslash, we +/// walk the string and unescape those. +fn unescape_regex(pattern: &str) -> String { + let mut out = String::with_capacity(pattern.len()); + let mut chars = pattern.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(next) = chars.next() { + out.push(next); + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use bson::doc; + + fn item() -> Document { + doc! { + "_id": "user#1", + "pk": "user#1", + "item_data": { + "name": { "S": "alice" }, + "age": { "N": "30" }, + "tags": { "SS": ["admin", "beta"] }, + "profile": { + "M": { + "email": { "S": "a@x.com" } + } + } + } + } + } + + #[test] + fn scalar_equality_present() { + let filter = doc! { "item_data.name.S": "alice" }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn scalar_equality_absent() { + let filter = doc! { "item_data.missing.S": "alice" }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn exists_true_present() { + let filter = doc! { "item_data.name": { "$exists": true } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn exists_false_absent() { + let filter = doc! { "item_data.missing": { "$exists": false } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn implicit_array_match_on_set() { + // `contains(tags, :s)` compiles to `{"item_data.tags.SS": "admin"}` + // and relies on implicit array-match to succeed when "admin" is a + // set member. + let filter = doc! { "item_data.tags.SS": "admin" }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn implicit_array_match_absent_member() { + let filter = doc! { "item_data.tags.SS": "nonmember" }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn lexicographic_lt() { + let filter = doc! { "item_data.name.S": { "$lt": "bob" } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn and_of_two_clauses() { + let filter = doc! { "$and": [ + { "item_data.name.S": "alice" }, + { "item_data.age.N": "30" } + ]}; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn or_short_circuit() { + let filter = doc! { "$or": [ + { "item_data.name.S": "wrong" }, + { "item_data.age.N": "30" } + ]}; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn nor_negates() { + let filter = doc! { "$nor": [ { "item_data.name.S": "bob" } ] }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn regex_prefix_match() { + let filter = doc! { "item_data.name.S": { "$regex": "^al" } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn regex_prefix_no_match() { + let filter = doc! { "item_data.name.S": { "$regex": "^bob" } }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn in_membership() { + let filter = doc! { "item_data.name.S": { "$in": ["alice", "bob"] } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn in_no_match() { + let filter = doc! { "item_data.name.S": { "$in": ["bob", "carol"] } }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn ne_true_when_different() { + let filter = doc! { "item_data.name.S": { "$ne": "bob" } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn empty_in_sentinel_never_matches() { + // Compiler emits this for empty IN () — a document must have _id + // and _id must have type "null", which contradicts each other for + // any well-formed item. + let filter = doc! { "$and": [ + { "_id": { "$exists": true } }, + { "_id": { "$type": "null" } } + ]}; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn nested_map_path() { + let filter = doc! { "item_data.profile.M.email.S": "a@x.com" }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn missing_intermediate_path_short_circuits() { + let filter = doc! { "item_data.profile.M.nonexistent.S": "any" }; + assert!(!eval_filter(&filter, &item())); + } +} diff --git a/crates/storage-mongodb/tests/interpreter_selftests.rs b/crates/storage-mongodb/tests/interpreter_selftests.rs new file mode 100644 index 00000000..e466b47a --- /dev/null +++ b/crates/storage-mongodb/tests/interpreter_selftests.rs @@ -0,0 +1,13 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Runs the interpreter's self-tests as an integration test binary. +//! The interpreter itself lives in `tests/common/mod.rs` — this file +//! exists so cargo compiles the module and runs its `#[cfg(test)]` tests. + +#[allow(dead_code, unused_imports)] +mod common; + +// The interpreter's self-tests live inside `common::tests` (gated by +// `#[cfg(test)]`). Cargo runs them automatically when this binary is +// built for `cargo test`. From a112a85cbe3c85d16777bf549a0d129678fe599d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:46:42 -0700 Subject: [PATCH 17/83] test(mongodb): proptest harness for pushdown-parity between compiler and DDB evaluator Adds crates/storage-mongodb/tests/pushdown_parity.rs. For every generated (item, expression) pair drawn from the pushable subset, the harness asserts that extenddb_core::expression::evaluate_condition(expr, item, maps) agrees with evaluating condition_to_filter(expr, maps) against the item's BSON representation using the interpreter from tests/common/mod.rs. 1024 proptest cases per run; shrinking enabled up to 4096 iters. Generation strategies: - arb_item(): random DDB Item with 0..5 attributes drawn from a fixed vocabulary. Not every name is populated in every item, so path-hit / path-miss interactions get exercised. - arb_safe_value(): S / B / BOOL / NULL / SS / BS / L. Excludes N and NS because the pushable expression grammar never references numeric operands. - arb_leaf_expr(): attribute_exists, attribute_not_exists, NOT attribute_exists, attribute_type on the safe type set, begins_with, contains, and =/<>/ on S. Each leaf yields (Expr AST, values-vec) for placeholder resolution. - arb_composed_expr(): AND/OR combining two leaves with renumbered placeholders so both are addressable in the merged values map. Compare on .B is temporarily excluded from the pushable subset: the compiler emits raw BSON Binary filter values for .B operands, but item_to_document stores .B fields as base64-encoded strings via the AttributeValue JSON serializer. The two formats never match. This pre-existing compiler / storage-layer contract bug is addressed in the next commit. MongoDB disambiguates operator documents from literal documents by whether any key starts with $. The step-1 interpreter unconditionally treated documents as operator docs, causing "unknown operator S" panics when the compiler emitted the literal {S: "..."} predicate for contains(L_field, :s). eval_predicate now checks the key prefix and falls back to equality match for pure literal documents. Adds proptest as a dev-dependency to the mongo crate. --- Cargo.lock | 1683 ++++++++++++----- crates/storage-mongodb/Cargo.toml | 5 + crates/storage-mongodb/tests/common/mod.rs | 10 + .../pushdown_parity.proptest-regressions | 8 + .../storage-mongodb/tests/pushdown_parity.rs | 384 ++++ 5 files changed, 1653 insertions(+), 437 deletions(-) create mode 100644 crates/storage-mongodb/tests/pushdown_parity.proptest-regressions create mode 100644 crates/storage-mongodb/tests/pushdown_parity.rs diff --git a/Cargo.lock b/Cargo.lock index 78e4d0d5..5ba716b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -26,7 +26,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -50,6 +50,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -122,15 +123,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -165,7 +166,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -177,7 +178,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -205,13 +206,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -237,9 +238,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "zeroize", @@ -247,14 +248,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -318,7 +320,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -388,6 +390,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -399,13 +416,25 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -415,6 +444,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blowfish" version = "0.9.1" @@ -425,6 +463,29 @@ dependencies = [ "cipher", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64 0.22.1", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap", + "js-sys", + "once_cell", + "rand 0.9.5", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -439,15 +500,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.62" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -461,21 +522,32 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "cipher" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] [[package]] name = "clap" -version = "4.6.1" +version = "4.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" dependencies = [ "clap_builder", "clap_derive", @@ -483,9 +555,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -495,14 +567,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -520,12 +592,28 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -559,7 +647,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" dependencies = [ "async-trait", - "convert_case", + "convert_case 0.6.0", "json5", "nom", "pathdiff", @@ -577,6 +665,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -606,6 +700,31 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -615,6 +734,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -639,38 +767,44 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -689,6 +823,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -698,6 +841,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -707,6 +859,54 @@ dependencies = [ "libc", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -719,7 +919,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -744,31 +944,87 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -861,28 +1117,20 @@ dependencies = [ [[package]] name = "extenddb" version = "0.1.2" -dependencies = [ - "anyhow", - "extenddb-app", - "extenddb-storage", - "extenddb-storage-postgres", -] - -[[package]] -name = "extenddb-app" -version = "0.1.2" dependencies = [ "anyhow", "base64 0.22.1", "clap", + "config", "daemonize", "extenddb-auth", "extenddb-cache", - "extenddb-config", "extenddb-core", "extenddb-engine", "extenddb-server", "extenddb-storage", + "extenddb-storage-mongodb", + "extenddb-storage-postgres", "libc", "rcgen", "rustls", @@ -890,6 +1138,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "syslog-tracing", "time", "tokio", "toml", @@ -907,9 +1156,9 @@ dependencies = [ "extenddb-core", "futures", "hex", - "hmac", + "hmac 0.12.1", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror", "time", "tokio", @@ -928,19 +1177,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "extenddb-config" -version = "0.1.2" -dependencies = [ - "anyhow", - "config", - "extenddb-core", - "extenddb-storage", - "serde", - "toml", - "tracing", -] - [[package]] name = "extenddb-core" version = "0.1.2" @@ -964,10 +1200,10 @@ dependencies = [ "extenddb-core", "extenddb-storage", "hex", - "hmac", + "hmac 0.12.1", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tokio", "tracing", "uuid", @@ -987,25 +1223,21 @@ dependencies = [ "crc32fast", "extenddb-auth", "extenddb-cache", - "extenddb-config", "extenddb-core", "extenddb-engine", "extenddb-storage", "futures", "hyper", - "libc", "metrics", - "rand 0.9.4", + "rand 0.9.5", "rustls", "serde", "serde_json", - "syslog-tracing", "time", "tokio", "tower", "tower-http", "tracing", - "tracing-subscriber", "uuid", ] @@ -1021,18 +1253,48 @@ dependencies = [ "extenddb-auth", "extenddb-core", "futures", - "rand 0.9.4", - "serde", + "inventory", + "rand 0.9.5", "serde_json", "thiserror", "time", "tokio", - "tokio-util", "toml", "tracing", "tracing-subscriber", ] +[[package]] +name = "extenddb-storage-mongodb" +version = "0.1.0" +dependencies = [ + "aes-gcm", + "anyhow", + "async-trait", + "base64 0.22.1", + "bcrypt", + "bson", + "crc32fast", + "dashmap", + "extenddb-auth", + "extenddb-core", + "extenddb-storage", + "futures", + "inventory", + "mongodb", + "proptest", + "rand 0.9.5", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "toml", + "tracing", + "uuid", + "zeroize", +] + [[package]] name = "extenddb-storage-postgres" version = "0.1.2" @@ -1047,7 +1309,8 @@ dependencies = [ "extenddb-core", "extenddb-storage", "futures", - "rand 0.9.4", + "inventory", + "rand 0.9.5", "serde", "serde_json", "sqlx", @@ -1060,6 +1323,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1110,9 +1379,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.3.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" dependencies = [ "autocfg", "tokio", @@ -1124,11 +1393,17 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -1141,9 +1416,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1151,15 +1426,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -1179,38 +1454,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -1240,8 +1515,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1251,22 +1528,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", + "rand_core 0.10.1", ] [[package]] @@ -1281,9 +1559,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -1356,13 +1634,83 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hkdf" -version = "0.12.4" +name = "hickory-net" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ - "hmac", -] + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] [[package]] name = "hmac" @@ -1370,7 +1718,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -1384,9 +1741,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1394,9 +1751,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1404,9 +1761,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1427,11 +1784,20 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "1.9.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -1546,10 +1912,10 @@ dependencies = [ ] [[package]] -name = "id-arena" -version = "2.3.0" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" @@ -1580,8 +1946,6 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", - "serde", - "serde_core", ] [[package]] @@ -1593,6 +1957,37 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1605,25 +2000,73 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1647,17 +2090,11 @@ dependencies = [ "spin", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "22053b6a34f84abc97f9129e61334f40174659a1b9bd18c970b83db6a9a6348b" [[package]] name = "libm" @@ -1667,14 +2104,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "bitflags", "libc", "plain", - "redox_syscall 0.7.5", + "redox_syscall 0.9.0", ] [[package]] @@ -1687,6 +2124,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -1704,9 +2147,57 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.119", +] [[package]] name = "matchers" @@ -1730,14 +2221,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "metrics" @@ -1773,9 +2274,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -1802,6 +2303,88 @@ dependencies = [ "uuid", ] +[[package]] +name = "mongocrypt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8426a875ded61430d4a811dbfda7633b6b8af0225c547fc6c28b8b0aa7d79a13" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.6+1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" + +[[package]] +name = "mongodb" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b814038f367d212f55de0a630cb35102a9b8ca23785a86955d62c0087c93846d" +dependencies = [ + "base64 0.22.1", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-net", + "hickory-proto", + "hickory-resolver", + "hmac 0.13.0", + "macro_magic", + "md-5 0.11.0", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2", + "percent-encoding", + "rand 0.9.5", + "rustc_version_runtime", + "rustls", + "serde", + "serde_bytes", + "serde_with", + "sha1 0.11.0", + "sha2 0.11.0", + "socket2", + "stringprep", + "strsim", + "take_mut", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots 1.0.9", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f736d2fbc56e0011a341fbb9172bd822fda75c5f93b82fae1c7aab1e2613c810" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nom" version = "7.1.3" @@ -1823,9 +2406,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -1842,7 +2425,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.6", + "rand 0.8.7", "smallvec", "zeroize", ] @@ -1864,11 +2447,10 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -1897,6 +2479,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -1955,6 +2541,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "pem" version = "3.0.6" @@ -1982,9 +2577,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" dependencies = [ "memchr", "ucd-trie", @@ -1992,9 +2587,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" dependencies = [ "pest", "pest_generator", @@ -2002,25 +2597,24 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "pest_meta" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" dependencies = [ "pest", - "sha2", ] [[package]] @@ -2069,16 +2663,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "potential_utf" @@ -2105,29 +2699,55 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "prefix-trie" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" dependencies = [ - "proc-macro2", - "syn", + "either", + "ipnet", + "num-traits", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec 0.8.0", + "bitflags", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -2144,11 +2764,17 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2157,14 +2783,25 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -2203,11 +2840,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rapidhash" -version = "4.4.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ "rustversion", ] @@ -2237,18 +2889,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ "bitflags", ] [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2257,9 +2909,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "ring" @@ -2293,8 +2951,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -2317,20 +2975,52 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -2353,9 +3043,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "zeroize", ] @@ -2374,9 +3064,21 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] [[package]] name = "ryu" @@ -2384,6 +3086,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2398,40 +3109,51 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -2471,15 +3193,48 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2489,8 +3244,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2504,9 +3270,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2524,15 +3290,31 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "slab" @@ -2542,18 +3324,18 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2561,9 +3343,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -2618,7 +3400,7 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror", "time", @@ -2640,7 +3422,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.119", ] [[package]] @@ -2658,12 +3440,12 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.119", "tokio", "url", ] @@ -2681,7 +3463,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -2691,18 +3473,18 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", - "rand 0.8.6", + "rand 0.8.7", "rsa", "serde", - "sha1", - "sha2", + "sha1 0.10.7", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -2732,18 +3514,18 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-bigint", "once_cell", - "rand 0.8.6", + "rand 0.8.7", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -2811,9 +3593,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" dependencies = [ "proc-macro2", "quote", @@ -2834,7 +3627,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2848,49 +3641,94 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.47" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -2900,15 +3738,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -2935,9 +3773,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -2950,9 +3788,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2967,13 +3805,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2999,13 +3837,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "libc", "pin-project-lite", "tokio", ] @@ -3118,7 +3959,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3173,11 +4014,31 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -3185,6 +4046,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -3214,9 +4081,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-xid" @@ -3230,7 +4097,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -3272,12 +4139,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -3300,27 +4168,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "wait-timeout" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] [[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ - "wit-bindgen 0.57.1", + "same-file", + "winapi-util", ] [[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -3331,9 +4209,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -3344,9 +4222,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3354,93 +4232,103 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] [[package]] -name = "wasm-encoder" -version = "0.244.0" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "leb128fmt", - "wasmparser", + "webpki-roots 1.0.9", ] [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "webpki-roots" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", + "rustls-pki-types", ] [[package]] -name = "wasmparser" -version = "0.244.0" +name = "whoami" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", + "libredox", + "wasite", ] [[package]] -name = "webpki-roots" -version = "0.26.11" +name = "widestring" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "webpki-roots 1.0.7", + "windows-sys 0.61.2", ] [[package]] -name = "webpki-roots" -version = "1.0.7" +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "rustls-pki-types", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "whoami" -version = "1.6.1" +name = "windows-result" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "libredox", - "wasite", + "windows-link", ] [[package]] -name = "windows-link" -version = "0.2.1" +name = "windows-strings" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] [[package]] name = "windows-sys" @@ -3599,15 +4487,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" @@ -3615,90 +4494,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" +name = "writeable" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "wit-parser" -version = "0.244.0" +name = "wyz" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", + "tap", ] -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - [[package]] name = "x509-parser" version = "0.18.1" @@ -3734,15 +4543,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", "time", ] [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -3757,28 +4566,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3798,28 +4607,28 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3852,11 +4661,11 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/storage-mongodb/Cargo.toml b/crates/storage-mongodb/Cargo.toml index a2d00fff..fbc5b862 100644 --- a/crates/storage-mongodb/Cargo.toml +++ b/crates/storage-mongodb/Cargo.toml @@ -54,3 +54,8 @@ crc32fast.workspace = true # In-process GSI existence cache dashmap.workspace = true + +[dev-dependencies] +# Property-based testing for the filter-pushdown parity harness +# (crates/storage-mongodb/tests/pushdown_parity.rs). +proptest = "1" diff --git a/crates/storage-mongodb/tests/common/mod.rs b/crates/storage-mongodb/tests/common/mod.rs index a5073418..581abc76 100644 --- a/crates/storage-mongodb/tests/common/mod.rs +++ b/crates/storage-mongodb/tests/common/mod.rs @@ -65,6 +65,16 @@ fn eval_clause(key: &str, val: &Bson, doc: &Document) -> bool { fn eval_predicate(pred: &Bson, field: &FieldValue<'_>) -> bool { match pred { Bson::Document(pred_doc) => { + // Disambiguate: MongoDB treats a document as an operator + // document when at least one of its keys starts with `$`; + // otherwise it's a literal document for equality match. This + // is the same rule the driver uses. The compiler relies on + // literal-match for `contains(L_field, :s)` which emits a + // predicate like `{S: "value"}` — no `$` keys. + let is_operator_doc = pred_doc.keys().any(|k| k.starts_with('$')); + if !is_operator_doc { + return eq_or_array_match(field, pred); + } // Operator document: every operator must match. MongoDB's // behavior with multiple operator keys in one doc is that they // are ANDed together. diff --git a/crates/storage-mongodb/tests/pushdown_parity.proptest-regressions b/crates/storage-mongodb/tests/pushdown_parity.proptest-regressions new file mode 100644 index 00000000..87c79459 --- /dev/null +++ b/crates/storage-mongodb/tests/pushdown_parity.proptest-regressions @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 786e25608dd3707c250764c4ed74630a114a629ebbef777a12f98670ef693a98 # shrinks to item = {}, expr_pair = (Or(Function { name: "contains", args: [Path([Attribute("a")]), Placeholder(":v0")] }, Function { name: "attribute_exists", args: [Path([Attribute("a")])] }), [S("")]) +cc ba22175a8084a07e6dc59ba349772e95cdcce6902424574c1083700f364ba6e9 # shrinks to item = {"c": B([0])}, expr_pair = (Compare { left: Path([Attribute("c")]), op: Eq, right: Placeholder(":v0") }, [B([])]) diff --git a/crates/storage-mongodb/tests/pushdown_parity.rs b/crates/storage-mongodb/tests/pushdown_parity.rs new file mode 100644 index 00000000..2efda64b --- /dev/null +++ b/crates/storage-mongodb/tests/pushdown_parity.rs @@ -0,0 +1,384 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Filter-pushdown parity harness. +//! +//! For every generated (item, expression) pair drawn from the "pushable +//! subset" (see todo.md A5), assert that +//! +//! `extenddb_core::expression::evaluate_condition(expr, item, maps)` +//! +//! agrees with evaluating +//! +//! `condition_to_filter(expr, maps)` +//! +//! against the item's BSON representation using the interpreter in +//! `tests/common/mod.rs`. Any divergence indicates the compiler emits a +//! filter whose semantics differ from DDB's — a bug in the compiler. +//! +//! The harness deliberately excludes expression shapes that fall outside +//! the pushable subset: no numeric comparisons, no `size()`, and `NOT` +//! only around `attribute_exists` / `attribute_not_exists`. Those cases +//! are handled by fallback to session-scoped in-Rust evaluation in step 3 +//! of A5, and don't need pushdown parity. + +#[allow(dead_code)] +mod common; + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use bson::{Bson, Document}; +use proptest::prelude::*; +use proptest::sample::select; + +use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement, evaluate_condition}; +use extenddb_core::types::AttributeValue; +use extenddb_storage_mongodb::condition::condition_to_filter; + +use common::eval_filter; + +// --------------------------------------------------------------------------- +// Attribute-value strategies +// --------------------------------------------------------------------------- + +/// A small vocabulary of attribute names. Reusing names across items and +/// expressions is what surfaces path-hit / path-miss interactions. +const NAMES: &[&str] = &["a", "b", "c", "x", "y"]; + +/// Constrained string alphabet — short, printable, small alphabet. +/// Keeps proptest shrinking behavior tractable and matches the kinds of +/// values real DDB workloads use for status flags, tags, ETags, etc. +fn arb_short_str() -> impl Strategy { + proptest::string::string_regex("[a-z]{0,4}").unwrap() +} + +fn arb_short_bytes() -> impl Strategy> { + proptest::collection::vec(any::(), 0..4) +} + +/// Attribute values in the "safe subset" — everything except N/NS. +/// The pushable expression grammar never references numeric operands, so +/// items don't need to contain them either. Including L/M would require a +/// recursive strategy; we keep depth flat for now (paths in the grammar +/// are single-name only, so nested M/L values wouldn't be reachable +/// anyway). +fn arb_safe_value() -> impl Strategy { + prop_oneof![ + arb_short_str().prop_map(AttributeValue::S), + arb_short_bytes().prop_map(AttributeValue::B), + any::().prop_map(AttributeValue::Bool), + Just(AttributeValue::Null), + proptest::collection::btree_set(arb_short_str(), 0..3).prop_map(AttributeValue::SS), + proptest::collection::btree_set(arb_short_bytes(), 0..3).prop_map(AttributeValue::BS), + // Lists of strings — the compiler's contains-on-list path takes a + // scalar and matches any element. Mixed-type lists aren't in the + // pushable grammar so we don't generate them. + proptest::collection::vec(arb_short_str().prop_map(AttributeValue::S), 0..3) + .prop_map(AttributeValue::L), + ] +} + +/// A random DDB Item. Not every name is present in every item — that's +/// how the "path missing" edge cases get exercised. +fn arb_item() -> impl Strategy> { + proptest::collection::vec( + (select(NAMES).prop_map(String::from), arb_safe_value()), + 0..NAMES.len(), + ) + .prop_map(|pairs| { + let mut m = BTreeMap::new(); + for (k, v) in pairs { + m.insert(k, v); + } + m + }) +} + +// --------------------------------------------------------------------------- +// Expression strategies +// --------------------------------------------------------------------------- + +/// A random attribute name from the vocabulary. +fn arb_name_expr() -> impl Strategy { + select(NAMES).prop_map(|n| Expr::Path(vec![PathElement::Attribute(n.to_string())])) +} + +/// A random placeholder reference (`:v0`, `:v1`, ...). +fn arb_placeholder_ref(idx: usize) -> Expr { + Expr::Placeholder(format!(":v{idx}")) +} + +/// A leaf comparison expression that is safely pushable. Each returns: +/// - the AST for the expression +/// - the values map required to resolve any placeholders in the AST +/// +/// The placeholders are numbered per-expression starting at :v0. When +/// composing with AND/OR (below), we renumber to keep them globally unique. +/// +/// Binary (`.B`) operands are currently excluded: the compiler emits raw +/// BSON `Binary` filters for `.B` comparisons, but `item_to_document` +/// (in production) stores `.B` fields as base64-encoded strings via the +/// AttributeValue JSON serializer. The two formats never match — a +/// pre-existing bug in the compiler / storage-layer contract. A5 step 3 +/// will either fix the compiler to emit string filters (matching storage) +/// or fix the storage to emit BSON binary (matching the compiler), and +/// re-enable `.B` comparisons in this harness. +#[allow(clippy::redundant_closure)] +fn arb_leaf_expr() -> impl Strategy)> { + // We union several leaf shapes. Each yields (Expr, values-list). + let name = || arb_name_expr(); + let str_val = || arb_short_str().prop_map(AttributeValue::S); + + prop_oneof![ + // attribute_exists(name) + name().prop_map(|n| ( + Expr::Function { + name: "attribute_exists".into(), + args: vec![n], + }, + vec![] + )), + // attribute_not_exists(name) + name().prop_map(|n| ( + Expr::Function { + name: "attribute_not_exists".into(), + args: vec![n], + }, + vec![] + )), + // NOT attribute_exists(name) — the only NOT the pushable subset allows + name().prop_map(|n| ( + Expr::Not(Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![n], + })), + vec![] + )), + // attribute_type(name, :t) for the S / B / BOOL / NULL / SS / BS / L / M types + ( + name(), + select(&["S", "B", "BOOL", "NULL", "SS", "BS", "L", "M"][..]).prop_map(String::from) + ) + .prop_map(|(n, t)| ( + Expr::Function { + name: "attribute_type".into(), + args: vec![n, arb_placeholder_ref(0)], + }, + vec![AttributeValue::S(t)], + )), + // begins_with(name, :prefix) + (name(), arb_short_str()).prop_map(|(n, p)| ( + Expr::Function { + name: "begins_with".into(), + args: vec![n, arb_placeholder_ref(0)], + }, + vec![AttributeValue::S(p)], + )), + // contains(name, :substr) — matches when name is S (substring) or SS/BS/L (membership) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Function { + name: "contains".into(), + args: vec![n, arb_placeholder_ref(0)], + }, + vec![v], + )), + // name = :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Eq, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name <> :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Ne, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name < :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Lt, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name > :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Gt, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + ] +} + +/// Compose two leaf expressions with AND or OR. Renumbers the second +/// expression's placeholders so both are addressable in the merged +/// values map. +fn arb_composed_expr() -> impl Strategy)> { + (arb_leaf_expr(), arb_leaf_expr(), any::()).prop_map(|(l, r, is_and)| { + let (lhs, mut lvals) = l; + let (rhs, rvals) = r; + let rhs_offset = lvals.len(); + // Renumber rhs placeholders to `:v`. + let rhs = renumber_placeholders(rhs, rhs_offset); + lvals.extend(rvals); + let composed = if is_and { + Expr::And(Box::new(lhs), Box::new(rhs)) + } else { + Expr::Or(Box::new(lhs), Box::new(rhs)) + }; + (composed, lvals) + }) +} + +fn renumber_placeholders(expr: Expr, offset: usize) -> Expr { + match expr { + Expr::Placeholder(name) => { + if let Some(idx_str) = name.strip_prefix(":v") { + if let Ok(idx) = idx_str.parse::() { + return Expr::Placeholder(format!(":v{}", idx + offset)); + } + } + Expr::Placeholder(name) + } + Expr::Path(_) => expr, + Expr::Compare { left, op, right } => Expr::Compare { + left: Box::new(renumber_placeholders(*left, offset)), + op, + right: Box::new(renumber_placeholders(*right, offset)), + }, + Expr::And(l, r) => Expr::And( + Box::new(renumber_placeholders(*l, offset)), + Box::new(renumber_placeholders(*r, offset)), + ), + Expr::Or(l, r) => Expr::Or( + Box::new(renumber_placeholders(*l, offset)), + Box::new(renumber_placeholders(*r, offset)), + ), + Expr::Not(inner) => Expr::Not(Box::new(renumber_placeholders(*inner, offset))), + Expr::Function { name, args } => Expr::Function { + name, + args: args + .into_iter() + .map(|a| renumber_placeholders(a, offset)) + .collect(), + }, + Expr::Between { operand, low, high } => Expr::Between { + operand: Box::new(renumber_placeholders(*operand, offset)), + low: Box::new(renumber_placeholders(*low, offset)), + high: Box::new(renumber_placeholders(*high, offset)), + }, + Expr::In { operand, list } => Expr::In { + operand: Box::new(renumber_placeholders(*operand, offset)), + list: list + .into_iter() + .map(|a| renumber_placeholders(a, offset)) + .collect(), + }, + other => other, + } +} + +/// Generate either a leaf or a composed AND/OR expression. +fn arb_expr() -> impl Strategy)> { + prop_oneof![ + 3 => arb_leaf_expr(), + 1 => arb_composed_expr(), + ] +} + +// --------------------------------------------------------------------------- +// Item → BSON conversion for the interpreter side +// --------------------------------------------------------------------------- + +/// Serialize an Item to the BSON shape the compiler assumes: +/// { item_data: { : , ... } } +fn item_to_bson_doc(item: &BTreeMap) -> Document { + let item_data_json = serde_json::to_value(item).expect("item serializes"); + let item_data_bson: Bson = bson::to_bson(&item_data_json).expect("BSON conversion"); + let mut doc = Document::new(); + doc.insert("item_data", item_data_bson); + doc +} + +// --------------------------------------------------------------------------- +// The parity property +// --------------------------------------------------------------------------- + +proptest! { + #![proptest_config(ProptestConfig { + cases: 1024, + max_shrink_iters: 4096, + ..Default::default() + })] + + /// For any pushable expression and any item, the compiled MongoDB + /// filter (evaluated by our BSON interpreter) must agree with the + /// in-Rust DDB evaluator's pass/fail result. + #[test] + fn compiled_filter_matches_ddb_evaluator( + item in arb_item(), + expr_pair in arb_expr(), + ) { + let (expr, values_vec) = expr_pair; + + // Build the ExpressionMaps that the DDB evaluator and the compiler + // both consume. Placeholders are :v0, :v1, ... in insertion order. + let mut values = HashMap::new(); + for (idx, v) in values_vec.iter().enumerate() { + values.insert(format!(":v{idx}"), v.clone()); + } + let maps = ExpressionMaps::new(HashMap::new(), values); + + // Path A: in-Rust DDB evaluator against the logical Item. + let ddb_result = evaluate_condition(&expr, &item, &maps).unwrap_or(false); + + // Path B: compile to MongoDB filter, then evaluate the filter + // against the item's BSON representation using our interpreter. + let filter = match condition_to_filter(&expr, &maps) { + Ok(f) => f, + Err(e) => { + // The compiler rejected this expression. That's fine — it + // means A5's fallback path would kick in. Skip this case + // rather than treating it as a mismatch. + // + // We still assert the compiler doesn't reject something + // the DDB evaluator accepts as trivially-true or trivially- + // false — but the compiler rejecting compilation is + // logically different from evaluating to false. + let _ = e; + return Ok(()); + } + }; + let bson_doc = item_to_bson_doc(&item); + let mongo_result = eval_filter(&filter, &bson_doc); + + prop_assert_eq!( + ddb_result, + mongo_result, + "parity mismatch:\n expr: {:?}\n item: {:?}\n filter: {:?}\n ddb: {}\n mongo: {}", + expr, + item, + filter, + ddb_result, + mongo_result, + ); + } +} + +// Silence unused-import lint from `common` when running only a subset. +#[allow(dead_code)] +fn _keep_common_imported() { + let _ = std::mem::size_of::>(); + let _ = std::mem::size_of::>(); +} From e61b787cf34cb5220a97dbe1dd5eb987be310ebe Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 22:46:42 -0700 Subject: [PATCH 18/83] feat(mongodb): enable selective filter pushdown for conditional writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conditional DeleteItem and UpdateItem on tables with no streams and no GSIs now compile the condition into a MongoDB filter and execute the write as a single find_one_and_delete / find_one_and_replace with the merged (key filter ∧ condition filter), skipping the session / transaction overhead of the fallback path. Not on the pushdown path: - PutItem — insert-vs-update disambiguation makes the null-return case more subtle; deferred. - Any write on a table with an active stream or GSI — dependent writes still need session atomicity. - Expressions the analyzer marks NotPushable — fallback to the existing session-scoped read-then-check. Components: 1. crates/storage-mongodb/src/pushdown.rs — a compile-time analyzer is_pushable(&Expr, &ExpressionMaps) -> Pushable that walks the AST and returns Pushable::Yes only when every subexpression is safely translatable. Whitelist: - attribute_exists / attribute_not_exists - attribute_type on any type tag - begins_with(S, :prefix) - contains(S_field, :substr) - Compare on S operands (any op) - Compare on B operands with =/<> only - Compare on BOOL/NULL with =/<> only - Field vs. Field compare (compiled to $expr) - AND / OR of pushable subexpressions - NOT around attribute_exists / attribute_not_exists only Falls back to session-scoped in-Rust evaluation for: - Any operand of type N (numbers stored as strings; MongoDB comparators evaluate lexicographically, breaking numeric semantics) - size() (UTF-16 code unit count mismatch with $strLenBytes/CP) - NOT around anything non-existence (three-valued-logic drift vs. $nor on missing paths) - BETWEEN, IN - Ordering comparators on B operands (base64 string ordering does not preserve byte ordering across mismatched lengths) 2. crates/storage-mongodb/src/condition.rs::av_to_bson — fix for a compiler / storage-layer contract bug on B operands. Previously emitted raw BSON Binary for `.B` operands, but item_to_document stores B fields as base64-encoded strings via AttributeValue's JSON serializer. The two never matched — any FilterExpression or ConditionExpression against a B field failed silently against real stored data. Now emits the base64 string form so filters match storage. Same fix applied to the contains(B_field, :b) list-element predicate. 3. crates/storage-mongodb/src/data_engine.rs: - delete_item_pushdown: single find_one_and_delete with merged filter; follow-up find_one on null return to distinguish key-missing from condition-failed. - update_item_pushdown: two-round-trip (find matched doc + OCC replace) — DDB update-expression semantics are richer than MongoDB's atomic operators express in general (list_append, arithmetic on decimal strings, if_not_exists). Version guard via the existing _v field catches concurrent writers. - delete_item_impl / update_item_impl: pre-guard at the top — if condition is Some, no stream, no GSI cached, and analyzer says Yes, take the pushdown path; otherwise fall through unchanged. 4. crates/storage-mongodb/tests/pushdown_parity.rs — adds a second property test analyzer_yes_implies_filter_parity that asserts the analyzer's soundness: for every generated expression it marks Pushable::Yes, the compiled filter produces the same result as the DDB in-Rust evaluator on a random item. Re-enables .B equality/inequality in the generator; .B ordering stays fallback-only. Verification: - 35 unit tests (mongo crate lib) - 20 property tests (both parity properties, 1024 cases each) - 18 interpreter self-tests - End-to-end pushdown smoke test against live MongoDB 7 replica set: delete-success, delete-fail, delete-missing-key with attribute_not_exists, update-success with etag guard, update-fail with wrong etag — all correct. --- crates/storage-mongodb/src/condition.rs | 34 +- crates/storage-mongodb/src/data_engine.rs | 234 +++++++++++ crates/storage-mongodb/src/lib.rs | 1 + crates/storage-mongodb/src/pushdown.rs | 389 ++++++++++++++++++ .../storage-mongodb/tests/pushdown_parity.rs | 95 ++++- 5 files changed, 730 insertions(+), 23 deletions(-) create mode 100644 crates/storage-mongodb/src/pushdown.rs diff --git a/crates/storage-mongodb/src/condition.rs b/crates/storage-mongodb/src/condition.rs index d731e59d..c2301b94 100644 --- a/crates/storage-mongodb/src/condition.rs +++ b/crates/storage-mongodb/src/condition.rs @@ -53,19 +53,27 @@ fn resolve_path_to_field( /// Convert an `AttributeValue` to a BSON value for filter comparisons. /// -/// `DynamoDB` stores typed values like `{"S": "hello"}`, so when comparing -/// `item_data.foo.S` we need the raw string value, not the wrapped form. +/// The value must match how `data/mod.rs::item_to_document` stores the +/// value inside `item_data`, otherwise the compiled filter will not +/// match real stored items. Storage serializes each attribute value via +/// the `AttributeValue` `Serialize` impl (JSON-shape) and then converts +/// to BSON — so: /// -/// Numbers are kept as strings to match the storage format in `item_data` -/// (DynamoDB numbers are 38-digit decimals stored as their string representation). +/// - `S(s)` → JSON string → BSON string +/// - `N(n)` → JSON string (numbers are wire-encoded as strings) → BSON string +/// - `B(b)` → JSON string (base64) → BSON string +/// - `Bool(b)` → BSON boolean +/// - `Null` → JSON true (the `{"NULL": true}` tag) → BSON boolean true +/// +/// Emitting raw BSON `Binary` for `.B` here (as an earlier version did) +/// produced a filter that never matched real stored items because +/// storage writes the base64 string form. fn av_to_bson(av: &AttributeValue) -> Bson { + use base64::Engine; match av { AttributeValue::S(s) => Bson::String(s.clone()), AttributeValue::N(n) => Bson::String(n.clone()), - AttributeValue::B(b) => Bson::Binary(bson::Binary { - subtype: bson::spec::BinarySubtype::Generic, - bytes: b.clone(), - }), + AttributeValue::B(b) => Bson::String(base64::engine::general_purpose::STANDARD.encode(b)), AttributeValue::Bool(b) => Bson::Boolean(*b), AttributeValue::Null => Bson::Boolean(true), // NULL type stores {"NULL": true} _ => Bson::Null, // Sets and complex types handled differently @@ -316,12 +324,16 @@ fn compile_function( { &list_field: &list_elem }, ] }) } - AttributeValue::B(b) => { - // Binary membership in BS or L + AttributeValue::B(_) => { + // Binary membership in BS or L. Storage serializes B + // as base64 strings inside item_data (matches wire + // format via the JSON serializer), so both the field + // predicate and the list-element predicate use the + // base64 form. let bs_field = format!("{field}.BS"); let list_field = format!("{field}.L"); let bson_val = av_to_bson(&val); - let list_elem = doc! { "B": bson::Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: b.clone() } }; + let list_elem = doc! { "B": bson_val.clone() }; Ok(doc! { "$or": [ { &bs_field: &bson_val }, { &list_field: &list_elem }, diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index d5b7f087..d4887067 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -29,6 +29,7 @@ use crate::condition::condition_to_filter; use crate::data::{ data_collection_name, document_to_item, item_to_document, pk_filter, sk_field_name, }; +use crate::pushdown::{Pushable, is_pushable}; use extenddb_core::types::{AttributeDefinition, Projection, ProjectionType}; @@ -488,6 +489,20 @@ impl MongoEngine { maps: &ExpressionMaps, stream: Option<&StreamCapture>, ) -> Result, StorageError> { + // Pushdown fast path: conditional delete on a no-stream / no-GSI + // table with a pushable condition. Collapses read-then-check-then- + // write inside a session to a single `find_one_and_delete` with + // the merged filter. See `crates/storage-mongodb/src/pushdown.rs`. + if let Some(cond) = condition + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + && matches!(is_pushable(cond, maps), Pushable::Yes) + { + return self + .delete_item_pushdown(key_info, key, return_old, cond, maps) + .await; + } + let coll_name = data_collection_name(&key_info.table_id); let coll = self.data_db.collection::(&coll_name); @@ -593,6 +608,27 @@ impl MongoEngine { maps: &ExpressionMaps, stream: Option<&StreamCapture>, ) -> Result<(Option, Option), StorageError> { + // Pushdown fast path (A5): conditional update on a no-stream / + // no-GSI table with a pushable condition. Skips the session/ + // transaction overhead. See `crates/storage-mongodb/src/pushdown.rs`. + if let Some(cond) = condition + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + && matches!(is_pushable(cond, maps), Pushable::Yes) + { + match self + .update_item_pushdown(key_info, key, actions, return_old, return_new, cond, maps) + .await + { + Ok(pair) => return Ok(pair), + Err(StorageError::Internal(msg)) if msg.contains("raced by concurrent writer") => { + // Fall through to session-scoped path which + // has a proper retry loop. + } + Err(other) => return Err(other), + } + } + let coll_name = data_collection_name(&key_info.table_id); let coll = self.data_db.collection::(&coll_name); @@ -1898,6 +1934,204 @@ impl MongoEngine { } } } + + // ── Pushdown fast path (A5) ────────────────────────────────────── + // + // Callers must pre-check the guard conditions: + // - condition is Some(cond) + // - stream.is_none() + // - gsi_cache_get_fresh(table_id) == Some(false) + // - is_pushable(cond, maps) == Pushable::Yes + // + // Under those guards, the write's atomicity is provided by MongoDB's + // single-document find_one_and_* operators — no session needed, no + // GSI sync, no stream record. The compiled filter merges with the + // key filter so the operator matches only when both apply. On null + // return, we follow up with a `find_one` against the key alone to + // distinguish "key doesn't exist" from "condition failed". + + async fn delete_item_pushdown( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: &Expr, + maps: &ExpressionMaps, + ) -> Result, StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let cond_filter = condition_to_filter(condition, maps)?; + + // Merge key filter and condition filter under an $and so the + // delete only fires when both match. + let merged = doc! { "$and": [key_filter.clone(), cond_filter] }; + + let old_doc = coll + .find_one_and_delete(merged) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(doc) = old_doc { + let old_item = document_to_item(&doc)?; + return Ok(if return_old { Some(old_item) } else { None }); + } + + // Null return: either the key doesn't exist or the condition + // failed. Disambiguate with a follow-up find_one on the key. + let existing = coll + .find_one(key_filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + match existing { + Some(doc) => { + let existing_item = document_to_item(&doc)?; + Err(StorageError::ConditionFailed(Some(existing_item))) + } + None => { + // Key genuinely doesn't exist. Evaluate the condition + // against an empty item to match DDB semantics — some + // conditions (attribute_not_exists) evaluate to true + // even when the item is missing, in which case the + // delete is a no-op success rather than a condition + // failure. + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(condition, &empty, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if passed { + Ok(None) + } else { + Err(StorageError::ConditionFailed(None)) + } + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn update_item_pushdown( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: &Expr, + maps: &ExpressionMaps, + ) -> Result<(Option, Option), StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let cond_filter = condition_to_filter(condition, maps)?; + let merged = doc! { "$and": [key_filter.clone(), cond_filter] }; + + // Load the item first so we can apply the update in Rust and + // then replace it. This is a two-round-trip pushdown rather than + // a single-RT one because DDB update expressions have richer + // semantics than MongoDB's atomic update operators can express + // in general (e.g. list_append, if_not_exists, arithmetic on + // decimal strings). The win over the session-scoped fallback is + // that we skip the session start/commit round trips. + let existing_doc = coll + .find_one(merged.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let Some(existing) = existing_doc else { + // No document matched the key+condition filter. Disambiguate. + let by_key = coll + .find_one(key_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return match by_key { + Some(doc) => { + let existing_item = document_to_item(&doc)?; + Err(StorageError::ConditionFailed(Some(existing_item))) + } + None => { + // Key didn't exist. Evaluate condition against + // empty item (for attribute_not_exists-style + // guards that permit upsert). + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(condition, &empty, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + return Err(StorageError::ConditionFailed(None)); + } + // Condition allows the upsert. Build the new item + // from `key` + apply update actions. + let mut new_item = key.clone(); + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + let new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter, new_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok((None, if return_new { Some(new_item) } else { None })) + } + }; + }; + + let existing_item = document_to_item(&existing)?; + let mut new_item = existing_item.clone(); + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + + let new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + + // Bump the OCC version. The session-scoped path uses a versioned + // filter to catch concurrent modifications; the pushdown path + // does the same by merging the current version into the replace + // filter. If a concurrent writer bumps _v between our find_one + // and our replace_one, the replace matches nothing and we fall + // back to a retry. + let current_version = existing.get_i64("_v").unwrap_or(0); + let mut new_doc_versioned = new_doc; + new_doc_versioned.insert("_v", current_version + 1); + + let mut versioned_filter = key_filter.clone(); + if current_version == 0 { + versioned_filter.insert("_v", doc! { "$not": { "$gt": 0_i64 } }); + } else { + versioned_filter.insert("_v", current_version); + } + + let result = coll + .replace_one(versioned_filter, new_doc_versioned) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.matched_count == 0 { + // Concurrent update raced us. Fall back to the session-scoped + // path which has a retry loop. This is rare — return an + // Internal error the trait implementer can catch and retry. + return Err(StorageError::Internal( + "pushdown update raced by concurrent writer; retry via session-scoped path" + .to_owned(), + )); + } + + let old_out = if return_old { + Some(existing_item) + } else { + None + }; + let new_out = if return_new { Some(new_item) } else { None }; + Ok((old_out, new_out)) + } } // ── Transaction helper types ────────────────────────────────────────── diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 45b36898..1417b315 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -22,6 +22,7 @@ mod data_engine; mod management_store; mod metadata_engine; mod operations; +pub mod pushdown; mod stream_engine; mod table_engine; mod ttl_worker; diff --git a/crates/storage-mongodb/src/pushdown.rs b/crates/storage-mongodb/src/pushdown.rs new file mode 100644 index 00000000..4762566c --- /dev/null +++ b/crates/storage-mongodb/src/pushdown.rs @@ -0,0 +1,389 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Compile-time analyzer for filter-pushdown eligibility. +//! +//! For a DDB [`Expr`] and its accompanying [`ExpressionMaps`], returns +//! [`Pushable::Yes`] when the expression can be safely compiled to a +//! MongoDB filter and evaluated by the storage layer, or +//! [`Pushable::No`] with a reason string when at least one subexpression +//! must fall back to session-scoped in-Rust evaluation. +//! +//! Whole-condition all-or-nothing: if any subexpression is not pushable, +//! the entire expression falls back. You cannot cherry-pick — evaluating +//! part of an `AND` / `OR` in the storage layer and part in the +//! application layer would confuse the composition semantics. +//! +//! **Pushable subset** (see `todo.md` A5 for full rationale): +//! +//! - `attribute_exists(path)`, `attribute_not_exists(path)` +//! - `attribute_type(path, :t)` for any type tag +//! - `begins_with(path, :prefix)` where `:prefix` is `S` +//! - `contains(path, :val)` where `:val` is `S` +//! - `path :v` where `:v` is `S`, or `:v` is `B` and op is `Eq` / `Ne` +//! - `AND`, `OR` of pushable subexpressions +//! - `NOT attribute_exists(path)` / `NOT attribute_not_exists(path)` only +//! +//! **Not pushable** (falls back to in-Rust): +//! +//! - Any operand of type `N` in any position (numbers are stored as +//! strings, so MongoDB comparators evaluate lexicographically — +//! `"10" > "9"` is false string-wise, true numerically) +//! - `size(...)` (MongoDB's `$strLenBytes` and `$strLenCP` don't match +//! DDB's UTF-16 code unit count for strings) +//! - `NOT` around anything except `attribute_exists` / +//! `attribute_not_exists` (three-valued logic on missing paths +//! diverges from MongoDB's `$nor` semantics) +//! - `IN` and `BETWEEN` — the compiler emits them but the analyzer +//! currently marks them non-pushable pending proptest coverage. +//! Restoring them is a follow-up; the in-Rust fallback path is +//! already correct. +//! - `path :v` where `:v` is `B` (base64 string ordering +//! ≠ bytewise byte ordering across mismatched lengths) +//! - Any operand type the analyzer doesn't yet classify + +use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps}; +use extenddb_core::types::AttributeValue; + +/// Outcome of the pushdown analyzer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Pushable { + /// The expression compiles to a MongoDB filter with semantics that + /// match `extenddb_core::expression::evaluate_condition`. + Yes, + /// At least one subexpression falls outside the pushable subset. + /// The caller must evaluate the whole expression in-Rust. + No(&'static str), +} + +impl Pushable { + pub fn is_yes(&self) -> bool { + matches!(self, Pushable::Yes) + } +} + +/// Decide whether a compiled MongoDB filter for `expr` will agree with +/// `evaluate_condition(expr, item, maps)` for every item. +/// +/// Conservative: unknown constructs return `Pushable::No`. +pub fn is_pushable(expr: &Expr, maps: &ExpressionMaps) -> Pushable { + walk(expr, maps) +} + +fn walk(expr: &Expr, maps: &ExpressionMaps) -> Pushable { + match expr { + Expr::Function { name, args } => match name.to_lowercase().as_str() { + "attribute_exists" | "attribute_not_exists" => { + // Always pushable; missing-path semantics match MongoDB's + // `$exists`. + if args.len() == 1 { + Pushable::Yes + } else { + Pushable::No("attribute_exists/not_exists arity") + } + } + "attribute_type" => { + if args.len() != 2 { + return Pushable::No("attribute_type arity"); + } + // The type argument must resolve to a String (the type + // tag: "S", "N", "B", "BOOL", "NULL", "L", "M", "SS", + // "NS", "BS"). Anything else is a parse-time error but + // we defensively check. + if !arg_resolves_to_scalar_type(&args[1], maps, AttrKind::S) { + return Pushable::No("attribute_type non-string tag"); + } + Pushable::Yes + } + "begins_with" => { + if args.len() != 2 { + return Pushable::No("begins_with arity"); + } + if !arg_resolves_to_scalar_type(&args[1], maps, AttrKind::S) { + return Pushable::No("begins_with non-string prefix"); + } + Pushable::Yes + } + "contains" => { + if args.len() != 2 { + return Pushable::No("contains arity"); + } + // Only S operands. B is emittable but not yet + // proptest-covered; N is not pushable because number + // storage is string-based. + match value_kind(&args[1], maps) { + Some(AttrKind::S) => Pushable::Yes, + Some(AttrKind::N) => Pushable::No("contains on N operand"), + Some(AttrKind::B) => Pushable::No("contains on B operand (not yet covered)"), + _ => Pushable::No("contains on unsupported operand type"), + } + } + "size" => Pushable::No("size() — UTF-16 mismatch with MongoDB"), + _ => Pushable::No("unknown function"), + }, + Expr::Compare { left, op, right } => { + // Both operands must be pushable operand types. Numbers + // anywhere → not pushable. Sets / lists / maps in the + // operand position → not pushable in the current subset. + let left_kind = operand_kind(left, maps); + let right_kind = operand_kind(right, maps); + let (Some(lk), Some(rk)) = (left_kind, right_kind) else { + return Pushable::No("Compare with un-inferrable operand kind"); + }; + // Number anywhere disqualifies. + if matches!(lk, AttrKind::N) || matches!(rk, AttrKind::N) { + return Pushable::No("Compare with N operand"); + } + match (lk, rk, op) { + // Field S-value: any comparator OK (lex matches wire form). + (AttrKind::Field, AttrKind::S, _) | (AttrKind::S, AttrKind::Field, _) => { + Pushable::Yes + } + // Field = / <> B-value: OK. Ordering on B not OK + // (base64 string ordering ≠ bytewise). + (AttrKind::Field, AttrKind::B, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::B, AttrKind::Field, CompareOp::Eq | CompareOp::Ne) => Pushable::Yes, + (AttrKind::Field, AttrKind::B, _) | (AttrKind::B, AttrKind::Field, _) => { + Pushable::No("ordering comparator on B operand") + } + // Field = / <> BOOL / NULL: OK. + (AttrKind::Field, AttrKind::Bool, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::Bool, AttrKind::Field, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::Field, AttrKind::Null, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::Null, AttrKind::Field, CompareOp::Eq | CompareOp::Ne) => Pushable::Yes, + (AttrKind::Field, AttrKind::Bool | AttrKind::Null, _) + | (AttrKind::Bool | AttrKind::Null, AttrKind::Field, _) => { + Pushable::No("ordering on BOOL / NULL operand") + } + // Field vs. Field: the compiler emits $expr; pushable. + (AttrKind::Field, AttrKind::Field, _) => Pushable::Yes, + // Two literals — pushable but degenerate. + _ => Pushable::No("Compare with unusual operand kinds"), + } + } + Expr::And(l, r) => match (walk(l, maps), walk(r, maps)) { + (Pushable::Yes, Pushable::Yes) => Pushable::Yes, + (Pushable::No(r), _) | (_, Pushable::No(r)) => Pushable::No(r), + }, + Expr::Or(l, r) => match (walk(l, maps), walk(r, maps)) { + (Pushable::Yes, Pushable::Yes) => Pushable::Yes, + (Pushable::No(r), _) | (_, Pushable::No(r)) => Pushable::No(r), + }, + Expr::Not(inner) => { + // Only pushable when inner is exactly an existence check. + // Everything else — comparisons, functions, nested logic — + // is disallowed because MongoDB's $nor on missing paths + // returns true where DDB's three-valued logic returns false. + match inner.as_ref() { + Expr::Function { name, args } if args.len() == 1 => { + let n = name.to_lowercase(); + if n == "attribute_exists" || n == "attribute_not_exists" { + Pushable::Yes + } else { + Pushable::No("NOT around non-existence function") + } + } + _ => Pushable::No("NOT around non-existence expression"), + } + } + Expr::Between { .. } => Pushable::No("BETWEEN — analyzer coverage pending"), + Expr::In { .. } => Pushable::No("IN — analyzer coverage pending"), + Expr::Path(_) | Expr::Placeholder(_) | Expr::Arithmetic { .. } => { + Pushable::No("bare path/placeholder/arithmetic at top level") + } + } +} + +/// The compiler's operand-kind classification, used by the analyzer to +/// reason about type-mixing rules. +#[derive(Debug, Clone, Copy)] +enum AttrKind { + /// Reference to a document field (`Expr::Path`). + Field, + S, + N, + B, + Bool, + Null, +} + +fn operand_kind(expr: &Expr, maps: &ExpressionMaps) -> Option { + match expr { + Expr::Path(_) => Some(AttrKind::Field), + Expr::Placeholder(_) => value_kind(expr, maps), + _ => None, + } +} + +fn value_kind(expr: &Expr, maps: &ExpressionMaps) -> Option { + let Expr::Placeholder(name) = expr else { + return None; + }; + let av = maps.resolve_value(name).ok()?; + Some(match av { + AttributeValue::S(_) => AttrKind::S, + AttributeValue::N(_) => AttrKind::N, + AttributeValue::B(_) => AttrKind::B, + AttributeValue::Bool(_) => AttrKind::Bool, + AttributeValue::Null => AttrKind::Null, + _ => return None, + }) +} + +fn arg_resolves_to_scalar_type(expr: &Expr, maps: &ExpressionMaps, expected: AttrKind) -> bool { + matches!( + (value_kind(expr, maps), expected), + (Some(AttrKind::S), AttrKind::S) + | (Some(AttrKind::N), AttrKind::N) + | (Some(AttrKind::B), AttrKind::B) + | (Some(AttrKind::Bool), AttrKind::Bool) + | (Some(AttrKind::Null), AttrKind::Null) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::expression::PathElement; + use std::collections::HashMap; + + fn maps_with(values: &[(&str, AttributeValue)]) -> ExpressionMaps { + let mut m = HashMap::new(); + for (k, v) in values { + m.insert((*k).to_string(), v.clone()); + } + ExpressionMaps::new(HashMap::new(), m) + } + + fn path(name: &str) -> Expr { + Expr::Path(vec![PathElement::Attribute(name.to_string())]) + } + + #[test] + fn attribute_exists_is_pushable() { + let expr = Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + }; + assert_eq!(is_pushable(&expr, &maps_with(&[])), Pushable::Yes); + } + + #[test] + fn size_is_not_pushable() { + let expr = Expr::Function { + name: "size".into(), + args: vec![path("a")], + }; + assert!(!is_pushable(&expr, &maps_with(&[])).is_yes()); + } + + #[test] + fn number_operand_is_not_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":n".into())), + }; + let maps = maps_with(&[(":n", AttributeValue::N("42".into()))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn string_equality_is_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":s".into())), + }; + let maps = maps_with(&[(":s", AttributeValue::S("x".into()))]); + assert_eq!(is_pushable(&expr, &maps), Pushable::Yes); + } + + #[test] + fn binary_equality_is_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":b".into())), + }; + let maps = maps_with(&[(":b", AttributeValue::B(vec![0, 1, 2]))]); + assert_eq!(is_pushable(&expr, &maps), Pushable::Yes); + } + + #[test] + fn binary_ordering_is_not_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Lt, + right: Box::new(Expr::Placeholder(":b".into())), + }; + let maps = maps_with(&[(":b", AttributeValue::B(vec![0]))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn not_attribute_exists_is_pushable() { + let expr = Expr::Not(Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + })); + assert_eq!(is_pushable(&expr, &maps_with(&[])), Pushable::Yes); + } + + #[test] + fn not_around_comparison_is_not_pushable() { + let expr = Expr::Not(Box::new(Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":s".into())), + })); + let maps = maps_with(&[(":s", AttributeValue::S("x".into()))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn and_of_two_pushable_is_pushable() { + let expr = Expr::And( + Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + }), + Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("b")], + }), + ); + assert_eq!(is_pushable(&expr, &maps_with(&[])), Pushable::Yes); + } + + #[test] + fn and_taints_on_either_side() { + let expr = Expr::And( + Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + }), + Box::new(Expr::Function { + name: "size".into(), + args: vec![path("b")], + }), + ); + assert!(!is_pushable(&expr, &maps_with(&[])).is_yes()); + } + + #[test] + fn between_is_not_pushable() { + let expr = Expr::Between { + operand: Box::new(path("a")), + low: Box::new(Expr::Placeholder(":lo".into())), + high: Box::new(Expr::Placeholder(":hi".into())), + }; + let maps = maps_with(&[ + (":lo", AttributeValue::S("a".into())), + (":hi", AttributeValue::S("z".into())), + ]); + // BETWEEN is currently non-pushable pending analyzer coverage; + // this test locks in that decision. + assert!(!is_pushable(&expr, &maps).is_yes()); + } +} diff --git a/crates/storage-mongodb/tests/pushdown_parity.rs b/crates/storage-mongodb/tests/pushdown_parity.rs index 2efda64b..6e3cba54 100644 --- a/crates/storage-mongodb/tests/pushdown_parity.rs +++ b/crates/storage-mongodb/tests/pushdown_parity.rs @@ -34,6 +34,7 @@ use proptest::sample::select; use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement, evaluate_condition}; use extenddb_core::types::AttributeValue; use extenddb_storage_mongodb::condition::condition_to_filter; +use extenddb_storage_mongodb::pushdown::{Pushable, is_pushable}; use common::eval_filter; @@ -115,19 +116,15 @@ fn arb_placeholder_ref(idx: usize) -> Expr { /// The placeholders are numbered per-expression starting at :v0. When /// composing with AND/OR (below), we renumber to keep them globally unique. /// -/// Binary (`.B`) operands are currently excluded: the compiler emits raw -/// BSON `Binary` filters for `.B` comparisons, but `item_to_document` -/// (in production) stores `.B` fields as base64-encoded strings via the -/// AttributeValue JSON serializer. The two formats never match — a -/// pre-existing bug in the compiler / storage-layer contract. A5 step 3 -/// will either fix the compiler to emit string filters (matching storage) -/// or fix the storage to emit BSON binary (matching the compiler), and -/// re-enable `.B` comparisons in this harness. +/// `.B` operands are re-enabled here as of A5 step 3, which fixed +/// `av_to_bson` in `condition.rs` to emit the base64 string form that +/// matches how storage writes `.B` fields. #[allow(clippy::redundant_closure)] fn arb_leaf_expr() -> impl Strategy)> { // We union several leaf shapes. Each yields (Expr, values-list). let name = || arb_name_expr(); let str_val = || arb_short_str().prop_map(AttributeValue::S); + let bytes_val = || arb_short_bytes().prop_map(AttributeValue::B); prop_oneof![ // attribute_exists(name) @@ -218,6 +215,33 @@ fn arb_leaf_expr() -> impl Strategy)> { }, vec![v], )), + // name = :v (B) — base64-string equality equals underlying-byte + // equality, so pushdown is correct for `.B` under `=`. + (name(), bytes_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Eq, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name <> :v (B) — same, complement of equality. + (name(), bytes_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Ne, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // Note: `<`/`<=`/`>`/`>=` on `.B` is NOT in the pushable subset. + // The compiler emits a base64 string comparison, but DDB + // compares binary values bytewise. Base64 preserves byte + // ordering only for equal-length inputs; different-length + // inputs can invert the order (e.g. bytes [255] > bytes [0, 0] + // bytewise, but "/w==" < "AAA=" lexicographically). Step 3's + // analyzer marks binary ordering as NotPushable and falls + // back to session-scoped in-Rust evaluation. ] } @@ -244,10 +268,10 @@ fn arb_composed_expr() -> impl Strategy)> { fn renumber_placeholders(expr: Expr, offset: usize) -> Expr { match expr { Expr::Placeholder(name) => { - if let Some(idx_str) = name.strip_prefix(":v") { - if let Ok(idx) = idx_str.parse::() { - return Expr::Placeholder(format!(":v{}", idx + offset)); - } + if let Some(idx_str) = name.strip_prefix(":v") + && let Ok(idx) = idx_str.parse::() + { + return Expr::Placeholder(format!(":v{}", idx + offset)); } Expr::Placeholder(name) } @@ -374,6 +398,53 @@ proptest! { mongo_result, ); } + + /// The analyzer's soundness contract: for every expression it marks + /// `Pushable::Yes`, the compiled filter must agree with the DDB + /// evaluator on the generated item. + /// + /// This is a stricter check than the first property because the + /// analyzer whitelists a specific subset — if it marks Yes on an + /// expression whose compiled filter drifts from DDB, that's an + /// analyzer bug (whitelist too generous). The generator draws from + /// the same pushable grammar as the first property so most cases + /// hit the analyzer's Yes branch; when generation drifts into + /// non-pushable AST shapes (e.g., unhandled operand-type interactions + /// in composed expressions), the analyzer says No and the test + /// short-circuits without a comparison. + #[test] + fn analyzer_yes_implies_filter_parity( + item in arb_item(), + expr_pair in arb_expr(), + ) { + let (expr, values_vec) = expr_pair; + let mut values = HashMap::new(); + for (idx, v) in values_vec.iter().enumerate() { + values.insert(format!(":v{idx}"), v.clone()); + } + let maps = ExpressionMaps::new(HashMap::new(), values); + + if !matches!(is_pushable(&expr, &maps), Pushable::Yes) { + return Ok(()); + } + + let filter = condition_to_filter(&expr, &maps).expect( + "analyzer said Yes but compiler failed — analyzer/compiler drift", + ); + let bson_doc = item_to_bson_doc(&item); + let mongo_result = eval_filter(&filter, &bson_doc); + let ddb_result = evaluate_condition(&expr, &item, &maps).unwrap_or(false); + + prop_assert_eq!( + ddb_result, + mongo_result, + "analyzer said Pushable::Yes but results diverged:\n expr: {:?}\n filter: {:?}\n ddb: {}, mongo: {}", + expr, + filter, + ddb_result, + mongo_result, + ); + } } // Silence unused-import lint from `common` when running only a subset. From 788a157d3778f0f0e148255fa35ba25d16ec0a79 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 12:31:54 -0700 Subject: [PATCH 19/83] fix(mongodb): reject ProvisionedThroughput on PAY_PER_REQUEST in UpdateTable Real DynamoDB returns ValidationException with 'Neither ReadCapacityUnits nor WriteCapacityUnits can be specified when BillingMode is PAY_PER_REQUEST' when an UpdateTable request supplies ProvisionedThroughput while the effective billing mode is PAY_PER_REQUEST. Mongo backend was silently accepting the mismatch. Mirror the postgres check at update_table.rs:43-75. Effective mode is the requested billing_mode when the request changes it, otherwise the table's current mode read from the catalog. Failing test: test_update_table_billing_validation.py::test_update_table_rejects_throughput_on_pay_per_request_table. --- crates/storage-mongodb/src/table_engine.rs | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 633037ce..f77424e0 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -557,6 +557,39 @@ impl MongoEngine { let tables_coll = self.catalog_db.collection::("tables"); + // Reject ProvisionedThroughput when the effective billing mode is + // PAY_PER_REQUEST. The effective mode is the requested billing_mode + // when the request changes it, otherwise the table's current mode. + // Real DynamoDB returns "Neither ReadCapacityUnits nor WriteCapacityUnits + // can be specified when BillingMode is PAY_PER_REQUEST". Postgres does + // this same check under a FOR UPDATE row lock in update_table.rs; mongo + // reads the current billing_mode via find_one and relies on the fact + // that any concurrent billing-mode change would then be rejected by its + // own no-op check (not yet implemented — see R-8 followup). + if input.provisioned_throughput.is_some() { + let effective_ppr = match input.billing_mode { + Some(BillingMode::PayPerRequest) => true, + Some(BillingMode::Provisioned) => false, + None => { + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name }, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + table_doc + .and_then(|d| d.get_str("billing_mode").ok().map(str::to_owned)) + .as_deref() + == Some("PAY_PER_REQUEST") + } + }; + if effective_ppr { + return Err(StorageError::Validation( + "One or more parameter values were invalid: Neither ReadCapacityUnits nor WriteCapacityUnits can be specified when BillingMode is PAY_PER_REQUEST".to_owned(), + )); + } + } + // Build update document let mut update_doc = Document::new(); From 422e3bd907cff617ad2f4d39f9d949063be8e06c Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 12:37:15 -0700 Subject: [PATCH 20/83] fix(mongodb): persist and return TableClass, SSESpecification, OnDemandThroughput MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mongo backend hardcoded sse_description, table_class_summary, and on_demand_throughput to None during rebase resolution — none of the three were persisted on CreateTable, so DescribeTable returned an empty TableClassSummary, no SSEDescription, and no OnDemandThroughput. - table_engine.rs::create_table_impl now persists table_class, sse_specification, on_demand_throughput on the catalog document. - table_engine.rs::doc_to_table_description reads them back and synthesizes TableClassSummary / SSEDescription / OnDemandThroughput in the same shape as postgres (table_helpers.rs:329-353). - table_engine.rs::update_table_impl accepts table_class and on_demand_throughput on UpdateTable (SSE is create-only per DDB). - backup_engine.rs::create_backup preserves the three fields on the backup catalog document, and restore_table_from_backup replays them into CreateTableInput. SSEDescription synthesis matches postgres: 'Enabled: true' becomes status=ENABLED with a synthesized KMS ARN; anything else omits the field. Failing tests (5): - test_config_fields.py::TestTableClass::test_table_class_infrequent_access - test_config_fields.py::TestTableClass::test_update_table_class - test_config_fields.py::TestSSESpecification::test_sse_enabled_round_trips - test_config_fields.py::TestOnDemandThroughput::test_on_demand_throughput_create - test_config_fields.py::TestOnDemandThroughput::test_update_on_demand_throughput --- crates/storage-mongodb/src/backup_engine.rs | 45 ++++++++- crates/storage-mongodb/src/table_engine.rs | 102 ++++++++++++++++++-- 2 files changed, 136 insertions(+), 11 deletions(-) diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index c75f22f7..23de03c6 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -97,6 +97,22 @@ impl BackupEngine for MongoEngine { let table_size = table_doc.get_i64("table_size_bytes").unwrap_or(0); let item_count = table_doc.get_i64("item_count").unwrap_or(0); + // Preserve TableClass / SSESpecification / OnDemandThroughput so + // RestoreTableFromBackup can recreate the table with the same + // configuration. + let table_class_bson = table_doc + .get("table_class") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null); + let sse_spec_bson = table_doc + .get("sse_specification") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null); + let on_demand_bson = table_doc + .get("on_demand_throughput") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null); + let backup_arn = format!( "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{ts}", region = self.region, @@ -155,6 +171,9 @@ impl BackupEngine for MongoEngine { "billing_mode": &billing_mode, "created_at": mongodb::bson::DateTime::now(), "table_creation_date_time": created_at, + "table_class": table_class_bson, + "sse_specification": sse_spec_bson, + "on_demand_throughput": on_demand_bson, }; backups_coll @@ -408,6 +427,26 @@ impl BackupEngine for MongoEngine { Some(extenddb_core::types::BillingMode::Provisioned) }; + // Preserve the source table's TableClass / SSESpecification / + // OnDemandThroughput settings when recreating. + let table_class = backup_doc.get_str("table_class").ok().map(str::to_owned); + let sse_specification: Option = + backup_doc.get("sse_specification").and_then(|b| { + if matches!(b, mongodb::bson::Bson::Null) { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + let on_demand_throughput: Option = + backup_doc.get("on_demand_throughput").and_then(|b| { + if matches!(b, mongodb::bson::Bson::Null) { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + let create_input = extenddb_core::types::CreateTableInput { table_name: target_table_name.clone(), key_schema, @@ -422,9 +461,9 @@ impl BackupEngine for MongoEngine { stream_specification: None, tags: None, deletion_protection_enabled: None, - sse_specification: None, - table_class: None, - on_demand_throughput: None, + sse_specification, + table_class, + on_demand_throughput, }; let desc = self.create_table(&account_id, create_input).await?; diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index f77424e0..8ab380f2 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -11,8 +11,8 @@ use mongodb::options::{Collation, CollationStrength, IndexOptions}; use extenddb_core::types::{ BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeyType, ListTablesInput, ListTablesOutput, - LsiDescription, ProvisionedThroughputDescription, ScalarAttributeType, TableDescription, - TableKeyInfo, TableStatus, UpdateTableInput, + LsiDescription, OnDemandThroughput, ProvisionedThroughputDescription, ScalarAttributeType, + SseDescription, SseType, TableDescription, TableKeyInfo, TableStatus, UpdateTableInput, }; use extenddb_storage::TableEngine; use extenddb_storage::error::StorageError; @@ -164,6 +164,23 @@ impl MongoEngine { .as_ref() .map_or(bson::Bson::Null, |l| bson::Bson::String(l.clone())); + // Persist TableClass / SSESpecification / OnDemandThroughput on the + // catalog doc so DescribeTable can return TableClassSummary, + // SSEDescription, and OnDemandThroughput respectively. Mirrors the + // postgres backend at storage-postgres/src/create_table.rs:100-102. + let table_class_bson = input + .table_class + .as_deref() + .map_or(bson::Bson::Null, |tc| bson::Bson::String(tc.to_owned())); + let sse_spec_bson = input.sse_specification.as_ref().map_or_else( + || bson::Bson::Null, + |v| bson::to_bson(v).unwrap_or(bson::Bson::Null), + ); + let on_demand_bson = input.on_demand_throughput.as_ref().map_or_else( + || bson::Bson::Null, + |v| bson::to_bson(v).unwrap_or(bson::Bson::Null), + ); + let table_doc = doc! { "_id": { "account_id": account_id, "table_name": &input.table_name }, "key_schema": key_schema_bson, @@ -180,6 +197,9 @@ impl MongoEngine { "deletion_protection_enabled": deletion_protection, "ttl_attribute": bson::Bson::Null, "stream_label": stream_label_bson, + "table_class": table_class_bson, + "sse_specification": sse_spec_bson, + "on_demand_throughput": on_demand_bson, }; let tables_coll = self.catalog_db.collection::("tables"); @@ -389,6 +409,33 @@ impl MongoEngine { } } + // Derive the SSEDescription from the SSESpecification, mirroring the + // postgres backend (table_helpers.rs:329-346). The specification's + // `Enabled: true` becomes a KMS-status ENABLED description with a + // synthesized ARN. Anything else omits the field. + let sse_description = input.sse_specification.as_ref().and_then(|spec| { + let enabled = spec + .get("Enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if enabled { + Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some(format!( + "arn:aws:kms:{}:{}:key/default", + self.region, account_id + )), + }) + } else { + None + } + }); + let table_class_summary = input + .table_class + .as_deref() + .map(|tc| serde_json::json!({ "TableClass": tc })); + Ok(TableDescription { table_name: input.table_name, key_schema: input.key_schema, @@ -407,9 +454,9 @@ impl MongoEngine { latest_stream_arn: stream_arn_opt, latest_stream_label: stream_label_opt, deletion_protection_enabled: deletion_protection, - sse_description: None, - table_class_summary: None, - on_demand_throughput: None, + sse_description, + table_class_summary, + on_demand_throughput: input.on_demand_throughput, }) } @@ -610,6 +657,15 @@ impl MongoEngine { update_doc.insert("deletion_protection_enabled", dp); } + if let Some(tc) = &input.table_class { + update_doc.insert("table_class", tc); + } + + if let Some(odt) = &input.on_demand_throughput { + let odt_bson = bson::to_bson(odt).map_err(|e| StorageError::Internal(e.to_string()))?; + update_doc.insert("on_demand_throughput", odt_bson); + } + if let Some(ss) = &input.stream_specification { let ss_bson = bson::to_bson(ss).map_err(|e| StorageError::Internal(e.to_string()))?; update_doc.insert("stream_specification", ss_bson); @@ -1045,6 +1101,36 @@ impl MongoEngine { .as_ref() .map(|label| stream_arn(&self.region, account_id, &table_name, label)); + // TableClass / SSEDescription / OnDemandThroughput — read back the + // fields persisted at CreateTable time. Same shape as postgres' + // table_helpers.rs:329-353. + let table_class_summary = doc + .get_str("table_class") + .ok() + .map(|tc| serde_json::json!({ "TableClass": tc })); + let sse_description = doc.get("sse_specification").and_then(|b| { + let spec: serde_json::Value = bson::from_bson(b.clone()).ok()?; + let enabled = spec + .get("Enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if enabled { + Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some(format!( + "arn:aws:kms:{}:{}:key/default", + self.region, account_id + )), + }) + } else { + None + } + }); + let on_demand_throughput: Option = doc + .get("on_demand_throughput") + .and_then(|b| bson::from_bson(b.clone()).ok()); + Ok(TableDescription { table_name, key_schema, @@ -1063,9 +1149,9 @@ impl MongoEngine { latest_stream_arn: stream_arn_opt, latest_stream_label: stream_label, deletion_protection_enabled: deletion_protection, - sse_description: None, - table_class_summary: None, - on_demand_throughput: None, + sse_description, + table_class_summary, + on_demand_throughput, }) } } From 8cc46da7fa815bc462947fd60326c37518ce6f17 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 13:03:37 -0700 Subject: [PATCH 21/83] fix(mongodb): scope stream shards by table_id to prevent cross-tenant disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two tenants creating tables with the same name would previously get shard_ids of the form 'shardId-orders-000000000000' colliding in the shared stream_records collection. A caller in one account could synthesize the shard_id belonging to the other account and read its stream records via GetRecords, which flows through get_stream_records(shard_id) without account_id or validate_shard. Change the shard_id format to embed table_id (a UUID minted at CreateTable), not table_name: - stream_engine::build_shard_id — new function 'shardId-{table_id}-{i:012}'. UUIDs are not guessable so an attacker cannot synthesize another tenant's shard_id. - init_stream_shards signature loses table_name; two call sites updated. - delete_table_impl now cascades cleanup of stream_shards, stream_records, and per-shard sequence counters for the deleted table_id. Prevents a table recreated with the same name (which gets a fresh table_id) from inheriting the deleted table's stream history. - bootstrapper adds a unique index on stream_shards.shard_id so a concurrent re-init cannot insert a duplicate shard document. Three new unit tests cover the new shard_id format, cross-table non-collision, and stable ordering within a table. --- crates/storage-mongodb/src/bootstrapper.rs | 18 +++ crates/storage-mongodb/src/stream_engine.rs | 122 +++++++++++++++++++- crates/storage-mongodb/src/table_engine.rs | 15 ++- 3 files changed, 145 insertions(+), 10 deletions(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index 93657a82..132541b4 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -102,6 +102,24 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("Failed to create TTL index: {e}")))?; + // stream_shards: unique index on shard_id so a subsequent init/ + // recreate can never insert a duplicate shard document under the + // same shard_id. Combined with `table_id`-derived shard_ids + // (see stream_engine::build_shard_id) this rules out cross-tenant + // shard collisions structurally. + db.create_collection("stream_shards") + .await + .map_err(|e| OpError::Internal(format!("Failed to create stream_shards: {e}")))?; + db.collection::("stream_shards") + .create_index( + IndexModel::builder() + .keys(doc! { "shard_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_shards shard_id index: {e}")))?; + Ok(()) } diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index cb9b51c2..a18b2381 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -41,17 +41,35 @@ pub(crate) fn event_name_ddb_str(name: StreamEventName) -> &'static str { } } +/// Build the mongo `stream_shards.shard_id` for a given table_id + shard index. +/// +/// **Security invariant (RFC-0003 §5.3, §8.2):** shard_id must incorporate +/// the table's globally-unique `table_id` (a UUID), not the caller-visible +/// `table_name`. Table names are only unique per-account, so two accounts +/// creating "orders" tables would generate colliding shard_ids under a +/// name-derived scheme, letting one account's `GetRecords(shard_id)` read +/// the other's stream records. `table_id` is a per-table-instance UUID that +/// resets on `DeleteTable + CreateTable` — the recreated table gets fresh +/// shard_ids, so leftover stream records from the deleted table never +/// resurface either. +/// +/// UUIDs are not guessable, so an attacker cannot synthesize a shard_id +/// belonging to another tenant without first observing it (which itself +/// requires an authenticated path scoped to that tenant's account). +pub(crate) fn build_shard_id(table_id: &str, shard_index: u32) -> String { + format!("shardId-{table_id}-{shard_index:012}") +} + impl MongoEngine { /// Initialize stream shards for a table. Only creates shard documents; /// the caller is responsible for setting `stream_label` on the table doc. - pub(crate) async fn init_stream_shards( - &self, - table_name: &str, - table_id: &str, - ) -> Result<(), StorageError> { + /// + /// Uses the table's UUID (`table_id`), not `table_name`, in the shard_id + /// — see `build_shard_id` for the security rationale. + pub(crate) async fn init_stream_shards(&self, table_id: &str) -> Result<(), StorageError> { let shards_coll = self.data_db.collection::("stream_shards"); for i in 0..SHARDS_PER_STREAM { - let shard_id = format!("shardId-{table_name}-{i:012}"); + let shard_id = build_shard_id(table_id, i); let start_seq = format!("{:021}", 0); shards_coll .insert_one(doc! { @@ -65,6 +83,61 @@ impl MongoEngine { } Ok(()) } + + /// Delete every stream_shards document for a given table_id, and every + /// stream_records document written to any of its shards. Invoked from + /// `delete_table_impl` so that a table recreated with the same name + /// (which will get a fresh table_id) cannot inherit the deleted table's + /// stream history. + pub(crate) async fn cleanup_stream_state_for_table( + &self, + table_id: &str, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + let records_coll = self.data_db.collection::("stream_records"); + + // Collect shard_ids for this table so we can delete their records. + // Records don't carry table_id directly — they're addressed by shard_id. + let cursor = shards_coll + .find(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_ids: Vec = shard_docs + .iter() + .filter_map(|d| d.get_str("shard_id").ok().map(str::to_owned)) + .collect(); + + if !shard_ids.is_empty() { + records_coll + .delete_many(doc! { "shard_id": { "$in": &shard_ids } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + shards_coll + .delete_many(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Sequence-number counters are keyed as "stream_seq:". + let counters_coll = self.data_db.collection::("counters"); + let counter_ids: Vec = shard_ids + .iter() + .map(|sid| format!("stream_seq:{sid}")) + .collect(); + if !counter_ids.is_empty() { + counters_coll + .delete_many(doc! { "_id": { "$in": &counter_ids } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(()) + } } impl StreamEngine for MongoEngine { @@ -556,4 +629,41 @@ mod tests { assert_eq!(event_name_ddb_str(StreamEventName::Modify), "MODIFY"); assert_eq!(event_name_ddb_str(StreamEventName::Remove), "REMOVE"); } + + #[test] + fn shard_id_embeds_table_id_not_table_name() { + let table_id = "550e8400-e29b-41d4-a716-446655440000"; + assert_eq!( + build_shard_id(table_id, 0), + "shardId-550e8400-e29b-41d4-a716-446655440000-000000000000" + ); + assert_eq!( + build_shard_id(table_id, 3), + "shardId-550e8400-e29b-41d4-a716-446655440000-000000000003" + ); + } + + #[test] + fn shard_ids_for_different_table_ids_do_not_collide() { + // Regression test for RFC-0003 §5.3 (account and table isolation). + // Two tables with the same shard index (0) must have different + // shard_ids so a caller in one tenant cannot address the other's + // shard. + let table_a = "550e8400-e29b-41d4-a716-446655440000"; + let table_b = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + assert_ne!(build_shard_id(table_a, 0), build_shard_id(table_b, 0)); + } + + #[test] + fn shard_id_format_is_stable_across_shards_of_same_table() { + // Same table_id, different shard index → deterministic ordering. + let table_id = "550e8400-e29b-41d4-a716-446655440000"; + let s0 = build_shard_id(table_id, 0); + let s1 = build_shard_id(table_id, 1); + let s2 = build_shard_id(table_id, 2); + let s3 = build_shard_id(table_id, 3); + assert!(s0 < s1); + assert!(s1 < s2); + assert!(s2 < s3); + } } diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 8ab380f2..dca7a03e 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -254,10 +254,11 @@ impl MongoEngine { .map_err(|e| StorageError::Internal(e.to_string()))?; } - // Initialize stream shards if streaming is enabled + // Initialize stream shards if streaming is enabled. shard_id is + // derived from table_id (UUID), never table_name — see + // stream_engine::build_shard_id for the security rationale. if stream_label_opt.is_some() { - self.init_stream_shards(&input.table_name, &table_id) - .await?; + self.init_stream_shards(&table_id).await?; } // Handle GSI creation @@ -504,6 +505,12 @@ impl MongoEngine { self.gsi_cache_invalidate(&desc.table_id); + // Delete stream_shards, stream_records, and their sequence counters + // for this table. Prevents a table recreated with the same name + // (which will get a fresh table_id) from inheriting the deleted + // table's stream history. RFC-0003 §8.2 (table-name reuse). + self.cleanup_stream_state_for_table(&desc.table_id).await?; + // Delete the table metadata tables_coll .delete_one( @@ -682,7 +689,7 @@ impl MongoEngine { .format(&time::format_description::well_known::Iso8601::DEFAULT) .unwrap_or_else(|_| "unknown".to_string()); update_doc.insert("stream_label", &label); - self.init_stream_shards(&input.table_name, table_id).await?; + self.init_stream_shards(table_id).await?; } } From dac66dd01c1851a35b90a978ef102454d96d85f9 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 13:10:50 -0700 Subject: [PATCH 22/83] fix(mongodb): use netstring encoding for composite _id to avoid delimiter collisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document _id for tables with both partition and sort keys was previously built as "{pk_text}#{sk_text}". A raw '#' delimiter is collision-prone because '#' can appear in either component. Two distinct DynamoDB items `{pk: "a#b", sk: "c"}` and `{pk: "a", sk: "b#c"}` both produced _id = "a#b#c", so the second insert either failed with E11000 (misclassified as ConditionalCheckFailedException) or an unconditional replace silently targeted the wrong document. Scan pagination with `_id > start` was similarly ambiguous. Add a `composite_id(pk_text, sk_text)` helper that uses netstring encoding (already available via extenddb_storage::util::encode_netstring_composite): each part is length-prefixed, so the boundary between pk and sk is unambiguous regardless of contents. Two call sites updated: - item_to_document — the write-time _id. - scan_impl — the pagination cursor. Three unit tests: delimiter-collision non-collision, stable encoding on normal inputs, and end-to-end that item_to_document emits the netstring form. The pk-only table branch (`_id = pk_text`) is unchanged — a single value is trivially collision-free. --- crates/storage-mongodb/src/data/mod.rs | 58 ++++++++++++++++++++++- crates/storage-mongodb/src/data_engine.rs | 5 +- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index 5d8d0750..f0d482e3 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -13,13 +13,27 @@ use extenddb_core::types::{ AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, }; use extenddb_storage::error::StorageError; -use extenddb_storage::util::{composite_pk_to_text, pk_to_text, sk_info}; +use extenddb_storage::util::{ + composite_pk_to_text, encode_netstring_composite, pk_to_text, sk_info, +}; /// Returns the `MongoDB` collection name for a `DynamoDB` table. pub fn data_collection_name(table_id: &str) -> String { format!("_ddb_{table_id}") } +/// Build the mongo document `_id` for a composite (partition + sort) key. +/// +/// Uses netstring encoding — `:,:,` — so the boundary +/// between `pk` and `sk` is unambiguous regardless of the contents of +/// either. A naive `"{pk}#{sk}"` scheme collides when `pk` or `sk` contains +/// the delimiter (e.g., `pk="a#b", sk="c"` and `pk="a", sk="b#c"` both +/// produce `"a#b#c"`). +#[must_use] +pub fn composite_id(pk_text: &str, sk_text: &str) -> String { + encode_netstring_composite(&[pk_text.to_owned(), sk_text.to_owned()]) +} + /// Returns the `MongoDB` collection name for a secondary index. pub fn index_collection_name(index_id: &str) -> String { format!("_ddb_{index_id}") @@ -48,7 +62,9 @@ pub fn item_to_document( .get(sk_name) .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; let sk_text = sk_to_text(sk_value)?; - doc.insert("_id", format!("{pk_text}#{sk_text}")); + // Netstring-encoded composite _id — see composite_id() for why the + // naive "{pk}#{sk}" form is collision-prone. + doc.insert("_id", composite_id(&pk_text, &sk_text)); doc.insert("pk", pk_text); // Insert the typed sort key field @@ -264,4 +280,42 @@ mod tests { let err = pk_filter(&key, &schema, &attrs).unwrap_err(); assert!(matches!(err, StorageError::Validation(_))); } + + #[test] + fn composite_id_disambiguates_delimiter_in_pk_or_sk() { + // Two items whose naive "{pk}#{sk}" strings would collide must + // produce distinct netstring-encoded _ids. + let a = composite_id("a#b", "c"); + let b = composite_id("a", "b#c"); + assert_ne!( + a, b, + "composite _id must not collide on delimiter-containing keys" + ); + } + + #[test] + fn composite_id_stable_on_normal_inputs() { + // Reasonable inputs still round-trip through netstring cleanly. + assert_eq!( + composite_id("user1", "2024-01-01"), + "5:user1,10:2024-01-01," + ); + assert_eq!(composite_id("", "sk"), "0:,2:sk,"); + assert_eq!(composite_id("pk", ""), "2:pk,0:,"); + } + + #[test] + fn composite_id_is_written_by_item_to_document() { + let (schema, attrs) = schema_pk_str_sk_num(); + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("user1".to_owned())); + item.insert("sk".to_owned(), AttributeValue::N("42".to_owned())); + + let doc = item_to_document(&item, &schema, &attrs).unwrap(); + let id = doc.get_str("_id").unwrap(); + assert!( + id.starts_with("5:user1,"), + "expected netstring-encoded _id, got {id:?}" + ); + } } diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index d4887067..843bc294 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -27,7 +27,8 @@ use extenddb_storage::{ use crate::MongoEngine; use crate::condition::condition_to_filter; use crate::data::{ - data_collection_name, document_to_item, item_to_document, pk_filter, sk_field_name, + composite_id, data_collection_name, document_to_item, item_to_document, pk_filter, + sk_field_name, }; use crate::pushdown::{Pushable, is_pushable}; @@ -1020,7 +1021,7 @@ impl MongoEngine { } _ => return Err(StorageError::Internal("invalid sk type".to_string())), }; - let start_id = format!("{start_pk}#{sk_text}"); + let start_id = composite_id(&start_pk, &sk_text); filter.insert("_id", doc! { "$gt": start_id }); } } else { From f0b4ed9bd4bf0f6897f37c54145392e1d746079e Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 13:16:06 -0700 Subject: [PATCH 23/83] fix(mongodb): propagate stream records and GSI updates inside TransactWriteItems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The storage trait contract at crates/storage/src/lib.rs states that when 'stream' is Some on a TransactWriteOp, the stream record for that op is inserted in the same transaction as the data write. The mongo backend was dropping both the stream field and the GSI propagation on every transact op — an application using TransactWriteItems on a streams-enabled or GSI-bearing table would commit the base rows but silently skip every stream record and every index update. Fix by threading stream through OwnedTransactWriteOp for Put/Delete/Update (ConditionCheck has no data mutation and no stream in the trait), and calling sync_indexes_in_session and write_stream_inline_in_session after every write inside the same ClientSession used by the transact. Also fetch existing_item unconditionally on Put and Delete (previously only when a condition was present) so: - sync_indexes_in_session can delete stale GSI entries when the write changes or removes an indexed attribute. - write_stream_inline_in_session receives the correct OldImage for MODIFY records. For Update, propagate is_creating (existing_item.is_none() before apply_update) so the stream layer emits INSERT with no OldImage rather than a MODIFY with a synthesized key-only pre-image. For Delete, skip the stream write when existing_item is None — DDB does not emit a stream record for a delete on a nonexistent key. Existing unit tests continue to pass. Integration coverage is transitive via the pytest transaction suites. --- crates/storage-mongodb/src/data_engine.rs | 140 ++++++++++++++++++---- 1 file changed, 116 insertions(+), 24 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 843bc294..63a41057 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1681,7 +1681,7 @@ impl MongoEngine { condition, maps, return_values_on_ccf, - .. + stream, } => { validation::validate_item_keys( item, @@ -1698,18 +1698,24 @@ impl MongoEngine { pk_filter(item, &key_info.key_schema, &key_info.attribute_definitions) .map_err(TransactOpError::Storage)?; + // Always fetch the pre-image. Needed to (a) evaluate any + // condition against it, (b) let sync_indexes_in_session delete + // stale index entries when this write changes or removes a + // GSI key attribute, and (c) supply OldImage to any attached + // stream capture. let existing_doc = coll .find_one(key_filter.clone()) .session(&mut *session) .await .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) + } else { + None + }; + if let Some(cond) = condition { - let existing_item = if let Some(doc) = existing_doc.as_ref() { - Some(document_to_item(doc).map_err(TransactOpError::Storage)?) - } else { - None - }; let for_eval = existing_item.clone().unwrap_or_default(); let passed = expression::evaluate_condition(cond, &for_eval, maps).map_err(|e| { @@ -1740,6 +1746,31 @@ impl MongoEngine { .await .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + // Propagate to secondary indexes and the stream within the + // same transaction session — otherwise a transactional write + // to a streams-enabled or GSI-bearing table would commit + // the base row while silently dropping its dependent side + // effects. + self.sync_indexes_in_session( + key_info, + existing_item.as_ref(), + Some(item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + existing_item.as_ref(), + Some(item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + } + Ok(()) } OwnedTransactWriteOp::Delete { @@ -1748,7 +1779,7 @@ impl MongoEngine { condition, maps, return_values_on_ccf, - .. + stream, } => { validation::validate_key_only( key, @@ -1765,20 +1796,22 @@ impl MongoEngine { pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) .map_err(TransactOpError::Storage)?; - if let Some(cond) = condition { - let existing_doc = coll - .find_one(key_filter.clone()) - .session(&mut *session) - .await - .map_err(|e| { - TransactOpError::Storage(StorageError::Internal(e.to_string())) - })?; + // Always fetch the pre-image. Needed for condition evaluation, + // stale-index deletion in sync_indexes_in_session, and OldImage + // capture for any attached stream. + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; - let existing_item = if let Some(doc) = existing_doc.as_ref() { - Some(document_to_item(doc).map_err(TransactOpError::Storage)?) - } else { - None - }; + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) + } else { + None + }; + + if let Some(cond) = condition { let for_eval = existing_item.clone().unwrap_or_default(); let passed = expression::evaluate_condition(cond, &for_eval, maps).map_err(|e| { @@ -1801,6 +1834,28 @@ impl MongoEngine { .await .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + // Propagate to secondary indexes and the stream within the + // same transaction session. + self.sync_indexes_in_session(key_info, existing_item.as_ref(), None, &mut *session) + .await + .map_err(TransactOpError::Storage)?; + if let Some(capture) = stream { + // DDB semantics: a delete on a non-existent key is a + // no-op, and no stream record is emitted. Guard on + // existing_item.is_some() to match. + if existing_item.is_some() { + self.write_stream_inline_in_session( + key_info, + capture, + existing_item.as_ref(), + None, + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + } + } + Ok(()) } OwnedTransactWriteOp::Update { @@ -1810,7 +1865,7 @@ impl MongoEngine { condition, maps, return_values_on_ccf, - .. + stream, } => { validation::validate_key_only( key, @@ -1838,6 +1893,7 @@ impl MongoEngine { } else { None }; + let is_creating = existing_item.is_none(); let mut item = existing_item.clone().unwrap_or_else(|| key.clone()); @@ -1881,6 +1937,36 @@ impl MongoEngine { .await .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + // Propagate to secondary indexes and the stream within the + // same transaction session. When the update creates the + // item (previously did not exist), pass None as the old + // image so the stream layer produces an INSERT record, not + // a MODIFY with a synthesized key-only OldImage. + self.sync_indexes_in_session( + key_info, + existing_item.as_ref(), + Some(&item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + if let Some(capture) = stream { + let old_for_stream = if is_creating { + None + } else { + existing_item.as_ref() + }; + self.write_stream_inline_in_session( + key_info, + capture, + old_for_stream, + Some(&item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + } + Ok(()) } OwnedTransactWriteOp::ConditionCheck { @@ -2167,6 +2253,7 @@ enum OwnedTransactWriteOp { condition: Option, maps: ExpressionMaps, return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + stream: Option, }, Delete { key_info: TableKeyInfo, @@ -2174,6 +2261,7 @@ enum OwnedTransactWriteOp { condition: Option, maps: ExpressionMaps, return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + stream: Option, }, Update { key_info: TableKeyInfo, @@ -2182,6 +2270,7 @@ enum OwnedTransactWriteOp { condition: Option, maps: ExpressionMaps, return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + stream: Option, }, ConditionCheck { key_info: TableKeyInfo, @@ -2200,13 +2289,14 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { condition, maps, return_values_on_ccf, - .. + stream, } => OwnedTransactWriteOp::Put { key_info: (*key_info).clone(), item: (*item).clone(), condition: condition.cloned(), maps: (*maps).clone(), return_values_on_ccf: *return_values_on_ccf, + stream: stream.clone(), }, TransactWriteOp::Delete { key_info, @@ -2214,13 +2304,14 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { condition, maps, return_values_on_ccf, - .. + stream, } => OwnedTransactWriteOp::Delete { key_info: (*key_info).clone(), key: (*key).clone(), condition: condition.cloned(), maps: (*maps).clone(), return_values_on_ccf: *return_values_on_ccf, + stream: stream.clone(), }, TransactWriteOp::Update { key_info, @@ -2229,7 +2320,7 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { condition, maps, return_values_on_ccf, - .. + stream, } => OwnedTransactWriteOp::Update { key_info: (*key_info).clone(), key: (*key).clone(), @@ -2237,6 +2328,7 @@ fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { condition: condition.cloned(), maps: (*maps).clone(), return_values_on_ccf: *return_values_on_ccf, + stream: stream.clone(), }, TransactWriteOp::ConditionCheck { key_info, From 5c0a5c4db5f306567af67bd9d5bc0703766f9636 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 13:20:36 -0700 Subject: [PATCH 24/83] fix(mongodb): assign stream sequence numbers inside the write transaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sequence-number assignment was calling next_sequence_number outside of any transaction session. A fast writer B whose transaction started later could obtain seq=6 and commit before a slow writer A (which had obtained seq=5) committed. A consumer polling with after_sequence_number=cursor between B's commit and A's commit would see seq=6 and advance its iterator past it; when A finally committed, seq=5 landed behind the cursor and was never returned by GetRecords. Silent, permanent record loss. Add next_sequence_number_in_session and assign_shard_in_session on MongoEngine that take &mut ClientSession, so the counter increment and the shard lookup participate in the same transaction as the data write and the record insert. Under snapshot isolation this also serializes concurrent writers on the same shard: two writers racing to $inc the same counter document conflict at commit time and one retries — the transaction retry loop handles this. Update write_stream_inline_in_session to use the new session-scoped helpers. It no longer needs to re-read the tables catalog to resolve table_id — that value is already carried by TableKeyInfo. Delete the non-session write_stream_inline helper, which had no callers. The public trait methods next_sequence_number and assign_shard remain as-is for engine-layer compatibility, though no data-engine path uses them post this commit. --- crates/storage-mongodb/src/data_engine.rs | 124 +++----------------- crates/storage-mongodb/src/stream_engine.rs | 96 +++++++++++++++ 2 files changed, 115 insertions(+), 105 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 63a41057..52537c76 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -238,99 +238,6 @@ impl DataEngine for MongoEngine { } impl MongoEngine { - async fn write_stream_inline( - &self, - key_info: &TableKeyInfo, - capture: &StreamCapture, - old_item: Option<&Item>, - new_item: Option<&Item>, - ) -> Result<(), StorageError> { - use extenddb_core::types::StreamViewType; - - let source_item = new_item.or(old_item); - let Some(source) = source_item else { - return Ok(()); - }; - - let event = match (old_item, new_item) { - (None, Some(_)) => StreamEventName::Insert, - (Some(_), Some(_)) => StreamEventName::Modify, - (Some(_), None) => StreamEventName::Remove, - (None, None) => return Ok(()), - }; - - let keys: std::collections::BTreeMap = key_info - .key_schema - .iter() - .filter_map(|ks| { - source - .get(&ks.attribute_name) - .map(|v| (ks.attribute_name.clone(), v.clone())) - }) - .collect(); - - let new_image = match capture.view_type { - StreamViewType::NewImage | StreamViewType::NewAndOldImages => new_item.cloned(), - _ => None, - }; - let old_image = match capture.view_type { - StreamViewType::OldImage | StreamViewType::NewAndOldImages => old_item.cloned(), - _ => None, - }; - - let size = source_item.map_or(0, |i| i64::try_from(item_size_bytes(i)).unwrap_or(i64::MAX)); - - let pk_name = &key_info.key_schema[0].attribute_name; - let pk_str = source - .get(pk_name) - .map(|v| match v { - AttributeValue::S(s) => s.clone(), - AttributeValue::N(n) => n.clone(), - AttributeValue::B(b) => { - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, b) - } - _ => String::new(), - }) - .unwrap_or_default(); - - let shard_id = self - .assign_shard(&key_info.account_id, &key_info.table_name, &pk_str) - .await?; - let seq = self.next_sequence_number(&shard_id).await?; - - let record = StreamRecord { - event_id: uuid::Uuid::new_v4().to_string(), - event_name: event, - event_version: "1.1".to_owned(), - event_source: "aws:dynamodb".to_owned(), - aws_region: capture.region.to_string(), - dynamodb: StreamRecordData { - approximate_creation_date_time: i64::try_from( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(), - ) - .unwrap_or(i64::MAX), - keys, - new_image, - old_image, - sequence_number: seq, - size_bytes: size, - stream_view_type: capture.view_type, - }, - user_identity: capture.user_identity.clone(), - }; - - self.write_stream_record( - &key_info.account_id, - &record, - &shard_id, - &key_info.table_name, - ) - .await - } - async fn put_item_impl( &self, key_info: &TableKeyInfo, @@ -1434,10 +1341,22 @@ impl MongoEngine { }) .unwrap_or_default(); + // Both shard resolution and sequence-number assignment run inside + // the same session as the data write. This is what makes stream + // ordering safe under contention — see + // stream_engine::next_sequence_number_in_session for the full + // rationale. let shard_id = self - .assign_shard(&key_info.account_id, &key_info.table_name, &pk_str) + .assign_shard_in_session( + &key_info.account_id, + &key_info.table_name, + &pk_str, + &mut *session, + ) + .await?; + let seq = self + .next_sequence_number_in_session(&shard_id, &mut *session) .await?; - let seq = self.next_sequence_number(&shard_id).await?; let record = StreamRecord { event_id: uuid::Uuid::new_v4().to_string(), @@ -1468,16 +1387,11 @@ impl MongoEngine { let record_bson = bson::to_bson(&record_json).map_err(|e| StorageError::Internal(e.to_string()))?; - let tables_coll = self.catalog_db.collection::("tables"); - let table_doc = tables_coll - .find_one(doc! { "_id": { "account_id": &key_info.account_id, "table_name": &key_info.table_name } }) - .session(&mut *session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - .ok_or_else(|| { - StorageError::Internal(format!("Table {} not found in catalog", key_info.table_name)) - })?; - let table_id = table_doc.get_str("table_id").unwrap_or_default(); + // key_info already carries table_id — no need to re-read the catalog + // just to resolve it, and re-reading inside the session against the + // tables collection would join to the counter/records write set + // needlessly. + let table_id = &key_info.table_id; let records_coll = self.data_db.collection::("stream_records"); records_coll diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index a18b2381..8f75e780 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -84,6 +84,102 @@ impl MongoEngine { Ok(()) } + /// Draw the next sequence number for a shard *inside* the given + /// transaction session. + /// + /// Sequence assignment must participate in the same transaction as the + /// stream-record insert, otherwise a fast writer B can obtain seq=6 + /// and commit before a slow writer A (which obtained seq=5) commits. + /// A consumer polling at `after_sequence_number=cursor` between B's + /// commit and A's commit sees seq=6 and advances past it; when A + /// finally commits, seq=5 lands behind the cursor and is never + /// returned. RFC-0003 §5.1 (atomicity with data writes) and §5.2 + /// (per-shard ordering). + /// + /// Placing the counter increment inside the session also serializes + /// concurrent writers on the same shard: two writes racing to + /// $inc the same counter under snapshot isolation will conflict at + /// commit time, so the loser retries — the transaction retry loop + /// upstream in the caller (see D-C3 followup) handles this. + pub(crate) async fn next_sequence_number_in_session( + &self, + shard_id: &str, + session: &mut mongodb::ClientSession, + ) -> Result { + let counters_coll = self.data_db.collection::("counters"); + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(mongodb::options::ReturnDocument::After) + .build(); + let counter_id = format!("stream_seq:{shard_id}"); + let doc = counters_coll + .find_one_and_update( + doc! { "_id": counter_id }, + doc! { "$inc": { "value": 1_i64 } }, + ) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Internal("Failed to generate sequence number".to_owned()) + })?; + + let seq_val = doc.get_i64("value").unwrap_or(1); + Ok(format!("{seq_val:021}")) + } + + /// Resolve the shard_id for a given (account, table, partition-key) + /// *inside* the given transaction session. Pairs with + /// `next_sequence_number_in_session` so the shard set the write is + /// routed to is read at the same snapshot as the sequence draw. + pub(crate) async fn assign_shard_in_session( + &self, + account_id: &str, + table_name: &str, + partition_key: &str, + session: &mut mongodb::ClientSession, + ) -> Result { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::Internal(format!("Table {table_name} not found")))?; + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let shards_coll = self.data_db.collection::("stream_shards"); + let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); + let mut cursor = shards_coll + .find(doc! { "table_id": table_id }) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_docs: Vec = cursor + .stream(&mut *session) + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if shard_docs.is_empty() { + return Err(StorageError::Internal(format!( + "No stream shards for table {table_name}" + ))); + } + + let shard_ids: Vec<&str> = shard_docs + .iter() + .filter_map(|d| d.get_str("shard_id").ok()) + .collect(); + + let hash = crc32fast::hash(partition_key.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shard_ids.len(); + Ok(shard_ids[idx].to_owned()) + } + /// Delete every stream_shards document for a given table_id, and every /// stream_records document written to any of its shards. Invoked from /// `delete_table_impl` so that a table recreated with the same name From 8fa02883c037521d3f4a2ef0532903ffe01b879f Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 13:27:59 -0700 Subject: [PATCH 25/83] fix(mongodb): embed base-table keys as first-class fields on index documents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GSI keys are non-unique — multiple base items can share identical (index_pk, index_sk) values. The index-document _id was previously built from index keys alone, so two base items sharing GSI keys hashed to the same _id and the second replace_one silently overwrote the first. A delete on the base then removed the surviving entry (which now represented a different item), so both vanished from the index. Add base_pk (text) and base_sk_s / base_sk_n / base_sk_b (typed) fields to every index document at write time, alongside the existing pk / sk_s / sk_n / sk_b for the index's own keys. Encode all four components into _id via netstring so each index entry is uniquely identified by (index_pk, index_sk, base_pk, base_sk). Sync-index delete filters now use the full compound tuple as well. Query and Scan pagination on an index collection uses a compound cursor over the same four components, expressed as a lexicographic $or of the shape (a > A) OR (a == A AND b > B) OR ... — so paginating through items sharing an index key no longer loses or duplicates records. Sort order matches. LastEvaluatedKey now carries both the index keys and the base-table keys so the next page's ExclusiveStartKey resolves the compound cursor. Base-table document construction is unchanged in behavior; the typed-sk-field emission was refactored into a shared helper (insert_typed_sk) also used by the new index-document builder. New public API: - index_document(projected, idx_key_schema, base_key_schema, attr_defs) - index_entry_filter(projected, idx_key_schema, base_key_schema, attr_defs) - sk_suffix(sk_type) Unit tests cover: base-key encoding in index docs, _id disambiguation of duplicate index keys, filter-matches-doc invariant, and hash-only GSI on composite base tables. --- crates/storage-mongodb/src/data/mod.rs | 350 ++++++++++++++++++--- crates/storage-mongodb/src/data_engine.rs | 355 ++++++++++++++++++---- 2 files changed, 612 insertions(+), 93 deletions(-) diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index f0d482e3..32754fb8 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -66,42 +66,8 @@ pub fn item_to_document( // naive "{pk}#{sk}" form is collision-prone. doc.insert("_id", composite_id(&pk_text, &sk_text)); doc.insert("pk", pk_text); - - // Insert the typed sort key field - match sk_type { - ScalarAttributeType::S => { - if let AttributeValue::S(s) = sk_value { - doc.insert("sk_s", s.clone()); - } - } - ScalarAttributeType::N => { - if let AttributeValue::N(n) = sk_value { - // Store as Decimal128 for correct numeric ordering. - // Values that exceed Decimal128's 34 significant digits (or - // any parse failure) are rejected rather than downcasting to - // f64, which would silently lose precision and can produce - // incorrect ordering. DynamoDB supports up to 38 digits; this - // limitation is documented in docs/differences-from-dynamodb.md. - let d = n.parse::().map_err(|_| { - StorageError::Validation(format!( - "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" - )) - })?; - doc.insert("sk_n", d); - } - } - ScalarAttributeType::B => { - if let AttributeValue::B(b) = sk_value { - doc.insert( - "sk_b", - bson::Binary { - subtype: bson::spec::BinarySubtype::Generic, - bytes: b.clone(), - }, - ); - } - } - } + let sk_field = format!("sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut doc, &sk_field, sk_type, sk_value)?; } else { // PK-only table doc.insert("_id", pk_text.clone()); @@ -112,6 +78,185 @@ pub fn item_to_document( Ok(doc) } +/// Insert a typed sort-key value into a document under the given field name. +/// +/// Shared between base tables (`sk_s`/`sk_n`/`sk_b`) and index documents +/// carrying the base table's sort key (`base_sk_s`/`base_sk_n`/`base_sk_b`). +fn insert_typed_sk( + doc: &mut Document, + field: &str, + sk_type: ScalarAttributeType, + sk_value: &AttributeValue, +) -> Result<(), StorageError> { + match (sk_type, sk_value) { + (ScalarAttributeType::S, AttributeValue::S(s)) => { + doc.insert(field, s.clone()); + } + (ScalarAttributeType::N, AttributeValue::N(n)) => { + // Store as Decimal128 for correct numeric ordering. Values that + // exceed Decimal128's 34 significant digits are rejected rather + // than downcasting to f64, which would silently lose precision. + let d = n.parse::().map_err(|_| { + StorageError::Validation(format!( + "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + })?; + doc.insert(field, d); + } + (ScalarAttributeType::B, AttributeValue::B(b)) => { + doc.insert( + field, + bson::Binary { + subtype: bson::spec::BinarySubtype::Generic, + bytes: b.clone(), + }, + ); + } + _ => { + // Mismatched types are silently skipped — matches the existing + // behavior of item_to_document. Callers can rely on + // validate_index_keys / validate_item_keys upstream to reject + // these before write. + } + } + Ok(()) +} + +/// Build a `MongoDB` index-collection document. +/// +/// Index documents differ from base-table documents in two ways: +/// +/// 1. The `_id` incorporates the base-table primary key in addition to the +/// index primary key. GSI keys are non-unique — multiple base items can +/// share identical `(index_pk, index_sk)` values. Encoding the base key +/// into `_id` gives each index entry a unique identity keyed to the base +/// item it describes. +/// +/// 2. The document carries the base-table key attributes as first-class +/// fields — `base_pk` (text) and `base_sk_s`/`base_sk_n`/`base_sk_b` +/// (typed). This lets index pagination form a compound cursor +/// `(index_sk, base_pk, base_sk)` without traversing the JSON +/// `item_data` payload for base-key values. +/// +/// The `item_data` payload is unchanged — it is the full projected item +/// as serialized by `AttributeValue`. +/// +/// `projected` is the item projected into the index (see `project_item` in +/// `data_engine.rs`). It must contain both the index-key attributes and +/// the base-table key attributes. +pub fn index_document( + projected: &Item, + idx_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let idx_pk_text = composite_pk_to_text(projected, idx_key_schema)?; + let base_pk_text = composite_pk_to_text(projected, base_key_schema)?; + + let idx_sk = sk_info(idx_key_schema, attribute_definitions); + let base_sk = sk_info(base_key_schema, attribute_definitions); + + // Build the netstring composite _id. Order: + // [index_pk, index_sk_or_"", base_pk, base_sk_or_""] + // Netstring parts are self-delimiting, so absent sk components encode as + // "0:," and the boundary is preserved. + let idx_sk_text = match idx_sk { + Some((sk_name, _)) => projected + .get(sk_name) + .map(sk_to_text) + .transpose()? + .unwrap_or_default(), + None => String::new(), + }; + let base_sk_text = match base_sk { + Some((sk_name, _)) => projected + .get(sk_name) + .map(sk_to_text) + .transpose()? + .unwrap_or_default(), + None => String::new(), + }; + let id = encode_netstring_composite(&[ + idx_pk_text.clone(), + idx_sk_text, + base_pk_text.clone(), + base_sk_text, + ]); + + let mut doc = Document::new(); + doc.insert("_id", id); + doc.insert("pk", &idx_pk_text); + doc.insert("base_pk", &base_pk_text); + + if let Some((sk_name, sk_type)) = idx_sk + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut doc, &field, sk_type, sk_value)?; + } + if let Some((sk_name, sk_type)) = base_sk + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("base_sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut doc, &field, sk_type, sk_value)?; + } + + let item_json = + serde_json::to_value(projected).map_err(|e| StorageError::Internal(e.to_string()))?; + let item_bson = bson::to_bson(&item_json).map_err(|e| StorageError::Internal(e.to_string()))?; + doc.insert("item_data", item_bson); + + Ok(doc) +} + +/// Build a delete filter for a specific index entry. +/// +/// The filter must match the exact base item's index entry, so it needs +/// both the index-key and base-key components — a filter on index keys +/// alone would delete every base item's entry that shares those index +/// keys (silent data loss on GSIs with duplicate keys). Returns a filter +/// on `(pk, sk?, base_pk, base_sk?)` — the same tuple that composes +/// the `_id`, but we filter on the individual fields so mongo can use +/// per-field indexes if present. +pub fn index_entry_filter( + projected: &Item, + idx_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let idx_pk_text = composite_pk_to_text(projected, idx_key_schema)?; + let base_pk_text = composite_pk_to_text(projected, base_key_schema)?; + let mut filter = doc! { + "pk": idx_pk_text, + "base_pk": base_pk_text, + }; + + if let Some((sk_name, sk_type)) = sk_info(idx_key_schema, attribute_definitions) + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut filter, &field, sk_type, sk_value)?; + } + if let Some((sk_name, sk_type)) = sk_info(base_key_schema, attribute_definitions) + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("base_sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut filter, &field, sk_type, sk_value)?; + } + Ok(filter) +} + +/// Sort-key column suffix for a scalar attribute type. Shared by index and +/// base-key field naming. +#[must_use] +pub fn sk_suffix(sk_type: ScalarAttributeType) -> &'static str { + match sk_type { + ScalarAttributeType::S => "s", + ScalarAttributeType::N => "n", + ScalarAttributeType::B => "b", + } +} + /// Convert a `MongoDB` document back to a `DynamoDB` Item. pub fn document_to_item(doc: &Document) -> Result { let item_data = doc @@ -318,4 +463,137 @@ mod tests { "expected netstring-encoded _id, got {id:?}" ); } + + // ── index_document / index_entry_filter ───────────────────────── + + fn base_schema_composite() -> (Vec, Vec) { + ( + vec![ + KeySchemaElement { + attribute_name: "customer_id".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "order_id".to_owned(), + key_type: KeyType::Range, + }, + ], + vec![ + AttributeDefinition { + attribute_name: "customer_id".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "order_id".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "status".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "priority".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ], + ) + } + + fn gsi_schema_hash_range() -> Vec { + vec![ + KeySchemaElement { + attribute_name: "status".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "priority".to_owned(), + key_type: KeyType::Range, + }, + ] + } + + fn item_with(customer_id: &str, order_id: &str, status: &str, priority: &str) -> Item { + let mut item = Item::new(); + item.insert( + "customer_id".to_owned(), + AttributeValue::S(customer_id.to_owned()), + ); + item.insert( + "order_id".to_owned(), + AttributeValue::S(order_id.to_owned()), + ); + item.insert("status".to_owned(), AttributeValue::S(status.to_owned())); + item.insert( + "priority".to_owned(), + AttributeValue::N(priority.to_owned()), + ); + item + } + + #[test] + fn index_document_encodes_base_keys() { + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = gsi_schema_hash_range(); + let item = item_with("cust1", "order1", "pending", "5"); + + let doc = index_document(&item, &idx_schema, &base_schema, &attrs).unwrap(); + assert_eq!(doc.get_str("pk").unwrap(), "pending"); + assert_eq!(doc.get_str("base_pk").unwrap(), "cust1"); + // Index sk (N) is Decimal128, base sk (S) is a plain string. + assert!(doc.get("sk_n").is_some(), "expected sk_n field"); + assert_eq!(doc.get_str("base_sk_s").unwrap(), "order1"); + } + + #[test] + fn index_document_id_disambiguates_duplicate_index_keys() { + // Two base items sharing (status, priority) but different base keys + // must produce distinct index _ids. Without base keys in _id both + // upserts would write to the same document — the D-C1 data-loss bug. + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = gsi_schema_hash_range(); + + let a = item_with("custA", "orderA", "pending", "5"); + let b = item_with("custB", "orderB", "pending", "5"); + + let da = index_document(&a, &idx_schema, &base_schema, &attrs).unwrap(); + let db = index_document(&b, &idx_schema, &base_schema, &attrs).unwrap(); + assert_ne!(da.get_str("_id").unwrap(), db.get_str("_id").unwrap()); + } + + #[test] + fn index_entry_filter_matches_own_document() { + // The filter built for a projected item must select exactly that + // item's index document — same _id, base_pk, and base_sk fields. + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = gsi_schema_hash_range(); + let item = item_with("cust1", "order1", "pending", "5"); + + let doc = index_document(&item, &idx_schema, &base_schema, &attrs).unwrap(); + let filter = index_entry_filter(&item, &idx_schema, &base_schema, &attrs).unwrap(); + // Every filter field must appear in the doc with the same value. + for (k, v) in filter.iter() { + let actual = doc.get(k).expect("filter field missing on doc"); + assert_eq!(v, actual, "filter field {k} mismatch"); + } + } + + #[test] + fn index_document_supports_hash_only_gsi_on_composite_base() { + // R-2 shape: hash-only GSI on a composite base table. The doc must + // still carry base_pk and base_sk so pagination can tie-break. + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = vec![KeySchemaElement { + attribute_name: "status".to_owned(), + key_type: KeyType::Hash, + }]; + let item = item_with("cust1", "order1", "pending", "5"); + + let doc = index_document(&item, &idx_schema, &base_schema, &attrs).unwrap(); + assert_eq!(doc.get_str("pk").unwrap(), "pending"); + assert!( + doc.get("sk_s").is_none() && doc.get("sk_n").is_none() && doc.get("sk_b").is_none() + ); + assert_eq!(doc.get_str("base_pk").unwrap(), "cust1"); + assert_eq!(doc.get_str("base_sk_s").unwrap(), "order1"); + } } diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 52537c76..431ba876 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -27,8 +27,8 @@ use extenddb_storage::{ use crate::MongoEngine; use crate::condition::condition_to_filter; use crate::data::{ - composite_id, data_collection_name, document_to_item, item_to_document, pk_filter, - sk_field_name, + composite_id, data_collection_name, document_to_item, index_document, index_entry_filter, + item_to_document, pk_filter, sk_field_name, sk_suffix, }; use crate::pushdown::{Pushable, is_pushable}; @@ -787,27 +787,113 @@ impl MongoEngine { } } - // Apply exclusive_start_key pagination - if let Some(start_key) = exclusive_start_key - && let Some(sk_f) = sk_field - { - // Get the sort key value from the start key - if let Some((sk_name, sk_type)) = - sk_info(&effective_key_schema, &key_info.attribute_definitions) - && let Some(sk_val) = start_key.get(sk_name) + // Apply exclusive_start_key pagination. + // + // For a base-table query the cursor is a single sort-key comparison: + // base-table items are uniquely keyed by (pk, sk), so `sk > cursor` + // is unambiguous. + // + // For an index query the cursor is a compound tuple over + // (index_sk?, base_pk, base_sk?). Index-key values are non-unique — + // duplicates fall through to the base-key tie-breaker. Express as + // a lexicographic `$or` of the shape + // (a > A) OR (a == A AND b > B) OR (a == A AND b == B AND c > C) + // (with `<` when reverse). RFC-0003 §2.6. + let is_index = index_name.is_some(); + if let Some(start_key) = exclusive_start_key { + let cmp_gt = if forward { "$gt" } else { "$lt" }; + if is_index { + let idx_sk_pair = + match sk_info(&effective_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (sk_field.expect("sk_field present when sk_info is Some"), b)), + None => None, + }; + let base_pk_bson: Option = { + // Build the base_pk text the same way the write path + // does — composite_pk_to_text on the base key schema. + // If the start_key is malformed we skip pagination + // (result is a query that may return duplicates). + let text = composite_pk_to_text(start_key, &key_info.base_key_schema).ok(); + text.map(bson::Bson::String) + }; + let base_sk_pair = + match sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (format!("base_sk_{}", sk_suffix(sk_type)), b)), + None => None, + }; + + let mut or_clauses: Vec = Vec::new(); + if let Some((sk_f, sk_bson)) = idx_sk_pair.clone() { + or_clauses.push(doc! { sk_f: { cmp_gt: sk_bson } }); + } + if let Some(bp) = base_pk_bson.clone() { + let mut clause = Document::new(); + if let Some((sk_f, sk_bson)) = idx_sk_pair.clone() { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", doc! { cmp_gt: bp }); + or_clauses.push(clause); + } + if let (Some(bp), Some((base_sk_f, base_sk_bson))) = + (base_pk_bson, base_sk_pair.clone()) + { + let mut clause = Document::new(); + if let Some((sk_f, sk_bson)) = idx_sk_pair { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", bp); + clause.insert(base_sk_f, doc! { cmp_gt: base_sk_bson }); + or_clauses.push(clause); + } + + if !or_clauses.is_empty() { + // Merge with any existing $or (unlikely — sk_condition + // uses ranged operators, not $or) by wrapping in $and. + if filter.contains_key("$or") { + let existing = filter.remove("$or").unwrap(); + filter.insert( + "$and", + bson::bson!([{ "$or": existing }, { "$or": or_clauses }]), + ); + } else { + filter.insert("$or", or_clauses); + } + } + } else if let (Some(sk_f), Some((sk_name, sk_type))) = ( + sk_field, + sk_info(&effective_key_schema, &key_info.attribute_definitions), + ) && let Some(sk_val) = start_key.get(sk_name) { let sk_bson = sk_to_bson(sk_val, sk_type)?; - if forward { - filter.insert(sk_f, doc! { "$gt": sk_bson }); - } else { - filter.insert(sk_f, doc! { "$lt": sk_bson }); - } + filter.insert(sk_f, doc! { cmp_gt: sk_bson }); } } - // Build sort direction + // Build sort direction. For indexes the sort tuple is + // (index_sk?, base_pk, base_sk?) so pagination lands deterministic + // within a group of items sharing index keys. let sort_direction = if forward { 1 } else { -1 }; - let sort_doc = if let Some(sk_f) = sk_field { + let sort_doc = if is_index { + let mut sd = Document::new(); + if let Some(sk_f) = sk_field { + sd.insert(sk_f, sort_direction); + } + sd.insert("base_pk", sort_direction); + if let Some((_, sk_type)) = + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + { + sd.insert(format!("base_sk_{}", sk_suffix(sk_type)), sort_direction); + } + sd + } else if let Some(sk_f) = sk_field { doc! { sk_f: sort_direction } } else { doc! { "pk": sort_direction } @@ -870,15 +956,27 @@ impl MongoEngine { } } - // Handle pagination + // Handle pagination. For an index query, LEK carries both the + // index-key components and the base-key components so the next + // page's ExclusiveStartKey can resolve the compound cursor. + // RFC-0003 §7.2. let last_evaluated_key = if let Some(l) = limit { #[allow(clippy::cast_sign_loss)] let l_usize = l as usize; if items.len() > l_usize { items.truncate(l_usize); - items - .last() - .map(|item| extract_key(item, &key_info.key_schema)) + items.last().map(|item| { + if is_index { + let mut key = extract_key(item, &effective_key_schema); + let base_key = extract_key(item, &key_info.base_key_schema); + for (k, v) in base_key { + key.entry(k).or_insert(v); + } + key + } else { + extract_key(item, &key_info.key_schema) + } + }) } else { None } @@ -900,39 +998,118 @@ impl MongoEngine { ) -> Result<(Vec, Option), StorageError> { use futures::TryStreamExt; - let coll_name = if let Some(idx_name) = index_name { + // The effective key schema for the collection under scan: index + // schema for index scans (where the collection's _id encodes index + // keys), base schema for base-table scans. + let (coll_name, effective_key_schema) = if let Some(idx_name) = index_name { let idx_info = self .index_info_by_table_id_impl(&key_info.table_id, idx_name) .await?; - data_collection_name(&idx_info.index_id) + ( + data_collection_name(&idx_info.index_id), + idx_info.key_schema.clone(), + ) } else { - data_collection_name(&key_info.table_id) + ( + data_collection_name(&key_info.table_id), + key_info.key_schema.clone(), + ) }; let coll = self.data_db.collection::(&coll_name); + let is_index = index_name.is_some(); let mut filter = Document::new(); - // Apply exclusive_start_key for pagination (using _id for scan ordering) + // Apply exclusive_start_key for pagination. Base tables use _id + // ordering (netstring-encoded); index scans use a compound cursor + // over (pk, sk?, base_pk, base_sk?) so items with duplicate index + // keys don't confuse pagination. RFC-0003 §7.2, §2.6. if let Some(start_key) = exclusive_start_key { - let start_pk = composite_pk_to_text(start_key, &key_info.key_schema)?; - if let Some((sk_name, sk_type)) = - sk_info(&key_info.key_schema, &key_info.attribute_definitions) - { - if let Some(sk_val) = start_key.get(sk_name) { - let sk_text = match sk_val { - AttributeValue::S(s) => s.clone(), - AttributeValue::N(n) => n.clone(), - AttributeValue::B(b) => { - use base64::Engine; - base64::engine::general_purpose::STANDARD.encode(b) - } - _ => return Err(StorageError::Internal("invalid sk type".to_string())), + if is_index { + let idx_pk_bson: Option = + composite_pk_to_text(start_key, &effective_key_schema) + .ok() + .map(bson::Bson::String); + let idx_sk_pair = + match sk_info(&effective_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (format!("sk_{}", sk_suffix(sk_type)), b)), + None => None, + }; + let base_pk_bson: Option = + composite_pk_to_text(start_key, &key_info.base_key_schema) + .ok() + .map(bson::Bson::String); + let base_sk_pair = + match sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (format!("base_sk_{}", sk_suffix(sk_type)), b)), + None => None, }; - let start_id = composite_id(&start_pk, &sk_text); - filter.insert("_id", doc! { "$gt": start_id }); + + let mut or_clauses: Vec = Vec::new(); + if let Some(ip) = idx_pk_bson.clone() { + or_clauses.push(doc! { "pk": { "$gt": ip } }); + } + if let (Some(ip), Some((sk_f, sk_bson))) = + (idx_pk_bson.clone(), idx_sk_pair.clone()) + { + or_clauses.push(doc! { + "pk": ip, + sk_f: { "$gt": sk_bson }, + }); + } + if let (Some(ip), Some(bp)) = (idx_pk_bson.clone(), base_pk_bson.clone()) { + let mut clause = doc! { "pk": ip }; + if let Some((sk_f, sk_bson)) = idx_sk_pair.clone() { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", doc! { "$gt": bp }); + or_clauses.push(clause); + } + if let (Some(ip), Some(bp), Some((base_sk_f, base_sk_bson))) = + (idx_pk_bson, base_pk_bson, base_sk_pair) + { + let mut clause = doc! { "pk": ip }; + if let Some((sk_f, sk_bson)) = idx_sk_pair { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", bp); + clause.insert(base_sk_f, doc! { "$gt": base_sk_bson }); + or_clauses.push(clause); + } + + if !or_clauses.is_empty() { + filter.insert("$or", or_clauses); } } else { - filter.insert("_id", doc! { "$gt": &start_pk }); + // Base-table scan: unique (pk, sk) means _id > cursor. + let start_pk = composite_pk_to_text(start_key, &effective_key_schema)?; + if let Some((sk_name, _)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + if let Some(sk_val) = start_key.get(sk_name) { + let sk_text = match sk_val { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(b) + } + _ => return Err(StorageError::Internal("invalid sk type".to_string())), + }; + let start_id = composite_id(&start_pk, &sk_text); + filter.insert("_id", doc! { "$gt": start_id }); + } + } else { + filter.insert("_id", doc! { "$gt": &start_pk }); + } } } @@ -950,8 +1127,29 @@ impl MongoEngine { } }); + // Sort key. Index scans sort by (pk, sk?, base_pk, base_sk?) so + // pagination is well-defined across items sharing index keys. + // Base-table scans sort by _id (unique). + let sort_doc = if is_index { + let mut sd = doc! { "pk": 1 }; + if let Some((_, sk_type)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + sd.insert(format!("sk_{}", sk_suffix(sk_type)), 1); + } + sd.insert("base_pk", 1); + if let Some((_, sk_type)) = + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + { + sd.insert(format!("base_sk_{}", sk_suffix(sk_type)), 1); + } + sd + } else { + doc! { "_id": 1 } + }; + let opts = mongodb::options::FindOptions::builder() - .sort(doc! { "_id": 1 }) + .sort(sort_doc) .limit(fetch_limit) .build(); @@ -994,15 +1192,25 @@ impl MongoEngine { } } - // Handle pagination + // Handle pagination. For index scans, LEK includes both the + // index-key and base-key components. RFC-0003 §7.2. let last_evaluated_key = if let Some(l) = limit { #[allow(clippy::cast_sign_loss)] let l_usize = l as usize; if items.len() > l_usize { items.truncate(l_usize); - items - .last() - .map(|item| extract_key(item, &key_info.key_schema)) + items.last().map(|item| { + if is_index { + let mut key = extract_key(item, &effective_key_schema); + let base_key = extract_key(item, &key_info.base_key_schema); + for (k, v) in base_key { + key.entry(k).or_insert(v); + } + key + } else { + extract_key(item, &key_info.key_schema) + } + }) } else { None } @@ -1170,11 +1378,21 @@ impl MongoEngine { let idx_coll_name = data_collection_name(&index_id); let idx_coll = self.data_db.collection::(&idx_coll_name); - // Delete old index entry + // Delete old index entry. The filter must match on both the + // index-key AND the base-key components, because GSIs allow + // duplicate index-key values across base items. See D-C1 / + // RFC-0003 §2.1. if let Some(old) = old_item && item_has_index_keys(old, &idx_key_schema) { - let old_filter = pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; + let projected_old = + project_item(old, &idx_key_schema, &key_info.key_schema, &projection); + let old_filter = index_entry_filter( + &projected_old, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; let _ = idx_coll.delete_one(old_filter).await; } @@ -1184,10 +1402,18 @@ impl MongoEngine { { let projected = project_item(new, &idx_key_schema, &key_info.key_schema, &projection); - let idx_doc = - item_to_document(&projected, &idx_key_schema, &key_info.attribute_definitions)?; - let filter = - pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let idx_doc = index_document( + &projected, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let filter = index_entry_filter( + &projected, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; let opts = mongodb::options::ReplaceOptions::builder() .upsert(true) .build(); @@ -1256,7 +1482,14 @@ impl MongoEngine { if let Some(old) = old_item && item_has_index_keys(old, &idx_key_schema) { - let old_filter = pk_filter(old, &idx_key_schema, &key_info.attribute_definitions)?; + let projected_old = + project_item(old, &idx_key_schema, &key_info.key_schema, &projection); + let old_filter = index_entry_filter( + &projected_old, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; let _ = idx_coll.delete_one(old_filter).session(&mut *session).await; } @@ -1265,10 +1498,18 @@ impl MongoEngine { { let projected = project_item(new, &idx_key_schema, &key_info.key_schema, &projection); - let idx_doc = - item_to_document(&projected, &idx_key_schema, &key_info.attribute_definitions)?; - let filter = - pk_filter(&projected, &idx_key_schema, &key_info.attribute_definitions)?; + let idx_doc = index_document( + &projected, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let filter = index_entry_filter( + &projected, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; let opts = mongodb::options::ReplaceOptions::builder() .upsert(true) .build(); From d0c825f77115be1709323df998f864521f2f160b Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:11:52 -0700 Subject: [PATCH 26/83] fix(mongodb): emit INSERT (not MODIFY) when UpdateItem creates an item MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When UpdateItem targeted a nonexistent key with streams enabled, the mongo backend passed a fabricated key-only "existing item" to the stream layer. That produced a MODIFY record with a phantom OldImage containing only the key attributes, instead of an INSERT with no OldImage. Guard old_item on existing_doc.is_some() — the key-only stub is only useful as a seed for apply_update; it must not leak into the stream capture or into the caller's ReturnValues=ALL_OLD result. The TransactWriteItems Update path was already correct via its is_creating check; no other call sites are affected. --- crates/storage-mongodb/src/data_engine.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 431ba876..6e17e1dc 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -632,7 +632,12 @@ impl MongoEngine { } let need_old = return_old || stream.is_some(); - let old_item = if need_old { + // Only surface a pre-image when the item actually existed. + // When existing_doc is None, `existing_item` is a fabricated + // key-only stub used to seed apply_update — feeding it to + // stream/ReturnValues would emit MODIFY with a phantom + // OldImage instead of INSERT (§5.4 in RFC-0003). + let old_item = if need_old && existing_doc.is_some() { Some(existing_item.clone()) } else { None From ad2040d990da21f5363a091b12354110ee1a186c Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:14:18 -0700 Subject: [PATCH 27/83] fix(mongodb): unique index on idempotency tokens + E11000 race handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add unique compound index on (account_id, token) to idempotency_tokens at bootstrap time. Without the constraint, two concurrent TransactWriteItems requests carrying the same token both did a pre-check inside snapshot-isolation transactions, neither saw the other's uncommitted insert, and both proceeded — executing the operation twice. With the unique index in place, the second insert now surfaces E11000. The write path catches the duplicate-key error, aborts its transaction, re-reads the winner outside the session, and returns IdempotentReplay or IdempotentMismatch depending on fingerprint match. Result matches the read-check path exactly. Bootstrap-only schema change; no data migration needed since the collection was created empty at init time. --- crates/storage-mongodb/src/bootstrapper.rs | 19 +++++++++++- crates/storage-mongodb/src/data_engine.rs | 34 +++++++++++++++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index 132541b4..889b7f6a 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -88,8 +88,9 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("Failed to create data db: {e}")))?; - // Create TTL index on idempotency_tokens.created_at (10 min expiry) let coll = db.collection::("idempotency_tokens"); + + // Create TTL index on idempotency_tokens.created_at (10 min expiry) let ttl_index = IndexModel::builder() .keys(doc! { "created_at": 1 }) .options( @@ -102,6 +103,22 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("Failed to create TTL index: {e}")))?; + // Unique compound index on (account_id, token). Without this, + // two concurrent TransactWriteItems calls with the same token + // both do a snapshot read that misses the other's uncommitted + // insert, and both commit — the operation executes twice. + // With the unique index in place, the second inserter fails + // E11000 and the write path converts that into a retryable + // error, giving the client the read-check path on retry. + coll.create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "token": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("idempotency_tokens unique index: {e}")))?; + // stream_shards: unique index on shard_id so a subsequent init/ // recreate can never insert a duplicate shard document under the // same shard_id. Combined with `table_id`-derived shard_ids diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 6e17e1dc..de56d33c 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1780,8 +1780,13 @@ impl MongoEngine { return Err(StorageError::IdempotentMismatch); } - // Store the token - idem_coll + // Store the token. A unique index on (account_id, token) + // catches the case where a concurrent request under snapshot + // isolation didn't see our pre-check but raced us to the + // insert. On E11000, abort our txn and resolve the winner + // by re-reading outside the session — same replay/mismatch + // logic as the pre-check path. + let insert_res = idem_coll .insert_one(doc! { "account_id": key.account_id, "token": key.token, @@ -1789,8 +1794,29 @@ impl MongoEngine { "created_at": mongodb::bson::DateTime::now(), }) .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .await; + if let Err(e) = insert_res { + let is_dup = matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ); + if is_dup { + let _ = session.abort_transaction().await; + let winner = idem_coll + .find_one(doc! { "account_id": key.account_id, "token": key.token }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Err( + match winner.as_ref().and_then(|d| d.get_str("fingerprint").ok()) { + Some(fp) if fp == key.fingerprint => StorageError::IdempotentReplay, + _ => StorageError::IdempotentMismatch, + }, + ); + } + return Err(StorageError::Internal(e.to_string())); + } } let mut reasons: Vec = Vec::with_capacity(ops.len()); From 73a53fcac2144357ecbbfd4652b08f93720701de Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:15:52 -0700 Subject: [PATCH 28/83] fix(mongodb): preserve sort-key range on Query pagination resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Document::insert(sk_f, {$gt: cursor})` replaced any existing sort-key predicate wholesale, so `Query sk BETWEEN 5 AND 10 LIMIT 2` returned page 2 items outside the BETWEEN range. Same failure for `begins_with` (upper prefix bound lost) and reverse pagination. Merge the resume bound into the existing filter document instead: - No prior predicate → insert as-is. - Prior operator map → add the cursor operator; on same-operator collision the cursor bound is always strictly more restrictive for the paging direction, so overwriting is safe. - Prior equality scalar → combine both under `$and` since a scalar binding cannot hold a `$gt` sibling. Index queries were already correct via the compound `$or` cursor; this only affects base-table queries. RFC-0003 §7.2. --- crates/storage-mongodb/src/data_engine.rs | 48 ++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index de56d33c..eeecdf9a 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -878,7 +878,53 @@ impl MongoEngine { ) && let Some(sk_val) = start_key.get(sk_name) { let sk_bson = sk_to_bson(sk_val, sk_type)?; - filter.insert(sk_f, doc! { cmp_gt: sk_bson }); + // Merge the resume bound into any existing sort-key + // predicate rather than replacing it. Naive + // `filter.insert(sk_f, {$gt: cursor})` drops the + // caller's original range/prefix/eq bound and returns + // items outside it on page 2+ (RFC-0003 §7.2). + let cursor_bound = doc! { cmp_gt: sk_bson }; + match filter.remove(sk_f) { + None => { + filter.insert(sk_f, cursor_bound); + } + Some(bson::Bson::Document(mut existing)) => { + // Existing predicate already uses operators + // ($gte/$lte/$lt/...); merge ours into the + // same operator map. If the caller and the + // cursor share an operator (both $gt on a + // forward page whose caller filtered $gt), + // overwriting with the cursor is correct — + // the cursor's bound is always strictly + // beyond the caller's for that direction. + for (k, v) in cursor_bound { + existing.insert(k, v); + } + filter.insert(sk_f, existing); + } + Some(scalar) => { + // Caller's predicate was an equality + // (`sk = X`). Combine with the resume bound + // under $and — a scalar sk_f binding can't + // hold a $gt sibling. + let clauses = bson::bson!([ + { sk_f: scalar }, + { sk_f: cursor_bound }, + ]); + if let Some(existing_and) = filter.remove("$and") { + let mut combined = match existing_and { + bson::Bson::Array(a) => a, + other => vec![other], + }; + if let bson::Bson::Array(new) = clauses { + combined.extend(new); + } + filter.insert("$and", combined); + } else { + filter.insert("$and", clauses); + } + } + } } } From fcd84af8323d17fcab84551701aed3e2f4954876 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:17:28 -0700 Subject: [PATCH 29/83] fix(mongodb): enforce 24-hour stream retention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stream records were retained indefinitely: no TTL index on stream_records, and the existing `cleanup_expired_stream_records` routine had no caller. TRIM_HORIZON iterators replayed the entire lifetime of a table instead of the DDB-mandated 24-hour window, and storage grew unbounded. Enforcement lives in the schema now: a TTL index on `stream_records.created_at` with `expireAfterSeconds = 86400`. MongoDB's TTL monitor deletes records ~1 minute after they age out. A `stream_record_cleanup_worker` also runs every hour as a defense-in-depth path — mirrors the postgres worker. --- crates/storage-mongodb/src/bootstrapper.rs | 23 ++++++++++++++++++++++ crates/storage-mongodb/src/lib.rs | 6 +++++- crates/storage-mongodb/src/ttl_worker.rs | 20 ++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index 889b7f6a..ee0cc713 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -137,6 +137,29 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("stream_shards shard_id index: {e}")))?; + // stream_records: TTL index enforcing DDB's 24-hour retention. + // The `expireAfterSeconds` is the delta from the field value, + // not a hard cutoff, so a MongoDB background thread will + // delete records ~1 minute after created_at + 24h. The + // TTL cleanup worker in `ttl_worker.rs` provides a defense- + // in-depth deletion path but the primary enforcement is here. + db.create_collection("stream_records") + .await + .map_err(|e| OpError::Internal(format!("Failed to create stream_records: {e}")))?; + db.collection::("stream_records") + .create_index( + IndexModel::builder() + .keys(doc! { "created_at": 1 }) + .options( + IndexOptions::builder() + .expire_after(std::time::Duration::from_secs(24 * 3600)) + .build(), + ) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_records TTL index: {e}")))?; + Ok(()) } diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 1417b315..7014e16b 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -139,7 +139,11 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { let storage_for_ttl = self.engine.clone(); let metrics = ctx.metrics.clone(); tokio::spawn(async move { ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics).await }); - tracing::info!("MongoDB backend: TTL cleanup worker spawned"); + let storage_for_stream = self.engine.clone(); + tokio::spawn(async move { + ttl_worker::stream_record_cleanup_worker(storage_for_stream).await; + }); + tracing::info!("MongoDB backend: TTL and stream cleanup workers spawned"); } fn backend_info(&self) -> Option { diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index 9a09ed5b..66b4a203 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -9,12 +9,14 @@ use std::time::Duration; use extenddb_core::metrics::MetricsCollector; use extenddb_core::types::UserIdentity; use extenddb_storage::error::StorageError; -use extenddb_storage::{DataEngine, MetadataEngine, TableEngine}; +use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine, TableEngine}; use crate::MongoEngine; const SCAN_INTERVAL: Duration = Duration::from_secs(60); const BATCH_SIZE: usize = 100; +const STREAM_RETENTION_HOURS: i64 = 24; +const STREAM_CLEANUP_INTERVAL: Duration = Duration::from_secs(3600); pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { let region_arc: Arc = Arc::from(storage.region.as_str()); @@ -26,6 +28,22 @@ pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { + loop { + tokio::time::sleep(STREAM_CLEANUP_INTERVAL).await; + match StreamEngine::cleanup_expired_stream_records(&*storage, STREAM_RETENTION_HOURS).await + { + Ok(0) => {} + Ok(n) => tracing::info!("Stream cleanup worker: deleted {n} expired record(s)"), + Err(e) => tracing::warn!("Stream cleanup worker: delete failed: {e}"), + } + } +} + async fn retry_pending_indexes(storage: &MongoEngine) { let Ok(pending) = MetadataEngine::all_tables_with_ttl(storage).await else { return; From 27db867c0beec156d4fbcae309a923e68df4dc77 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:22:26 -0700 Subject: [PATCH 30/83] fix(mongodb): make UpdateTable stream enable idempotent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repeated UpdateTable with StreamEnabled=true (e.g. an idempotent IaC re-apply) previously inserted a fresh set of shards on every call, so DescribeStream reported N × k shards and consumers received the same record N times. It also rewrote `stream_label`, invalidating stream ARNs previously handed out to consumers. Before calling init_stream_shards, look up the table's existing shards in `stream_shards`. If any are present, reuse them and leave `stream_label` alone — the existing ARN keeps resolving. Only mint a new label on first-enable, or on re-enable after a disable that cleared the label. Matches the postgres behavior in update_table.rs. --- crates/storage-mongodb/src/table_engine.rs | 37 +++++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index dca7a03e..54d3bc30 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -685,11 +685,38 @@ impl MongoEngine { let table_id = table_doc .get_str("table_id") .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; - let label = time::OffsetDateTime::now_utc() - .format(&time::format_description::well_known::Iso8601::DEFAULT) - .unwrap_or_else(|_| "unknown".to_string()); - update_doc.insert("stream_label", &label); - self.init_stream_shards(table_id).await?; + + // Idempotent re-enable: if shards already exist for this + // table, reuse them and preserve the existing + // stream_label. Otherwise a repeat UpdateTable would + // insert duplicate shards (DescribeStream would then + // report N × k) and rotate stream_label, invalidating + // stream ARNs previously handed out to consumers. + let shards_coll = self.data_db.collection::("stream_shards"); + let existing_shard = shards_coll + .find_one(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if existing_shard.is_none() { + let label = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Iso8601::DEFAULT) + .unwrap_or_else(|_| "unknown".to_string()); + update_doc.insert("stream_label", &label); + self.init_stream_shards(table_id).await?; + } else if table_doc + .get_str("stream_label") + .ok() + .filter(|s| !s.is_empty()) + .is_none() + { + // Shards exist but the label was cleared by a + // previous disable — restore a fresh label so the + // ARN resolves again. + let label = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Iso8601::DEFAULT) + .unwrap_or_else(|_| "unknown".to_string()); + update_doc.insert("stream_label", &label); + } } } From cafa6c322552208ffdbcba15c0543d962c7b07af Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:23:58 -0700 Subject: [PATCH 31/83] fix(mongodb): bump _v on UpdateItem native fast path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native fast path built by try_build_native_update issued raw $set/$unset without touching `_v`. A concurrent slow-path update reading the doc before the fast-path write, and committing after it, would find its versioned-filter guard still valid against the pre-fast-path `_v` — and overwrite the fast-path result. Classic lost-update against RFC-0003 §4.4 (UpdateItem must serialize). Always $inc `_v` by 1 on the fast path. Restructures the emit so the $inc doc gets built even when no numeric ADD action is present. --- crates/storage-mongodb/src/data_engine.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index eeecdf9a..6801cc77 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1360,9 +1360,6 @@ impl MongoEngine { } let mut update = Document::new(); - if !inc_doc.is_empty() { - update.insert("$inc", inc_doc); - } if !set_doc.is_empty() { update.insert("$set", set_doc); } @@ -1370,10 +1367,18 @@ impl MongoEngine { update.insert("$unset", unset_doc); } - if update.is_empty() { + if update.is_empty() && inc_doc.is_empty() { return None; } + // Bump `_v` on every native fast-path write. Without this a + // fast-path commit leaves `_v` at its previous value, and a + // slow-path update running concurrently against that same + // stale value can pass its versioned-filter guard and + // overwrite the fast-path write (lost update, RFC-0003 §4.4). + inc_doc.insert("_v", 1_i64); + update.insert("$inc", inc_doc); + Some(update) } From 4a2f61008884fe3ae94540f8a3e8abf33e4c8551 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:25:41 -0700 Subject: [PATCH 32/83] fix(mongodb): capture pre-image on UpdateItem so GSI deltas apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UpdateItem left orphaned GSI entries whenever an indexed attribute changed value or was removed. Two paths were affected: 1. Slow path — `old_item` was set only when the caller requested ReturnValues=ALL_OLD or streams were enabled, so the common "just update this attribute" call passed None to sync_indexes_in_session and its "delete old entry" branch never fired. Always compute the pre-image from `existing_doc` (which we already read for the version-guard) and pass it in for index sync. Keep the ReturnValues/stream-facing pre-image separate so the INSERT-vs-MODIFY distinction from D-C8 still holds. 2. Native fast path — `find_one_and_update` doesn't read the doc so there is no pre-image at all; sync_indexes was called with old_item=None. Gate the fast path on the GSI cache reporting "no GSIs on this table"; when unknown or GSIs present, fall through to the slow path. The fast path's sync_indexes call is also dropped, since the gate now guarantees no GSIs. RFC-0003 §2.2. --- crates/storage-mongodb/src/data_engine.rs | 53 +++++++++++++++-------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 6801cc77..2d1ea6e3 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -543,10 +543,18 @@ impl MongoEngine { let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; // Fast path: use native MongoDB atomic operators when possible. - // This avoids transactions and retries for simple unconditional updates. + // This avoids transactions and retries for simple unconditional + // updates. Gated on the table having no GSIs — the fast path + // does not read the pre-image, so it has no way to compute the + // GSI-key delta and would leave stale index entries when an + // indexed attribute is $set to a new value or $unset (RFC-0003 + // §2.2). The GSI cache lets us avoid a catalog query in the + // common (no-GSI) case; when the cache is stale-or-unknown + // we fall through to the slow path which is authoritative. if condition.is_none() && !return_old && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) && let Some(mongo_update) = self.try_build_native_update(actions, maps) { let opts = mongodb::options::FindOneAndUpdateOptions::builder() @@ -565,12 +573,6 @@ impl MongoEngine { None }; - // Sync GSI (non-transactional but data write is atomic) - if let Some(ref doc) = result_doc { - let item = document_to_item(doc)?; - self.sync_indexes(key_info, None, Some(&item)).await?; - } - return Ok((None, new_item)); } @@ -631,14 +633,27 @@ impl MongoEngine { } } - let need_old = return_old || stream.is_some(); - // Only surface a pre-image when the item actually existed. + // The pre-image is required for two independent reasons: + // (a) surfacing it to the caller (ReturnValues=ALL_OLD or + // stream capture) and (b) computing the GSI-key delta so + // stale index rows can be deleted when the update changes + // a GSI-key attribute (§2.2 in RFC-0003). + // + // Historically we only tracked (a). That left the fast + // path — no ReturnValues + no stream — running + // sync_indexes_in_session with old_item=None, so the + // "delete old entry" branch never fired and updates that + // moved an item between GSI-key values left the old entry + // orphaned. Always compute the pre-image on the slow path + // — we already read `existing_doc` for the version guard, + // so the additional cost is a clone. + // // When existing_doc is None, `existing_item` is a fabricated - // key-only stub used to seed apply_update — feeding it to - // stream/ReturnValues would emit MODIFY with a phantom - // OldImage instead of INSERT (§5.4 in RFC-0003). - let old_item = if need_old && existing_doc.is_some() { - Some(existing_item.clone()) + // key-only stub used only to seed apply_update; the stream + // path must not see it (else INSERT looks like MODIFY, §5.4). + let pre_image = existing_doc.as_ref().map(|_| existing_item.clone()); + let old_item_for_stream = if return_old || stream.is_some() { + pre_image.clone() } else { None }; @@ -686,10 +701,12 @@ impl MongoEngine { .map_err(|e| StorageError::Internal(e.to_string()))?; } - // Sync GSI collections within the transaction + // Sync GSI collections within the transaction — pass the + // true pre-image so index rows for moved GSI-key values + // can be deleted, not just the caller-visible old_item. self.sync_indexes_in_session( key_info, - old_item.as_ref(), + pre_image.as_ref(), Some(&new_item), &mut session, ) @@ -700,7 +717,7 @@ impl MongoEngine { self.write_stream_inline_in_session( key_info, capture, - old_item.as_ref(), + old_item_for_stream.as_ref(), Some(&new_item), &mut session, ) @@ -712,7 +729,7 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let old_item_result = if return_old { old_item } else { None }; + let old_item_result = if return_old { old_item_for_stream } else { None }; let new_item_result = if return_new { Some(new_item) } else { None }; return Ok((old_item_result, new_item_result)); } From 1ea3b797952d0e33673b3ceef96c98e9e23b4265 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 15:35:10 -0700 Subject: [PATCH 33/83] fix(mongodb): retry transient write conflicts (folds D-M2, D-M3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under snapshot isolation MongoDB aborts one side of every same-document race, and Docker's default majority-write concern only widens the window. The backend previously mapped every abort to StorageError::Internal, so callers saw HTTP 500 on any contention — including the everyday case of two PutItems targeting the same key. Wire up transient-error retries at every write path: - Add `is_transient_write_conflict` recognizing `WriteConflict` (112), the `TransientTransactionError` label, and the `UnknownTransactionCommitResult` label. Add `is_duplicate_key` for the shared E11000 detection. - Retry-loop put_item_impl, delete_item_impl, and update_item_impl end-to-end with bounded exponential backoff + jitter. The retry ceiling `TRANSIENT_RETRY_ATTEMPTS = 50` matches the existing OCC ceiling so a hot key can't lock a partition. - Conditional-insert races on PutItem now correctly map either E11000 (unique-index race) or WriteConflict (snapshot race) to ConditionalCheckFailedException with the winner's image, resolving D-M3's dead-code E11000 branch. - TransactWriteItems retries the whole txn on any transient conflict. After the ceiling, surfaces as TransactionCanceled with synthetic per-op `TransactionConflict` reason codes rather than a bare 500 — the wire-side response carries a DDB-shaped error the client can retry rather than an opaque internal failure. - Also removes the E11000 string-matching in the token/put paths in favor of the structured `is_duplicate_key` check. RFC-0003 §4.1, §4.3. --- crates/storage-mongodb/src/data_engine.rs | 957 ++++++++++++++-------- 1 file changed, 597 insertions(+), 360 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 2d1ea6e3..db407ad7 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -250,8 +250,6 @@ impl MongoEngine { let coll_name = data_collection_name(&key_info.table_id); let coll = self.data_db.collection::(&coll_name); - let new_doc = - item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions)?; let key_filter = pk_filter(&item, &key_info.key_schema, &key_info.attribute_definitions)?; let mut session = self @@ -269,106 +267,137 @@ impl MongoEngine { ) .build(); - session - .start_transaction() - .with_options(tx_options) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let old_item: Option; - let return_val: Option; - - if let Some(cond) = condition { - let existing_doc = coll - .find_one(key_filter.clone()) - .session(&mut session) + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + session + .start_transaction() + .with_options(tx_options.clone()) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(ref existing) = existing_doc { - let existing_item = document_to_item(existing)?; - let passed = expression::evaluate_condition(cond, &existing_item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; - if !passed { - let _ = session.abort_transaction().await; - return Err(StorageError::ConditionFailed(Some(existing_item))); - } - let opts = FindOneAndReplaceOptions::builder() - .return_document(ReturnDocument::Before) - .build(); - let old_doc = coll - .find_one_and_replace(key_filter, new_doc) - .with_options(opts) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let attempt_res: Result, TxErr> = async { + let old_item: Option; - old_item = old_doc.as_ref().map(document_to_item).transpose()?; - return_val = if return_old { old_item.clone() } else { None }; - } else { - let empty = std::collections::BTreeMap::new(); - let passed = expression::evaluate_condition(cond, &empty, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; - if !passed { - let _ = session.abort_transaction().await; - return Err(StorageError::ConditionFailed(None)); - } - let result = coll.insert_one(new_doc).session(&mut session).await; - if let Err(e) = result { - if e.to_string().contains("E11000") { - let _ = session.abort_transaction().await; - let winner = coll - .find_one(key_filter) + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + if let Some(ref existing) = existing_doc { + let existing_item = document_to_item(existing)?; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(Some( + existing_item, + )))); + } + let opts = FindOneAndReplaceOptions::builder() + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) .await - .map_err(|e2| StorageError::Internal(e2.to_string()))? - .map(|d| document_to_item(&d)) - .transpose()?; - return Err(StorageError::ConditionFailed(winner)); + .map_err(TxErr::from)?; + old_item = old_doc.as_ref().map(document_to_item).transpose()?; + } else { + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(cond, &empty, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(None))); + } + // Conditional insert: a concurrent inserter + // manifests either as E11000 (unique-index + // race) or as WriteConflict (snapshot-isolation + // race). Both are the runtime signature of a + // failed condition. Map dup-key to CCF with + // the winner's image; let WriteConflict fall + // through TxErr::Transient and retry — the + // retry will re-read and see the winner. + if let Err(e) = coll.insert_one(new_doc).session(&mut session).await { + if is_duplicate_key(&e) { + let _ = session.abort_transaction().await; + let winner = coll + .find_one(key_filter.clone()) + .await + .map_err(|e2| { + TxErr::Fatal(StorageError::Internal(e2.to_string())) + })? + .map(|d| document_to_item(&d)) + .transpose()?; + return Err(TxErr::Fatal(StorageError::ConditionFailed(winner))); + } + return Err(TxErr::from(e)); + } + old_item = None; } - let _ = session.abort_transaction().await; - return Err(StorageError::Internal(e.to_string())); + } else { + let opts = FindOneAndReplaceOptions::builder() + .upsert(true) + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(TxErr::from)?; + old_item = old_doc.as_ref().map(document_to_item).transpose()?; } - old_item = None; - return_val = None; - } - } else { - let opts = FindOneAndReplaceOptions::builder() - .upsert(true) - .return_document(ReturnDocument::Before) - .build(); - let old_doc = coll - .find_one_and_replace(key_filter, new_doc) - .with_options(opts) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - old_item = old_doc.as_ref().map(document_to_item).transpose()?; - return_val = if return_old { old_item.clone() } else { None }; - } + self.sync_indexes_in_session( + key_info, + old_item.as_ref(), + Some(&item), + &mut session, + ) + .await?; - // Sync GSI collections within the transaction - self.sync_indexes_in_session(key_info, old_item.as_ref(), Some(&item), &mut session) - .await?; + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + old_item.as_ref(), + Some(&item), + &mut session, + ) + .await?; + } - // Write stream record within the transaction - if let Some(capture) = stream { - self.write_stream_inline_in_session( - key_info, - capture, - old_item.as_ref(), - Some(&item), - &mut session, - ) - .await?; + Ok(if return_old { old_item } else { None }) + } + .await; + + match attempt_res { + Ok(return_val) => match session.commit_transaction().await { + Ok(()) => return Ok(return_val), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } } - session - .commit_transaction() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - Ok(return_val) + Err(StorageError::Internal( + "PutItem: too many concurrent write conflicts, giving up".to_owned(), + )) } async fn get_item_impl( @@ -431,77 +460,104 @@ impl MongoEngine { ) .build(); - session - .start_transaction() - .with_options(tx_options) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let deleted_item: Option; - - if let Some(cond) = condition { - let existing_doc = coll - .find_one(key_filter.clone()) - .session(&mut session) + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(ref existing) = existing_doc { - let existing_item = document_to_item(existing)?; - let passed = expression::evaluate_condition(cond, &existing_item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; - if !passed { + let attempt_res: Result, TxErr> = async { + let deleted_item: Option; + + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + if let Some(ref existing) = existing_doc { + let existing_item = document_to_item(existing)?; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(Some( + existing_item, + )))); + } + coll.delete_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + deleted_item = Some(existing_item); + } else { + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(cond, &empty, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(None))); + } + deleted_item = None; + } + } else { + let old_doc = coll + .find_one_and_delete(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + deleted_item = old_doc.as_ref().map(document_to_item).transpose()?; + } + + if deleted_item.is_some() { + self.sync_indexes_in_session( + key_info, + deleted_item.as_ref(), + None, + &mut session, + ) + .await?; + } + + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + deleted_item.as_ref(), + None, + &mut session, + ) + .await?; + } + + Ok(if return_old { deleted_item } else { None }) + } + .await; + + match attempt_res { + Ok(return_val) => match session.commit_transaction().await { + Ok(()) => return Ok(return_val), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Err(TxErr::Transient) => { let _ = session.abort_transaction().await; - return Err(StorageError::ConditionFailed(Some(existing_item))); + backoff_sleep(attempt).await; + continue; } - coll.delete_one(key_filter) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - deleted_item = Some(existing_item); - } else { - let empty = std::collections::BTreeMap::new(); - let passed = expression::evaluate_condition(cond, &empty, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; - if !passed { + Err(TxErr::Fatal(e)) => { let _ = session.abort_transaction().await; - return Err(StorageError::ConditionFailed(None)); + return Err(e); } - deleted_item = None; } - } else { - let old_doc = coll - .find_one_and_delete(key_filter) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - deleted_item = old_doc.as_ref().map(document_to_item).transpose()?; - } - - // Sync GSI collections within the transaction - if deleted_item.is_some() { - self.sync_indexes_in_session(key_info, deleted_item.as_ref(), None, &mut session) - .await?; - } - - // Write stream record within the transaction - if let Some(capture) = stream { - self.write_stream_inline_in_session( - key_info, - capture, - deleted_item.as_ref(), - None, - &mut session, - ) - .await?; } - session - .commit_transaction() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - Ok(if return_old { deleted_item } else { None }) + Err(StorageError::Internal( + "DeleteItem: too many concurrent write conflicts, giving up".to_owned(), + )) } #[allow(clippy::too_many_arguments)] @@ -591,151 +647,170 @@ impl MongoEngine { ) .build(); - for _attempt in 0..50 { + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { session .start_transaction() .with_options(tx_options.clone()) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let existing_doc = coll - .find_one(key_filter.clone()) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Sentinel returned by the attempt body to signal "the + // OCC version guard didn't match; retry from a fresh + // read." Distinct from TxErr::Transient because it isn't + // a mongo-side conflict — the whole snapshot succeeded, + // we just lost the CAS race. + struct StaleVersion; + #[allow(clippy::large_enum_variant)] + enum AttemptOk { + Committed(Option, Option), + Stale, + } - let current_version = existing_doc - .as_ref() - .and_then(|d| d.get_i64("_v").ok()) - .unwrap_or(0); + let attempt_res: Result = async { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; - let existing_item = if let Some(doc) = existing_doc.as_ref() { - document_to_item(doc)? - } else { - key.clone() - }; + let current_version = existing_doc + .as_ref() + .and_then(|d| d.get_i64("_v").ok()) + .unwrap_or(0); - if let Some(cond) = condition { - let eval_item = if existing_doc.is_some() { - &existing_item + let existing_item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc)? } else { - &std::collections::BTreeMap::new() + key.clone() }; - let passed = expression::evaluate_condition(cond, eval_item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; - if !passed { - let _ = session.abort_transaction().await; - return Err(StorageError::ConditionFailed(if existing_doc.is_some() { - Some(existing_item.clone()) + + if let Some(cond) = condition { + let eval_item = if existing_doc.is_some() { + &existing_item } else { - None - })); + &std::collections::BTreeMap::new() + }; + let passed = expression::evaluate_condition(cond, eval_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed( + if existing_doc.is_some() { + Some(existing_item.clone()) + } else { + None + }, + ))); + } } - } - // The pre-image is required for two independent reasons: - // (a) surfacing it to the caller (ReturnValues=ALL_OLD or - // stream capture) and (b) computing the GSI-key delta so - // stale index rows can be deleted when the update changes - // a GSI-key attribute (§2.2 in RFC-0003). - // - // Historically we only tracked (a). That left the fast - // path — no ReturnValues + no stream — running - // sync_indexes_in_session with old_item=None, so the - // "delete old entry" branch never fired and updates that - // moved an item between GSI-key values left the old entry - // orphaned. Always compute the pre-image on the slow path - // — we already read `existing_doc` for the version guard, - // so the additional cost is a clone. - // - // When existing_doc is None, `existing_item` is a fabricated - // key-only stub used only to seed apply_update; the stream - // path must not see it (else INSERT looks like MODIFY, §5.4). - let pre_image = existing_doc.as_ref().map(|_| existing_item.clone()); - let old_item_for_stream = if return_old || stream.is_some() { - pre_image.clone() - } else { - None - }; + let pre_image = existing_doc.as_ref().map(|_| existing_item.clone()); + let old_item_for_stream = if return_old || stream.is_some() { + pre_image.clone() + } else { + None + }; - let mut new_item = existing_item; - expression::apply_update(actions, &mut new_item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; + let mut new_item = existing_item; + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; - let mut new_doc = item_to_document( - &new_item, - &key_info.key_schema, - &key_info.attribute_definitions, - )?; - let new_version = current_version + 1; - new_doc.insert("_v", new_version); - - if existing_doc.is_some() { - let mut versioned_filter = key_filter.clone(); - if current_version == 0 { - versioned_filter.insert("_v", doc! { "$not": { "$gt": 0_i64 } }); - } else { - versioned_filter.insert("_v", current_version); - } - let result = coll - .replace_one(versioned_filter, new_doc) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + let mut new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let new_version = current_version + 1; + new_doc.insert("_v", new_version); - if result.matched_count == 0 { - let _ = session.abort_transaction().await; - let base_us = 50u64.saturating_mul(1u64 << _attempt.min(8)); - let jitter = rand::random_range(0..=base_us); - tokio::time::sleep(std::time::Duration::from_micros(jitter)).await; - continue; + if existing_doc.is_some() { + let mut versioned_filter = key_filter.clone(); + if current_version == 0 { + versioned_filter.insert("_v", doc! { "$not": { "$gt": 0_i64 } }); + } else { + versioned_filter.insert("_v", current_version); + } + let result = coll + .replace_one(versioned_filter, new_doc) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + if result.matched_count == 0 { + // OCC CAS lost: someone else bumped _v after + // our find_one. Not a mongo conflict — the + // snapshot txn succeeded, we just have a + // stale read. Signal retry. + let _ = StaleVersion; + return Ok(AttemptOk::Stale); + } + } else { + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(TxErr::from)?; } - } else { - let opts = mongodb::options::ReplaceOptions::builder() - .upsert(true) - .build(); - coll.replace_one(key_filter.clone(), new_doc) - .with_options(opts) - .session(&mut session) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - // Sync GSI collections within the transaction — pass the - // true pre-image so index rows for moved GSI-key values - // can be deleted, not just the caller-visible old_item. - self.sync_indexes_in_session( - key_info, - pre_image.as_ref(), - Some(&new_item), - &mut session, - ) - .await?; - - // Write stream record within the transaction - if let Some(capture) = stream { - self.write_stream_inline_in_session( + self.sync_indexes_in_session( key_info, - capture, - old_item_for_stream.as_ref(), + pre_image.as_ref(), Some(&new_item), &mut session, ) .await?; - } - session - .commit_transaction() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + old_item_for_stream.as_ref(), + Some(&new_item), + &mut session, + ) + .await?; + } - let old_item_result = if return_old { old_item_for_stream } else { None }; - let new_item_result = if return_new { Some(new_item) } else { None }; - return Ok((old_item_result, new_item_result)); + let old_item_result = if return_old { + old_item_for_stream + } else { + None + }; + let new_item_result = if return_new { Some(new_item) } else { None }; + Ok(AttemptOk::Committed(old_item_result, new_item_result)) + } + .await; + + match attempt_res { + Ok(AttemptOk::Committed(old, new)) => match session.commit_transaction().await { + Ok(()) => return Ok((old, new)), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Ok(AttemptOk::Stale) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } } Err(StorageError::Internal( - "UpdateItem: too many version conflicts, giving up".to_owned(), + "UpdateItem: too many concurrent write conflicts, giving up".to_owned(), )) } @@ -1798,7 +1873,6 @@ impl MongoEngine { idempotency: Option>, ) -> Result<(), StorageError> { use extenddb_core::types::CancellationReason; - use extenddb_core::validation; // Start a MongoDB multi-document transaction let mut session = self @@ -1816,108 +1890,173 @@ impl MongoEngine { ) .build(); - session - .start_transaction() - .with_options(tx_options) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Outcome of one attempt at running the whole idempotency check + // + op fan-out + commit. `Retry` means MongoDB aborted the txn + // as a transient conflict; the caller should re-run from the top. + enum AttemptOutcome { + Committed, + CanceledReasons(Vec), + Retry, + } - // Check idempotency token, scoped to the caller's account so that - // identical tokens from different accounts never collide. - if let Some(key) = idempotency { - let idem_coll = self.data_db.collection::("idempotency_tokens"); - let existing = idem_coll - .find_one(doc! { "account_id": key.account_id, "token": key.token }) - .session(&mut session) + // Rehydrate the `IdempotencyKey` per attempt from owned strings. + // The input struct holds `&str`s, so it cannot be moved across + // loop iterations. This keeps the retry loop lifetime-clean + // without asking upstream to change the trait signature. + let idem_owned = idempotency.map(|k| { + ( + k.account_id.to_owned(), + k.token.to_owned(), + k.fingerprint.to_owned(), + ) + }); + + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + let idempotency = idem_owned.as_ref().map(|(a, t, f)| IdempotencyKey { + account_id: a.as_str(), + token: t.as_str(), + fingerprint: f.as_str(), + }); + session + .start_transaction() + .with_options(tx_options.clone()) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(existing_doc) = existing { - let stored_fp = existing_doc.get_str("fingerprint").unwrap_or_default(); - if stored_fp == key.fingerprint { - session - .abort_transaction() + let outcome: Result = async { + // Check idempotency token, scoped to the caller's account + // so that identical tokens from different accounts never + // collide. + if let Some(key) = idempotency { + let idem_coll = self.data_db.collection::("idempotency_tokens"); + let existing = match idem_coll + .find_one(doc! { "account_id": key.account_id, "token": key.token }) + .session(&mut session) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - return Err(StorageError::IdempotentReplay); + { + Ok(v) => v, + Err(e) if is_transient_write_conflict(&e) => { + return Ok(AttemptOutcome::Retry); + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }; + + if let Some(existing_doc) = existing { + let stored_fp = existing_doc.get_str("fingerprint").unwrap_or_default(); + return Err(if stored_fp == key.fingerprint { + StorageError::IdempotentReplay + } else { + StorageError::IdempotentMismatch + }); + } + + // Store the token. A unique index on (account_id, token) + // catches the case where a concurrent request under + // snapshot isolation didn't see our pre-check but raced + // us to the insert. On E11000, resolve the winner by + // re-reading outside the session — same replay/mismatch + // logic as the pre-check path. + let insert_res = idem_coll + .insert_one(doc! { + "account_id": key.account_id, + "token": key.token, + "fingerprint": key.fingerprint, + "created_at": mongodb::bson::DateTime::now(), + }) + .session(&mut session) + .await; + if let Err(e) = insert_res { + if is_duplicate_key(&e) { + let winner = idem_coll + .find_one(doc! { + "account_id": key.account_id, + "token": key.token, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Err( + match winner.as_ref().and_then(|d| d.get_str("fingerprint").ok()) { + Some(fp) if fp == key.fingerprint => { + StorageError::IdempotentReplay + } + _ => StorageError::IdempotentMismatch, + }, + ); + } + if is_transient_write_conflict(&e) { + return Ok(AttemptOutcome::Retry); + } + return Err(StorageError::Internal(e.to_string())); + } } - session - .abort_transaction() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - return Err(StorageError::IdempotentMismatch); - } - // Store the token. A unique index on (account_id, token) - // catches the case where a concurrent request under snapshot - // isolation didn't see our pre-check but raced us to the - // insert. On E11000, abort our txn and resolve the winner - // by re-reading outside the session — same replay/mismatch - // logic as the pre-check path. - let insert_res = idem_coll - .insert_one(doc! { - "account_id": key.account_id, - "token": key.token, - "fingerprint": key.fingerprint, - "created_at": mongodb::bson::DateTime::now(), - }) - .session(&mut session) - .await; - if let Err(e) = insert_res { - let is_dup = matches!( - *e.kind, - mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( - mongodb::error::WriteError { code: 11000, .. } - )) - ); - if is_dup { - let _ = session.abort_transaction().await; - let winner = idem_coll - .find_one(doc! { "account_id": key.account_id, "token": key.token }) + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut any_failed = false; + + for op in ops { + match self + .execute_transact_write_op_in_session(op, &mut session) .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - return Err( - match winner.as_ref().and_then(|d| d.get_str("fingerprint").ok()) { - Some(fp) if fp == key.fingerprint => StorageError::IdempotentReplay, - _ => StorageError::IdempotentMismatch, - }, - ); + { + Ok(()) => reasons.push(CancellationReason::none()), + Err(TransactOpError::Cancel(r)) => { + any_failed = true; + reasons.push(r); + } + Err(TransactOpError::Transient) => { + return Ok(AttemptOutcome::Retry); + } + Err(TransactOpError::Storage(e)) => return Err(e), + } } - return Err(StorageError::Internal(e.to_string())); - } - } - - let mut reasons: Vec = Vec::with_capacity(ops.len()); - let mut any_failed = false; - for op in ops { - let reason = self - .execute_transact_write_op_in_session(op, &mut session) - .await; - match reason { - Ok(()) => reasons.push(CancellationReason::none()), - Err(TransactOpError::Cancel(r)) => { - any_failed = true; - reasons.push(r); + if any_failed { + return Ok(AttemptOutcome::CanceledReasons(reasons)); } - Err(TransactOpError::Storage(e)) => { + Ok(AttemptOutcome::Committed) + } + .await; + + match outcome { + Ok(AttemptOutcome::Committed) => match session.commit_transaction().await { + Ok(()) => return Ok(()), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Ok(AttemptOutcome::CanceledReasons(reasons)) => { + let _ = session.abort_transaction().await; + return Err(StorageError::TransactionCanceled(reasons)); + } + Ok(AttemptOutcome::Retry) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(e) => { let _ = session.abort_transaction().await; return Err(e); } } } - if any_failed { - let _ = session.abort_transaction().await; - return Err(StorageError::TransactionCanceled(reasons)); - } - - session - .commit_transaction() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - Ok(()) + // Exhausted retries under sustained contention. Surface as a + // canceled transaction with a synthetic per-op TransactionConflict + // reason so wire consumers see the DDB-canonical error string + // instead of a bare HTTP 500. The engine maps StorageError:: + // TransactionCanceled to TransactionCanceledException; the + // reason codes are echoed back in the message. + let reasons = ops + .iter() + .map(|_| CancellationReason { + code: "TransactionConflict".to_owned(), + message: Some("Transaction is ongoing for the item".to_owned()), + item: None, + }) + .collect(); + Err(StorageError::TransactionCanceled(reasons)) } async fn execute_transact_write_op_in_session( @@ -1961,7 +2100,7 @@ impl MongoEngine { .find_one(key_filter.clone()) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; let existing_item = if let Some(doc) = existing_doc.as_ref() { Some(document_to_item(doc).map_err(TransactOpError::Storage)?) @@ -1998,7 +2137,7 @@ impl MongoEngine { .with_options(opts) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; // Propagate to secondary indexes and the stream within the // same transaction session — otherwise a transactional write @@ -2057,7 +2196,7 @@ impl MongoEngine { .find_one(key_filter.clone()) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; let existing_item = if let Some(doc) = existing_doc.as_ref() { Some(document_to_item(doc).map_err(TransactOpError::Storage)?) @@ -2086,7 +2225,7 @@ impl MongoEngine { coll.delete_one(key_filter) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; // Propagate to secondary indexes and the stream within the // same transaction session. @@ -2140,7 +2279,7 @@ impl MongoEngine { .find_one(key_filter.clone()) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; let existing_item = if let Some(doc) = existing_doc.as_ref() { Some(document_to_item(doc).map_err(TransactOpError::Storage)?) @@ -2189,7 +2328,7 @@ impl MongoEngine { .with_options(opts) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; // Propagate to secondary indexes and the stream within the // same transaction session. When the update creates the @@ -2249,7 +2388,7 @@ impl MongoEngine { .find_one(key_filter) .session(&mut *session) .await - .map_err(|e| TransactOpError::Storage(StorageError::Internal(e.to_string())))?; + .map_err(TransactOpError::from)?; let existing_item = if let Some(doc) = existing_doc.as_ref() { Some(document_to_item(doc).map_err(TransactOpError::Storage)?) @@ -2475,11 +2614,109 @@ impl MongoEngine { } } +// ── Contention / retry helpers ────────────────────────────────────────── + +/// Maximum number of times to retry a write that MongoDB aborted as a +/// transient conflict. Small enough that we don't lock a partition on +/// sustained hot-key contention; large enough to absorb ordinary +/// snapshot-isolation aborts. Matches the OCC retry ceiling elsewhere +/// in this file. +const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; + +/// Error signal used inside per-attempt transaction bodies. Lets the +/// body use `?` for control flow while distinguishing "retry this +/// whole transaction" from "return this error to the caller." +enum TxErr { + Transient, + Fatal(StorageError), +} + +impl From for TxErr { + fn from(e: mongodb::error::Error) -> Self { + if is_transient_write_conflict(&e) { + TxErr::Transient + } else { + TxErr::Fatal(StorageError::Internal(e.to_string())) + } + } +} + +impl From for TxErr { + fn from(e: StorageError) -> Self { + TxErr::Fatal(e) + } +} + +/// Detect the family of errors MongoDB uses to signal "your write lost +/// to another concurrent writer under snapshot isolation; retry." +/// +/// The transient-transaction label is set on any error that a +/// `withTransaction` client would automatically retry. In addition to +/// abstract labels the raw `WriteConflict` (code 112) still shows up +/// when a same-document collision surfaces on the write itself rather +/// than at commit — check that too. RFC-0003 §4.1 / §4.3. +fn is_transient_write_conflict(e: &mongodb::error::Error) -> bool { + if e.contains_label(mongodb::error::TRANSIENT_TRANSACTION_ERROR) + || e.contains_label(mongodb::error::UNKNOWN_TRANSACTION_COMMIT_RESULT) + { + return true; + } + matches!(*e.kind, mongodb::error::ErrorKind::Command(ref c) if c.code == 112) + || matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 112, .. } + )) + ) +} + +/// Detect a duplicate-key error (E11000, code 11000). Used at +/// conditional-insert sites — a duplicate is the manifestation of a +/// conditional-put race, so it must be surfaced as +/// `ConditionalCheckFailedException` rather than a 500. +fn is_duplicate_key(e: &mongodb::error::Error) -> bool { + matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ) +} + +/// Exponential-backoff sleep with random jitter. Used inside the OCC +/// / WriteConflict retry loops so competing writers don't lock-step +/// re-retry into the same conflict window. +async fn backoff_sleep(attempt: u32) { + let base_us = 50u64.saturating_mul(1u64 << attempt.min(8)); + let jitter = rand::random_range(0..=base_us); + tokio::time::sleep(std::time::Duration::from_micros(jitter)).await; +} + // ── Transaction helper types ────────────────────────────────────────── enum TransactOpError { Cancel(extenddb_core::types::CancellationReason), Storage(StorageError), + /// MongoDB aborted the transaction as a transient conflict — + /// the whole transact_write_items txn should be retried from + /// the top. + Transient, +} + +impl From for TransactOpError { + fn from(e: mongodb::error::Error) -> Self { + if is_transient_write_conflict(&e) { + TransactOpError::Transient + } else { + TransactOpError::Storage(StorageError::Internal(e.to_string())) + } + } +} + +impl From for TransactOpError { + fn from(e: StorageError) -> Self { + TransactOpError::Storage(e) + } } /// Choose the `Item` value to include in a `CancellationReason` when a From bc43300f80e94e24f2292a80f2cd6fad4664b98f Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 16:25:41 -0700 Subject: [PATCH 34/83] fix(mongodb): async GSI backfill on UpdateTable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UpdateTable's GSI-create path used to insert the catalog document with `index_status: "ACTIVE"` without touching the base collection. On any populated table, the new GSI reported ACTIVE immediately but was permanently missing every item that existed before the call. Queries returned partial results silently — a §2.4 violation of RFC-0003 and a §9.1 no-silent-degradation violation. Split the create path into async DDB-style semantics: 1. UpdateTable now writes `index_status: "CREATING"` and returns immediately. Matches the DDB contract that GSI creation is asynchronous relative to the control-plane call. 2. New background worker `gsi_backfill_worker` in ttl_worker.rs polls the indexes catalog for `{index_status: "CREATING"}` rows, loads TableKeyInfo by table_id, and walks the base collection in 500-doc batches. Each item is projected and upserted into the index collection via the same primitives sync_indexes uses, so the on-disk shape is identical to a live write. 3. Persistent progress marker: after every batch, the worker writes the last `_id` scanned to `indexes..backfill_cursor`. A mid-backfill crash / server restart resumes from that cursor on the next tick — no full re-scan. 4. When a batch returns fewer docs than requested, the base is fully scanned; the worker flips `index_status` to `ACTIVE` and clears the cursor. Live writes during the backfill window continue to route through `sync_indexes` (already), which writes to CREATING indexes because membership is defined by catalog presence, not status. All writes are idempotent upserts on the same _id shape, so a base item touched by both backfill and a concurrent update converges regardless of interleaving. Supporting refactors: - `table_key_info_impl` now shares its parsing path with a new `table_key_info_by_table_id_impl` used by the worker (indexed lookup via the tables.table_id unique index). - `MongoEngine.catalog_db` is pub(crate) so the worker module can read the indexes catalog directly. - New `backfill_gsi_batch` helper on MongoEngine encapsulates one batch of item -> index-row projection. --- crates/storage-mongodb/src/data_engine.rs | 101 +++++++++++++++ crates/storage-mongodb/src/lib.rs | 8 +- crates/storage-mongodb/src/table_engine.rs | 59 ++++++++- crates/storage-mongodb/src/ttl_worker.rs | 138 ++++++++++++++++++++- 4 files changed, 297 insertions(+), 9 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index db407ad7..bd2229d0 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -2612,6 +2612,107 @@ impl MongoEngine { let new_out = if return_new { Some(new_item) } else { None }; Ok((old_out, new_out)) } + + // ── GSI Backfill ────────────────────────────────────────────── + // + // Called by the gsi_backfill_worker in ttl_worker.rs. Reads one + // batch of base-table items past the given cursor and upserts + // matching index rows. Returns the new cursor and whether more + // items remain to scan. The worker persists the cursor between + // batches so a mid-backfill server restart resumes from where it + // left off — see the CREATING → ACTIVE state machine in + // update_table_impl / spawn_workers. + + pub(crate) async fn backfill_gsi_batch( + &self, + key_info: &TableKeyInfo, + index_id: &str, + idx_key_schema: &[KeySchemaElement], + projection: &Projection, + cursor: Option<&bson::Bson>, + batch_size: i64, + ) -> Result { + use futures::TryStreamExt; + + let base_coll_name = data_collection_name(&key_info.table_id); + let base_coll = self.data_db.collection::(&base_coll_name); + let idx_coll_name = data_collection_name(index_id); + let idx_coll = self.data_db.collection::(&idx_coll_name); + + let mut filter = Document::new(); + if let Some(c) = cursor { + filter.insert("_id", doc! { "$gt": c.clone() }); + } + + let opts = mongodb::options::FindOptions::builder() + .sort(doc! { "_id": 1 }) + .limit(batch_size) + .build(); + + let base_cursor = base_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = base_cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let scanned = docs.len(); + let last_id = docs.last().and_then(|d| d.get("_id").cloned()); + + for doc in docs { + let item = document_to_item(&doc)?; + if !item_has_index_keys(&item, idx_key_schema) { + continue; + } + + let projected = project_item(&item, idx_key_schema, &key_info.key_schema, projection); + let idx_doc = index_document( + &projected, + idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let filter = index_entry_filter( + &projected, + idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // A short-read (fewer docs than the batch size) means we've + // reached the end of the base collection. Upstream flips the + // index to ACTIVE when that happens. + Ok(GsiBackfillProgress { + scanned, + last_id, + done: (scanned as i64) < batch_size, + }) + } +} + +/// Progress from one `backfill_gsi_batch` invocation. +pub(crate) struct GsiBackfillProgress { + /// Number of base-collection documents read in this batch (before + /// filtering out those missing index-key attributes). + pub scanned: usize, + /// The `_id` of the last document scanned; the next batch resumes + /// with `_id > last_id`. `None` when the batch was empty. + pub last_id: Option, + /// Whether the base collection has been fully scanned. + pub done: bool, } // ── Contention / retry helpers ────────────────────────────────────────── diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 7014e16b..c5d7eecd 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -143,7 +143,11 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { tokio::spawn(async move { ttl_worker::stream_record_cleanup_worker(storage_for_stream).await; }); - tracing::info!("MongoDB backend: TTL and stream cleanup workers spawned"); + let storage_for_backfill = self.engine.clone(); + tokio::spawn(async move { + ttl_worker::gsi_backfill_worker(storage_for_backfill).await; + }); + tracing::info!("MongoDB backend: TTL, stream cleanup, and GSI backfill workers spawned"); } fn backend_info(&self) -> Option { @@ -233,7 +237,7 @@ const GSI_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); /// `MongoDB` storage backend. pub struct MongoEngine { client: mongodb::Client, - catalog_db: mongodb::Database, + pub(crate) catalog_db: mongodb::Database, data_db: mongodb::Database, region: String, max_connections: u32, diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 54d3bc30..09d44cdd 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -751,13 +751,20 @@ impl MongoEngine { .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; + // Enter CREATING; the background gsi_backfill_worker + // in ttl_worker.rs discovers this row, backfills the + // base table, and flips the status to ACTIVE. Matches + // DDB's async UpdateTable contract (§2.4 in RFC-0003). + // Live writes during the backfill window sync through + // sync_indexes (upserts) so the eventual state is + // convergent regardless of interleaving. let index_doc = doc! { "_id": { "table_id": &desc.table_id, "index_name": &create.index_name }, "index_id": &index_id, "index_type": "GSI", "key_schema": key_schema_bson, "projection": projection_bson, - "index_status": "ACTIVE", + "index_status": "CREATING", "provisioned_throughput": pt_bson.unwrap_or(bson::Bson::Null), }; @@ -813,9 +820,50 @@ impl MongoEngine { .map_err(|e| StorageError::Internal(e.to_string()))? .ok_or_else(|| StorageError::TableNotFound(table_name.to_string()))?; + self.table_key_info_from_doc(&table_doc, true).await + } + + /// Load `TableKeyInfo` by `table_id`. The tables catalog has a + /// unique index on `table_id`, so this is a single-doc lookup. + /// Used by the GSI backfill worker, which discovers work items + /// keyed by `table_id`. Skips the ACTIVE-status guard so a table + /// that is temporarily in a transient state (CREATING, UPDATING) + /// can still be backfilled — backfill is decoupled from data-plane + /// availability. + pub(crate) async fn table_key_info_by_table_id_impl( + &self, + table_id: &str, + ) -> Result { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_id.to_string()))?; + + self.table_key_info_from_doc(&table_doc, false).await + } + + async fn table_key_info_from_doc( + &self, + table_doc: &Document, + require_active: bool, + ) -> Result { + let id_doc = table_doc + .get_document("_id") + .map_err(|_| StorageError::Internal("missing _id".to_string()))?; + let table_name = id_doc + .get_str("table_name") + .map_err(|_| StorageError::Internal("missing _id.table_name".to_string()))? + .to_string(); + let account_id = id_doc + .get_str("account_id") + .map_err(|_| StorageError::Internal("missing _id.account_id".to_string()))? + .to_string(); + let status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); - if status != "ACTIVE" { - return Err(StorageError::TableNotActive(table_name.to_string())); + if require_active && status != "ACTIVE" { + return Err(StorageError::TableNotActive(table_name)); } let table_id = table_doc @@ -846,7 +894,6 @@ impl MongoEngine { } }); - // Check for LSIs let indexes_coll = self.catalog_db.collection::("indexes"); let has_lsi = indexes_coll .count_documents(doc! { "_id.table_id": &table_id, "index_type": "LSI" }) @@ -855,8 +902,8 @@ impl MongoEngine { > 0; Ok(TableKeyInfo { - table_name: table_name.to_string(), - account_id: account_id.to_string(), + table_name, + account_id, table_id, base_key_schema: key_schema.clone(), key_schema, diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index 66b4a203..da02a437 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -6,10 +6,12 @@ use std::sync::Arc; use std::time::Duration; +use bson::{Document, doc}; use extenddb_core::metrics::MetricsCollector; -use extenddb_core::types::UserIdentity; +use extenddb_core::types::{KeySchemaElement, Projection, ProjectionType, UserIdentity}; use extenddb_storage::error::StorageError; use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine, TableEngine}; +use futures::TryStreamExt; use crate::MongoEngine; @@ -17,6 +19,8 @@ const SCAN_INTERVAL: Duration = Duration::from_secs(60); const BATCH_SIZE: usize = 100; const STREAM_RETENTION_HOURS: i64 = 24; const STREAM_CLEANUP_INTERVAL: Duration = Duration::from_secs(3600); +const GSI_BACKFILL_INTERVAL: Duration = Duration::from_secs(5); +const GSI_BACKFILL_BATCH: i64 = 500; pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { let region_arc: Arc = Arc::from(storage.region.as_str()); @@ -44,6 +48,138 @@ pub(crate) async fn stream_record_cleanup_worker(storage: Arc) { } } +/// Background worker that turns CREATING GSIs into ACTIVE ones. +/// +/// UpdateTable's GSI-create path leaves the index in `index_status: +/// "CREATING"` after inserting the catalog document. This worker +/// discovers each such row, iterates the base collection with a +/// persistent cursor, upserts projected items into the index +/// collection, and — once the base is fully scanned — flips the +/// index to `ACTIVE`. Restart-safe: the cursor is persisted after +/// every batch so a mid-backfill crash resumes where it left off. +/// +/// Live writes during the backfill window continue to route through +/// `sync_indexes` / `sync_indexes_in_session`, which write to +/// CREATING indexes too (indexes catalog membership, not status, is +/// what gates the write path). All writes are upserts on the same +/// `_id` shape, so a base item touched by both the backfill and a +/// concurrent write converges regardless of interleaving — +/// RFC-0003 §2.4. +pub(crate) async fn gsi_backfill_worker(storage: Arc) { + loop { + tokio::time::sleep(GSI_BACKFILL_INTERVAL).await; + + let indexes_coll = storage.catalog_db.collection::("indexes"); + let cursor = match indexes_coll + .find(doc! { "index_status": "CREATING", "index_type": "GSI" }) + .await + { + Ok(c) => c, + Err(e) => { + tracing::warn!("GSI backfill worker: list failed: {e}"); + continue; + } + }; + let jobs: Vec = match cursor.try_collect().await { + Ok(j) => j, + Err(e) => { + tracing::warn!("GSI backfill worker: collect failed: {e}"); + continue; + } + }; + + for job in jobs { + if let Err(e) = run_gsi_backfill_job(&storage, &job).await { + tracing::warn!( + "GSI backfill worker: job failed for index_id={}: {e}", + job.get_str("index_id").unwrap_or("?"), + ); + } + } + } +} + +async fn run_gsi_backfill_job(storage: &MongoEngine, job: &Document) -> Result<(), StorageError> { + let index_id = job + .get_str("index_id") + .map_err(|_| StorageError::Internal("missing index_id".to_owned()))? + .to_owned(); + let id_doc = job + .get_document("_id") + .map_err(|_| StorageError::Internal("missing _id".to_owned()))?; + let table_id = id_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing _id.table_id".to_owned()))? + .to_owned(); + + let key_info = storage.table_key_info_by_table_id_impl(&table_id).await?; + + let idx_key_schema_bson = job + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema".to_owned()))?; + let idx_key_schema: Vec = bson::from_bson(idx_key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("key_schema parse: {e}")))?; + + let projection: Projection = job + .get("projection") + .and_then(|p| bson::from_bson(p.clone()).ok()) + .unwrap_or(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }); + + let mut cursor = job.get("backfill_cursor").cloned(); + let indexes_coll = storage.catalog_db.collection::("indexes"); + + loop { + let progress = storage + .backfill_gsi_batch( + &key_info, + &index_id, + &idx_key_schema, + &projection, + cursor.as_ref(), + GSI_BACKFILL_BATCH, + ) + .await?; + + if progress.done { + // Full-scan complete. Flip to ACTIVE and drop the cursor. + indexes_coll + .update_one( + doc! { "index_id": &index_id, "index_status": "CREATING" }, + doc! { + "$set": { "index_status": "ACTIVE" }, + "$unset": { "backfill_cursor": "" }, + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tracing::info!( + "GSI backfill worker: index_id={index_id} ACTIVE (last batch scanned {} docs)", + progress.scanned, + ); + return Ok(()); + } + + if let Some(ref last_id) = progress.last_id { + indexes_coll + .update_one( + doc! { "index_id": &index_id }, + doc! { "$set": { "backfill_cursor": last_id.clone() } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + cursor = Some(last_id.clone()); + } else { + // Empty batch but not done — treat as done to avoid an + // infinite loop. Shouldn't happen in practice since + // backfill_gsi_batch marks done when scanned < batch_size. + return Ok(()); + } + } +} + async fn retry_pending_indexes(storage: &MongoEngine) { let Ok(pending) = MetadataEngine::all_tables_with_ttl(storage).await else { return; From 2d4d9ecdabf9a443f4477bd61b004082e73226cb Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 16:48:25 -0700 Subject: [PATCH 35/83] fix(mongodb): validate index-key types on write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PutItem and UpdateItem never invoked `extenddb_core::validation::validate_index_keys`. Two consequences: 1. An item with a wrong-type GSI-key attribute (e.g. a string where the schema declares N) silently produced a malformed index doc: `data/mod.rs::index_document` skips the typed `sk_*` field on type mismatch, so the row is written without its sort-key and later deletes can't match it. The GSI row leaks forever. 2. An empty-string / empty-binary index-key silently persisted; DDB rejects at write time with a top-level ValidationException. Wire up the validation: - `fetch_index_key_schemas` returns `(index_name, key_schema)` pairs for the table. Gated on the GSI cache so no-GSI tables pay nothing. - `validate_index_keys_for_item` runs before writes on the single- item paths (put_item_impl / update_item_impl), matching postgres put_item.rs — surfaces as `StorageError::Validation`. - Inside TransactWriteItems, the same check runs on Put's `item` and Update's post-apply `item`; a violation becomes a per-item `ValidationError` cancellation reason (matches postgres data/transactions.rs). RFC-0003 §2.3. --- crates/storage-mongodb/src/data_engine.rs | 147 ++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index bd2229d0..6e9d25b8 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -247,6 +247,12 @@ impl MongoEngine { maps: &ExpressionMaps, stream: Option<&StreamCapture>, ) -> Result, StorageError> { + // Up-front index-key validation — must run before any write + // work so the caller sees a top-level ValidationException on + // wrong-type or empty index-key attributes (D-M10, RFC-0003 + // §2.3). + self.validate_index_keys_for_item(key_info, &item).await?; + let coll_name = data_collection_name(&key_info.table_id); let coll = self.data_db.collection::(&coll_name); @@ -714,6 +720,14 @@ impl MongoEngine { expression::apply_update(actions, &mut new_item, maps) .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + // Reject wrong-type or empty index-key attributes on + // the resulting item — D-M10, RFC-0003 §2.3. Same + // shape as put_item's up-front check, but the + // post-update item is what actually gets written. + self.validate_index_keys_for_item(key_info, &new_item) + .await + .map_err(TxErr::Fatal)?; + let mut new_doc = item_to_document( &new_item, &key_info.key_schema, @@ -1476,6 +1490,83 @@ impl MongoEngine { // ── GSI Sync ────────────────────────────────────────────────────── + /// Fetch (index_name, key_schema) for every index on the table. + /// Used by up-front input validation so PutItem / UpdateItem + /// rejects wrong-type or empty index-key attributes with a + /// top-level ValidationException before doing any write work + /// (D-M10, matches postgres put_item.rs). + async fn fetch_index_key_schemas( + &self, + table_id: &str, + ) -> Result)>, StorageError> { + use futures::TryStreamExt; + + if let Some(false) = self.gsi_cache_get_fresh(table_id) { + return Ok(Vec::new()); + } + + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut cursor = indexes_coll + .find(doc! { "_id.table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::new(); + while let Some(idx_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let index_name = match idx_doc + .get_document("_id") + .and_then(|d| d.get_str("index_name")) + { + Ok(n) => n.to_string(), + Err(_) => continue, + }; + let key_schema: Vec = match idx_doc.get("key_schema") { + Some(ks) => bson::from_bson(ks.clone()).unwrap_or_default(), + None => continue, + }; + out.push((index_name, key_schema)); + } + Ok(out) + } + + /// Reject an item whose secondary-index key attributes have the + /// wrong scalar type or are empty. Called by put/update before + /// the transaction is opened — matches postgres semantics of + /// surfacing this as a top-level ValidationException rather than + /// letting sync_indexes silently drop the malformed index doc + /// (`data/mod.rs::index_document` skips typed sk fields on type + /// mismatch, leaving the row un-locatable for subsequent + /// deletes). RFC-0003 §2.3. + async fn validate_index_keys_for_item( + &self, + key_info: &TableKeyInfo, + item: &Item, + ) -> Result<(), StorageError> { + let idx_pairs = self.fetch_index_key_schemas(&key_info.table_id).await?; + if idx_pairs.is_empty() { + return Ok(()); + } + let refs: Vec> = idx_pairs + .iter() + .map( + |(name, ks)| extenddb_core::validation::IndexKeyRef { + index_name: name.as_str(), + key_schema: ks.as_slice(), + }, + ) + .collect(); + extenddb_core::validation::validate_index_keys( + item, + &refs, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string())) + } + async fn sync_indexes( &self, key_info: &TableKeyInfo, @@ -2085,6 +2176,34 @@ impl MongoEngine { TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; + // Index-key type/empty faults inside a transaction + // surface as per-item cancellation reasons (matches + // postgres data/transactions.rs). D-M10. + let idx_pairs = self + .fetch_index_key_schemas(&key_info.table_id) + .await + .map_err(TransactOpError::Storage)?; + if !idx_pairs.is_empty() { + let idx_refs: Vec> = + idx_pairs + .iter() + .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { + index_name: n.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); + extenddb_core::validation::validate_index_keys( + item, + &idx_refs, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + } + let coll_name = data_collection_name(&key_info.table_id); let coll = self.data_db.collection::(&coll_name); let key_filter = @@ -2317,6 +2436,34 @@ impl MongoEngine { TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; + // Validate index-key types/emptiness on the post-update + // item; a violation here surfaces as a per-item + // cancellation reason. D-M10, RFC-0003 §2.3. + let idx_pairs = self + .fetch_index_key_schemas(&key_info.table_id) + .await + .map_err(TransactOpError::Storage)?; + if !idx_pairs.is_empty() { + let idx_refs: Vec> = + idx_pairs + .iter() + .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { + index_name: n.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); + extenddb_core::validation::validate_index_keys( + &item, + &idx_refs, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + } + let new_doc = item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions) .map_err(TransactOpError::Storage)?; From 821403aa9b134d8ec28903183e971195001dc867 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 16:59:23 -0700 Subject: [PATCH 36/83] fix(mongodb): store binary sort keys as hex strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MongoDB's BSON Binary comparison is length-first-then-content, which diverges from DynamoDB's unsigned lexicographic byte order. Example: DDB says `[0x01, 0xFF] < [0x02]` because 0x01 < 0x02; BSON says `[0x02] < [0x01, 0xFF]` because len=1 < len=2. Every Query with a binary sort-key range, BETWEEN, or comparison operator returned wrong result sets and wrong order. Store binary sort keys as lowercase hex strings in the typed `sk_b` and `base_sk_b` fields. String lex comparison of hex-encoded bytes preserves DDB byte order for both same-length and cross-length inputs: DDB `[0x01, 0xFF] < [0x02]` ↔ hex `"01ff" < "02"` ✓ DDB `[0x01] < [0x01, 0x00]` ↔ hex `"01" < "0100"` ✓ DDB `[] < [0x00]` ↔ hex `"" < "00"` ✓ Storage cost is 2x the raw bytes (bounded at 2KB per sort key by DDB's 1024-byte cap). Encoding cost is a 16-entry lookup table per byte. Same trade-off postgres makes with BYTEA + byte-by-byte comparison. Side effects: - `begins_with` on binary is now a real `$gte / $lt` range filter over the hex form: `[hex(prefix), hex(increment_bytes(prefix)))`. Removes the post-fetch retain() pass in query_impl, which was itself unsound — items outside the initial `limit+1` window were silently dropped even when they matched the prefix. That's D-M14 folded in. - `sk_to_bson` in the query builder projects to the same hex form so range predicates on `sk_b` match the stored representation. - `pk_filter` and `insert_typed_sk` write hex, and `data_engine` proptest coverage is unaffected (item_data still holds the raw bytes, so condition evaluation on `.B` values keeps working). No shipped deployments, no migration. RFC-0003 §1.4 / §7.4. --- crates/storage-mongodb/src/data/mod.rs | 70 ++++++++++++++++++----- crates/storage-mongodb/src/data_engine.rs | 57 ++++++++---------- 2 files changed, 80 insertions(+), 47 deletions(-) diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index 32754fb8..31c5afdb 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -104,13 +104,11 @@ fn insert_typed_sk( doc.insert(field, d); } (ScalarAttributeType::B, AttributeValue::B(b)) => { - doc.insert( - field, - bson::Binary { - subtype: bson::spec::BinarySubtype::Generic, - bytes: b.clone(), - }, - ); + // Store as hex string, not BSON Binary. See `binary_sk_to_hex` + // for the rationale — MongoDB's Binary sort order diverges + // from DDB's unsigned-lex byte order for unequal-length + // values (D-M5 / RFC-0003 §1.4). + doc.insert(field, binary_sk_to_hex(b)); } _ => { // Mismatched types are silently skipped — matches the existing @@ -287,6 +285,27 @@ fn sk_to_text(value: &AttributeValue) -> Result { } } +/// Encode a byte slice as a lowercase hex string. +/// +/// Used to store binary sort keys as strings in the typed +/// `sk_b`/`base_sk_b` fields. Lexicographic comparison of hex-encoded +/// strings preserves DynamoDB's unsigned-lex byte order — MongoDB's +/// native BSON Binary comparison is length-first-then-content, which +/// diverges from DDB for values of different lengths (e.g., DDB says +/// `[0x01,0xFF] < [0x02]`; BSON Binary reverses that). Hex strings +/// also make `begins_with` implementable as a plain string range +/// filter instead of a full-partition post-fetch scan. RFC-0003 §1.4. +#[must_use] +pub fn binary_sk_to_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &b in bytes { + out.push(HEX[(b >> 4) as usize] as char); + out.push(HEX[(b & 0x0f) as usize] as char); + } + out +} + /// Build a primary key filter for `MongoDB` queries. pub fn pk_filter( key: &Item, @@ -318,13 +337,9 @@ pub fn pk_filter( } ScalarAttributeType::B => { if let AttributeValue::B(b) = sk_value { - filter.insert( - "sk_b", - bson::Binary { - subtype: bson::spec::BinarySubtype::Generic, - bytes: b.clone(), - }, - ); + // Hex-encoded string, matching how insert_typed_sk + // writes sk_b — see D-M5. + filter.insert("sk_b", binary_sk_to_hex(b)); } } } @@ -596,4 +611,31 @@ mod tests { assert_eq!(doc.get_str("base_pk").unwrap(), "cust1"); assert_eq!(doc.get_str("base_sk_s").unwrap(), "order1"); } + + #[test] + fn binary_sk_to_hex_preserves_ddb_byte_order() { + // DynamoDB compares binary sort keys as unsigned lex bytes. + // The stored hex-string form must preserve that ordering under + // MongoDB's default lexicographic string comparison — verify + // both same-length and cross-length cases. + let a = binary_sk_to_hex(&[0x01, 0xff]); + let b = binary_sk_to_hex(&[0x02]); + // DDB: [0x01, 0xff] < [0x02]. Hex: "01ff" < "02". + assert!(a < b, "{a} < {b}"); + + // Shorter-prefix rule: [0x01] < [0x01, 0x00] in DDB. + let a = binary_sk_to_hex(&[0x01]); + let b = binary_sk_to_hex(&[0x01, 0x00]); + assert!(a < b); + + // Same first byte, longer runner in DDB. + let a = binary_sk_to_hex(&[0x01, 0x00, 0x00]); + let b = binary_sk_to_hex(&[0x02]); + assert!(a < b); + + // Empty is the smallest. + let empty = binary_sk_to_hex(&[]); + let single = binary_sk_to_hex(&[0x00]); + assert!(empty < single); + } } diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 6e9d25b8..cbb2be6a 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -27,8 +27,8 @@ use extenddb_storage::{ use crate::MongoEngine; use crate::condition::condition_to_filter; use crate::data::{ - composite_id, data_collection_name, document_to_item, index_document, index_entry_filter, - item_to_document, pk_filter, sk_field_name, sk_suffix, + binary_sk_to_hex, composite_id, data_collection_name, document_to_item, index_document, + index_entry_filter, item_to_document, pk_filter, sk_field_name, sk_suffix, }; use crate::pushdown::{Pushable, is_pushable}; @@ -1091,27 +1091,12 @@ impl MongoEngine { .map(document_to_item) .collect::, _>>()?; - // Post-fetch filtering for binary begins_with (BSON Binary comparison - // sorts by length first, making $gte/$lt unreliable for prefix matching). - if let Some(SortKeyCondition::BeginsWith { prefix, .. }) = &key_condition.sk_condition { - let prefix_av = resolve_key_expr(prefix, maps)?; - if let AttributeValue::B(ref prefix_bytes) = prefix_av - && let Some((sk_name, _)) = - sk_info(&effective_key_schema, &key_info.attribute_definitions) - { - items.retain(|item| { - item.get(sk_name) - .and_then(|v| { - if let AttributeValue::B(b) = v { - Some(b.starts_with(prefix_bytes)) - } else { - None - } - }) - .unwrap_or(false) - }); - } - } + // Binary begins_with used to require a post-fetch pass because + // BSON Binary comparison is length-first and $gte/$lt-on-Binary + // dropped matches whenever the prefix was shorter than the stored + // value. Since D-M5 stores binary sort keys as hex strings, the + // $gte/$lt filter emitted by build_sk_filter is now authoritative + // and no post-fetch filtering is needed. RFC-0003 §1.4. // Handle pagination. For an index query, LEK carries both the // index-key components and the base-key components so the next @@ -3141,12 +3126,16 @@ fn build_sk_filter( let upper = increment_string(p); Ok(doc! { sk_field: { "$gte": p.as_str(), "$lt": &upper } }) } - AttributeValue::B(ref _b) => { - // BSON Binary comparison sorts by length first, then by content. - // This means $gte/$lt range queries don't work for prefix matching - // when the prefix is shorter than the stored values. Return an empty - // filter here and let the caller do post-fetch prefix filtering. - Ok(Document::new()) + AttributeValue::B(ref b) => { + // Binary sort keys are stored as hex strings (D-M5), + // so begins_with is a plain lexicographic range: + // `sk_b >= hex(prefix) AND sk_b < increment(hex(prefix))`. + // The exclusive upper bound is the next hex prefix, + // computed by incrementing the raw bytes and encoding + // again — carries are handled by `increment_bytes`. + let lo = binary_sk_to_hex(b); + let hi = binary_sk_to_hex(&increment_bytes(b)); + Ok(doc! { sk_field: { "$gte": lo, "$lt": hi } }) } _ => Err(StorageError::Validation( "begins_with requires string or binary sort key".to_string(), @@ -3196,10 +3185,12 @@ fn sk_to_bson( "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" )) }), - (ScalarAttributeType::B, AttributeValue::B(b)) => Ok(bson::Bson::Binary(bson::Binary { - subtype: bson::spec::BinarySubtype::Generic, - bytes: b.clone(), - })), + // Binary sort keys are stored as hex-encoded strings; see + // `binary_sk_to_hex` and the D-M5 rationale. Query/BETWEEN + // filters must project to the same encoding. + (ScalarAttributeType::B, AttributeValue::B(b)) => { + Ok(bson::Bson::String(binary_sk_to_hex(b))) + } _ => Err(StorageError::Internal("sort key type mismatch".to_string())), } } From 3090f874cd8fcb9842e5f1bf457178254fb68361 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:01:37 -0700 Subject: [PATCH 37/83] fix(mongodb): don't drop items in Parallel Scan under key-hash skew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel Scan uses a per-item post-fetch segment filter: `crc32(pk) % total_segments == segment`. It combined that filter with a hard server-side `limit = (limit + 1) * total_segments`. The intent was "fetch enough to have limit+1 rows in the target segment after filtering," but under any pk-hash skew the whole window could land in other segments — the target segment fills too few items, the scan returns short with no LastEvaluatedKey, and every item beyond the window is silently dropped forever. DDB guarantees every item is returned by exactly one segment. Replace the pre-computed hard limit with lazy cursor iteration: drop the `.limit()` on FindOptions, stream via `try_next`, apply the segment filter per-item, stop when items.len() reaches `limit + 1` (enough to emit a LEK) or the cursor exhausts. MongoDB batches internally so this consumes at most one extra network batch beyond what we return. Same shape works whether or not segment/total_segments are set — non-parallel scans just never see the continue branch. RFC-0003 §7.3. --- crates/storage-mongodb/src/data_engine.rs | 62 ++++++++++++----------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index cbb2be6a..2221c875 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1255,20 +1255,6 @@ impl MongoEngine { } } - // Parallel scan segment filtering - // segment/total_segments use CRC32 hash of pk modulo total_segments - let apply_segment_filter = segment.is_some() && total_segments.is_some(); - - // Apply limit - let fetch_limit = limit.map(|l| { - let extra = l + 1; - if apply_segment_filter { - extra * total_segments.unwrap_or(1) - } else { - extra - } - }); - // Sort key. Index scans sort by (pk, sk?, base_pk, base_sk?) so // pagination is well-defined across items sharing index keys. // Base-table scans sort by _id (unique). @@ -1290,27 +1276,45 @@ impl MongoEngine { doc! { "_id": 1 } }; + // Lazy cursor iteration. The segment filter (CRC32 hash of pk + // mod total_segments) is applied per-item after fetching, so + // any hard server-side limit interacts badly with skew: with + // a modest hot-key concentration, a whole `(limit+1) * + // total_segments` window can land in one segment and leave + // the others empty, terminating the scan early with the + // remaining items silently dropped. RFC-0003 §7.3. + // + // Instead, stream the cursor and stop when either + // (a) we have `limit + 1` in-segment items (so we know we + // need a LEK for the next page), or + // (b) the cursor is exhausted. + // mongo batches under the hood (~101 docs per network trip), + // so this is efficient without a hard limit — we consume at + // most one extra network batch beyond what we return. let opts = mongodb::options::FindOptions::builder() .sort(sort_doc) - .limit(fetch_limit) .build(); - let cursor = coll + let mut cursor = coll .find(filter) .with_options(opts) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let docs: Vec = cursor - .try_collect() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - let mut items: Vec = Vec::new(); - for doc in &docs { - let item = document_to_item(doc)?; + let target = limit.map(|l| { + #[allow(clippy::cast_sign_loss)] + let l = l as usize; + l + 1 + }); + + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let item = document_to_item(&doc)?; - // Apply segment filter if needed if let (Some(seg), Some(total)) = (segment, total_segments) { let pk_text = composite_pk_to_text(&item, &key_info.key_schema)?; let hash = crc32fast::hash(pk_text.as_bytes()); @@ -1325,12 +1329,10 @@ impl MongoEngine { items.push(item); - // Check if we have enough items - if let Some(l) = limit { - #[allow(clippy::cast_sign_loss)] - if items.len() > l as usize { - break; - } + if let Some(t) = target + && items.len() >= t + { + break; } } From 2f15ae9621f069530ce0b0537f4183036ca0cc37 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:09:16 -0700 Subject: [PATCH 38/83] fix(mongodb): reject non-primary read preferences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A connection string like `mongodb://.../?readPreference=secondaryPreferred` would silently route reads to a replica and return stale data. DDB's `ConsistentRead=true` promises linearizable reads, which MongoDB can only deliver from the primary. The caller has no way to detect the silent divergence. Reject at MongoEngine::new — parse the connection string via the driver's ClientOptions and refuse any selection_criteria that isn't `ReadPreference::Primary` with a clear Connection error. Operators get told what to change; the server fails to start rather than running in a mode that lies to consumers. --- crates/storage-mongodb/src/lib.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index c5d7eecd..2fb69d0e 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -260,6 +260,29 @@ impl MongoEngine { .map_err(|e| StorageError::Connection(e.to_string()))?; options.max_pool_size = Some(max_connections); + // Reject non-primary read preferences. DynamoDB's `ConsistentRead=true` + // requires linearizable reads; MongoDB's Primary read concern is the + // only mode that provides that. A connection string like + // `mongodb://.../?readPreference=secondaryPreferred` would silently + // route reads to a secondary and return stale data — a fidelity + // violation the caller has no way to detect. + if let Some(sel) = options.selection_criteria.as_ref() { + use mongodb::options::{ReadPreference, SelectionCriteria}; + let is_non_primary = match sel { + SelectionCriteria::ReadPreference(rp) => !matches!(rp, ReadPreference::Primary), + _ => false, + }; + if is_non_primary { + return Err(StorageError::Connection( + "MongoDB connection string must use readPreference=primary. \ + Non-primary read preferences (secondary, secondaryPreferred, \ + nearest, primaryPreferred) route reads to replicas and \ + silently break ConsistentRead=true." + .to_owned(), + )); + } + } + let client = mongodb::Client::with_options(options) .map_err(|e| StorageError::Connection(e.to_string()))?; From 5ddc1de9699a4b37c50b4c34469c944a8871717d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:11:08 -0700 Subject: [PATCH 39/83] =?UTF-8?q?fix(mongodb):=20keep=20idempotency=20rete?= =?UTF-8?q?ntion=20=E2=89=A410=20min?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MongoDB's TTL monitor runs on a ~60s cadence, so `expireAfterSeconds = 600` deletes tokens anywhere from 10:00 to ~11:00 after insertion. The DDB spec is a 10-minute dedup window; drifting above it means a retry at 10:30 could still be deduplicated when it shouldn't be. Two-part fix: - Shrink the TTL index to 540s (9 min). Worst-case retention with the monitor's tail is ≤10 min. - Filter reads on `created_at`: rows older than 600 ms · 1000 count as absent, independent of TTL cadence. Applied in both the pre- check read and the E11000-follow-up winner lookup. A winner that ages out between our insert attempt and the follow-up read is treated as retryable rather than a stale dup. --- crates/storage-mongodb/src/bootstrapper.rs | 11 +++++-- crates/storage-mongodb/src/data_engine.rs | 34 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index ee0cc713..e9038a4a 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -90,12 +90,19 @@ impl Bootstrapper for MongoBootstrapper { let coll = db.collection::("idempotency_tokens"); - // Create TTL index on idempotency_tokens.created_at (10 min expiry) + // DDB spec: `ClientRequestToken` dedups retries within a 10-minute + // window. MongoDB's TTL monitor runs on a ~60s cadence, so an + // `expireAfterSeconds = 600` index deletes rows anywhere from + // 10:00 to ~11:00 minutes after `created_at` — retention drifts + // above the spec. Shrink to 540s (9 min) so worst-case retention + // is ≤10 min. The data-plane read path also filters on `created_at` + // to enforce the boundary strictly, independent of TTL monitor + // cadence (see `transact_write_items_impl`). let ttl_index = IndexModel::builder() .keys(doc! { "created_at": 1 }) .options( IndexOptions::builder() - .expire_after(std::time::Duration::from_secs(600)) + .expire_after(std::time::Duration::from_secs(540)) .build(), ) .build(); diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 2221c875..ae6b5ef9 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -2019,6 +2019,21 @@ impl MongoEngine { Err(e) => return Err(StorageError::Internal(e.to_string())), }; + // Filter out rows older than the DDB spec's 10-minute + // dedup window. Even with the TTL index set to 540s, + // MongoDB's TTL monitor runs on a ~60s cadence so a + // just-expired row can linger briefly. Treating a stale + // row as "not present" makes the read strictly correct + // regardless of monitor timing. See D-m4. + let existing = existing.filter(|doc| { + doc.get_datetime("created_at").is_ok_and(|dt| { + let age_ms = mongodb::bson::DateTime::now() + .timestamp_millis() + .saturating_sub(dt.timestamp_millis()); + age_ms < 600_000 + }) + }); + if let Some(existing_doc) = existing { let stored_fp = existing_doc.get_str("fingerprint").unwrap_or_default(); return Err(if stored_fp == key.fingerprint { @@ -2052,6 +2067,25 @@ impl MongoEngine { }) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Same 10-min age filter as the pre-check — + // don't let a barely-expired row masquerade + // as a live token. D-m4. + let winner = winner.filter(|d| { + d.get_datetime("created_at").is_ok_and(|dt| { + let age_ms = mongodb::bson::DateTime::now() + .timestamp_millis() + .saturating_sub(dt.timestamp_millis()); + age_ms < 600_000 + }) + }); + // If the winner aged out between our insert + // and this follow-up read, the row will be + // TTL'd shortly and the request is not a + // real dup — retry so the next attempt + // inserts fresh. + if winner.is_none() { + return Ok(AttemptOutcome::Retry); + } return Err( match winner.as_ref().and_then(|d| d.get_str("fingerprint").ok()) { Some(fp) if fp == key.fingerprint => { From 06274b3a6ac4b184387382846397ba33c0aa1cdc Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:11:53 -0700 Subject: [PATCH 40/83] fix(mongodb): index stream_records(shard_id, sequence_number) GetRecords filters `stream_records` by `shard_id` and paginates by `sequence_number > cursor` under an ascending sort. Without a compound index on those two fields, every consumer poll ran a full-collection scan whose cost grew linearly in the retained record count (up to 24 hours' worth per D-M11). Add the index in the bootstrapper alongside the existing TTL index. Prefix order matches the query shape so mongo can serve `find({shard_id: X, sequence_number: {$gt: N}}).sort({sequence_number: 1})` directly from the index. --- crates/storage-mongodb/src/bootstrapper.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index e9038a4a..128edab9 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -153,7 +153,8 @@ impl Bootstrapper for MongoBootstrapper { db.create_collection("stream_records") .await .map_err(|e| OpError::Internal(format!("Failed to create stream_records: {e}")))?; - db.collection::("stream_records") + let stream_records = db.collection::("stream_records"); + stream_records .create_index( IndexModel::builder() .keys(doc! { "created_at": 1 }) @@ -167,6 +168,21 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("stream_records TTL index: {e}")))?; + // Query-side index on (shard_id, sequence_number). GetRecords + // filters by shard_id and paginates by sequence_number > cursor + // with an ascending sort — the only way this can be efficient + // is if the index prefix matches. Without it, GetRecords does a + // full collection scan every time consumer polls, and cost + // grows linearly in the retained record count. D-m6. + stream_records + .create_index( + IndexModel::builder() + .keys(doc! { "shard_id": 1, "sequence_number": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_records shard index: {e}")))?; + Ok(()) } From edf3fb8982886e9c43d89922709646cf5024ccf5 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:14:37 -0700 Subject: [PATCH 41/83] fix(mongodb): index GSI/LSI data collections on create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the base-table collection got mongo indexes at CreateTable time. GSI/LSI collections were left to be lazy-created on first index-row insert, and — critically — never got any indexes on their query columns. Every Query / Scan against a secondary index therefore ran a full-collection scan; cost was linear in the index's row count. Add `create_index_data_collection` on MongoEngine and invoke it from all three sites that create a GSI/LSI catalog row: - CreateTable initial GSI list - CreateTable initial LSI list - UpdateTable GSI-create path (async D-C5 backfill flow) The compound index is `(pk, sk?, base_pk, base_sk?)` — the same tuple that `sync_indexes` and `scan_impl` sort by after D-C1's schema change. String sort keys get the `simple` collation so range comparisons stay byte-wise, matching the base table. --- crates/storage-mongodb/src/table_engine.rs | 101 ++++++++++++++++++++- 1 file changed, 97 insertions(+), 4 deletions(-) diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 09d44cdd..7219bd61 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -9,10 +9,11 @@ use mongodb::IndexModel; use mongodb::options::{Collation, CollationStrength, IndexOptions}; use extenddb_core::types::{ - BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, DescribeTableInput, - GsiDescription, IndexInfo, IndexType, KeyType, ListTablesInput, ListTablesOutput, - LsiDescription, OnDemandThroughput, ProvisionedThroughputDescription, ScalarAttributeType, - SseDescription, SseType, TableDescription, TableKeyInfo, TableStatus, UpdateTableInput, + AttributeDefinition, BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, + DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeySchemaElement, KeyType, + ListTablesInput, ListTablesOutput, LsiDescription, OnDemandThroughput, + ProvisionedThroughputDescription, ScalarAttributeType, SseDescription, SseType, + TableDescription, TableKeyInfo, TableStatus, UpdateTableInput, }; use extenddb_storage::TableEngine; use extenddb_storage::error::StorageError; @@ -297,6 +298,14 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + self.create_index_data_collection( + &index_id, + &gsi.key_schema, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + descs.push(GsiDescription { index_name: gsi.index_name.clone(), key_schema: gsi.key_schema.clone(), @@ -350,6 +359,14 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + self.create_index_data_collection( + &index_id, + &lsi.key_schema, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + descs.push(LsiDescription { index_name: lsi.index_name.clone(), key_schema: lsi.key_schema.clone(), @@ -780,6 +797,19 @@ impl MongoEngine { } })?; + // Pre-create the mongo collection + query indexes + // before the backfill worker starts writing — the + // worker's upserts would work on an un-indexed + // collection but subsequent GetItem/Query traffic + // on the CREATING index would run coll-scans. D-m7. + self.create_index_data_collection( + &index_id, + &create.key_schema, + &desc.key_schema, + &desc.attribute_definitions, + ) + .await?; + self.gsi_cache_set(&desc.table_id, true); } @@ -1235,4 +1265,67 @@ impl MongoEngine { on_demand_throughput, }) } + + /// Create the mongo collection for a GSI/LSI and add the indexes + /// its query path relies on. Every read against the index goes + /// through `find` predicates on `(pk, sk_*, base_pk, base_sk_*)` + /// with sorts on the same fields — without indexes those queries + /// devolve to a collection scan per read. Called from + /// `create_table_impl` (initial GSI/LSI) and the UpdateTable GSI- + /// create path (D-m7). + pub(crate) async fn create_index_data_collection( + &self, + index_id: &str, + index_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let coll_name = data_collection_name(index_id); + // create_collection is idempotent on recent MongoDB; a duplicate + // means we retried through a crash after the first success. Log + // + continue rather than surfacing an error to the caller. + if let Err(e) = self.data_db.create_collection(&coll_name).await { + tracing::debug!("index collection {coll_name} pre-exists or race: {e}"); + } + let coll = self.data_db.collection::(&coll_name); + + // Sort/paginate key: (pk, sk?, base_pk, base_sk?). Same tuple + // that scan_impl / query_impl sort by post-D-C1. Not unique — + // GSI keys are non-unique across base items; index docs are + // disambiguated by base-key components in the _id. + let idx_sk_field = sk_info(index_key_schema, attribute_definitions).map(|(_, t)| match t { + ScalarAttributeType::S => ("sk_s", true), + ScalarAttributeType::N => ("sk_n", false), + ScalarAttributeType::B => ("sk_b", false), + }); + let base_sk_field = sk_info(base_key_schema, attribute_definitions).map(|(_, t)| match t { + ScalarAttributeType::S => "base_sk_s", + ScalarAttributeType::N => "base_sk_n", + ScalarAttributeType::B => "base_sk_b", + }); + + let mut keys = doc! { "pk": 1 }; + if let Some((sk_f, _)) = idx_sk_field { + keys.insert(sk_f, 1); + } + keys.insert("base_pk", 1); + if let Some(base_sk_f) = base_sk_field { + keys.insert(base_sk_f, 1); + } + + // String sort keys need the `simple` collation so range + // comparisons behave as byte-wise, matching the query path. + let uses_string_sort = matches!(idx_sk_field, Some((_, true))) + || matches!(base_sk_field, Some("base_sk_s")); + let mut opts = IndexOptions::builder().build(); + if uses_string_sort { + opts.collation = Some(Collation::builder().locale("simple".to_string()).build()); + } + + coll.create_index(IndexModel::builder().keys(keys).options(opts).build()) + .await + .map_err(|e| StorageError::Internal(format!("index-coll index: {e}")))?; + + Ok(()) + } } From e2582f9e3686d75aeab3505040a351f3ab5cd391 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:16:17 -0700 Subject: [PATCH 42/83] fix(mongodb): match postgres stream_label format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mongo backend generated `stream_label` values via the `time` crate's default Iso8601 formatter — nanosecond precision, `Z` suffix, `.123456789Z` tail. Postgres emits second precision without a timezone (`YYYY-MM-DDThh:mm:ss`) via a to_char cast. Clients parsing labels round-tripped through one backend but not the other, and comparison-based lookups (list-streams pagination, by-label ARN parsing) could see two backend runs produce incompatible ARNs for the same table. Add `format_stream_label` and route all three writer sites — the initial CreateTable label, the UpdateTable re-enable path, and the label-restore branch — through it. Byte-for-byte matches postgres output. --- crates/storage-mongodb/src/table_engine.rs | 32 ++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 7219bd61..28bd4ce6 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -22,6 +22,27 @@ use extenddb_storage::util::{index_arn, sk_info, stream_arn, table_arn}; use crate::MongoEngine; use crate::data::data_collection_name; +/// Format a timestamp as a DynamoDB-style stream label: +/// `YYYY-MM-DDThh:mm:ss` (second precision, no timezone). +/// +/// Matches the postgres backend's +/// `to_char(NOW(), 'YYYY-MM-DD"T"HH24:MI:SS')` output byte-for-byte +/// so a stream ARN issued by one backend is parseable by tooling that +/// only ever saw the other. The `time` crate's `Iso8601::DEFAULT` +/// emits nanoseconds with a trailing `Z` — pushing that through AWS- +/// SDK parsers or postgres-shaped tests failed unpredictably. D-m8. +fn format_stream_label(now: time::OffsetDateTime) -> String { + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + now.year(), + u8::from(now.month()), + now.day(), + now.hour(), + now.minute(), + now.second(), + ) +} + impl TableEngine for MongoEngine { fn create_table( &self, @@ -155,8 +176,7 @@ impl MongoEngine { .is_some_and(|ss| ss.stream_enabled) { Some( - now.format(&time::format_description::well_known::Iso8601::DEFAULT) - .unwrap_or_else(|_| "unknown".to_string()), + format_stream_label(now), ) } else { None @@ -715,9 +735,7 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; if existing_shard.is_none() { - let label = time::OffsetDateTime::now_utc() - .format(&time::format_description::well_known::Iso8601::DEFAULT) - .unwrap_or_else(|_| "unknown".to_string()); + let label = format_stream_label(time::OffsetDateTime::now_utc()); update_doc.insert("stream_label", &label); self.init_stream_shards(table_id).await?; } else if table_doc @@ -729,9 +747,7 @@ impl MongoEngine { // Shards exist but the label was cleared by a // previous disable — restore a fresh label so the // ARN resolves again. - let label = time::OffsetDateTime::now_utc() - .format(&time::format_description::well_known::Iso8601::DEFAULT) - .unwrap_or_else(|_| "unknown".to_string()); + let label = format_stream_label(time::OffsetDateTime::now_utc()); update_doc.insert("stream_label", &label); } } From 0e6e1c2cbb83cde3764e2077700ad56592c40ac4 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:18:01 -0700 Subject: [PATCH 43/83] fix(mongodb): stub out obsolete non-transactional write_stream_record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `StreamEngine::write_stream_record` trait method predates D-C2 — back when stream records were written outside the data-write transaction. Every real caller now routes through `MongoEngine::write_stream_inline_in_session`, which enrolls the stream write in the same session as the base-table mutation so a rolled-back data write can't leave a phantom stream record behind (RFC-0003 §3.2, §5.1). The non-transactional impl remained in the file — 40+ lines of live code that would silently break stream/data atomicity if called externally. Replace the body with an immediate error explaining the supersession. Trait method stays (it's declared upstream in `extenddb-storage`), the body no longer does subtly-wrong work. --- crates/storage-mongodb/src/stream_engine.rs | 55 +++++++-------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index 8f75e780..554f6cc3 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -237,47 +237,28 @@ impl MongoEngine { } impl StreamEngine for MongoEngine { + /// Superseded by `MongoEngine::write_stream_inline_in_session` which + /// writes the stream record inside the same session as the data + /// write (D-C2 / RFC-0003 §3.2). The trait method has no callers + /// in the mongo backend after that change — it lives on only + /// because the `StreamEngine` trait still declares it. If invoked + /// externally, it would race against concurrent data writes: this + /// path does not enroll in any transaction and can commit a stream + /// record whose base-table write later rolls back. Return an + /// explicit error rather than performing a subtly-wrong write. fn write_stream_record( &self, - account_id: &str, - record: &StreamRecord, - shard_id: &str, - table_name: &str, + _account_id: &str, + _record: &StreamRecord, + _shard_id: &str, + _table_name: &str, ) -> BoxFuture<'_, Result<(), StorageError>> { - let account_id = account_id.to_owned(); - let record = record.clone(); - let shard_id = shard_id.to_owned(); - let table_name = table_name.to_owned(); Box::pin(async move { - let record_json = - serde_json::to_value(&record).map_err(|e| StorageError::Internal(e.to_string()))?; - let record_bson = - bson::to_bson(&record_json).map_err(|e| StorageError::Internal(e.to_string()))?; - - // Look up table_id - let tables_coll = self.catalog_db.collection::("tables"); - let table_doc = tables_coll - .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - .ok_or_else(|| { - StorageError::Internal(format!("Table {table_name} not found in catalog")) - })?; - let table_id = table_doc.get_str("table_id").unwrap_or_default(); - - let records_coll = self.data_db.collection::("stream_records"); - records_coll - .insert_one(doc! { - "sequence_number": &record.dynamodb.sequence_number, - "shard_id": &shard_id, - "table_id": table_id, - "event_name": event_name_ddb_str(record.event_name), - "record_data": record_bson, - "created_at": BsonDateTime::now(), - }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - Ok(()) + Err(StorageError::Internal( + "MongoDB backend: non-transactional write_stream_record is unused; \ + the data-plane path always routes through the in-session variant" + .to_owned(), + )) }) } From 7725626ac390a1bd6d55cd6e6940e632b5843493 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:20:08 -0700 Subject: [PATCH 44/83] fix(mongodb): correct begins_with upper bound on string sort keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `build_sk_filter` computed the exclusive upper bound for `BEGINS_WITH P` as `P + char::MAX` and matched with `$lt`. That misses any stored value equal to `P + char::MAX` — which still begins with `P` — and any value that extends past it (a string whose first char::MAX is followed by more content). Rare in practice but a real correctness gap. Replace `increment_string` with `next_string_prefix`: walk the prefix from the right, find the first char that isn't `char::MAX`, increment it (skipping the surrogate gap via `char::from_u32`), and truncate everything to its right. That yields the least string strictly greater than every P-starting string. Edge case: a prefix consisting entirely of `char::MAX` (or an empty prefix — but the engine layer already rejects that) has no upper bound; drop the `$lt` clause so mongo matches every value ≥ P. Matches DDB's behavior. --- crates/storage-mongodb/src/data_engine.rs | 82 ++++++++++++++++++++--- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index ae6b5ef9..e93f2560 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -3158,9 +3158,22 @@ fn build_sk_filter( let prefix_av = resolve_key_expr(prefix, maps)?; match prefix_av { AttributeValue::S(ref p) => { - // For begins_with on string sort keys: sk_s >= prefix AND sk_s < prefix + max_char - let upper = increment_string(p); - Ok(doc! { sk_field: { "$gte": p.as_str(), "$lt": &upper } }) + // `sk BEGINS_WITH P` matches every X where P is a + // prefix of X. Emit that as `sk >= P AND sk < P'`, + // where P' is the least string strictly greater + // than any P-starting string. + // + // `next_string_prefix` finds P' by incrementing + // the last non-`char::MAX` code point. If P is + // entirely `char::MAX`, no such P' exists — return + // just the lower-bound filter and let mongo match + // every string ≥ P (which is what DDB does). + match next_string_prefix(p) { + Some(upper) => Ok(doc! { + sk_field: { "$gte": p.as_str(), "$lt": upper } + }), + None => Ok(doc! { sk_field: { "$gte": p.as_str() } }), + } } AttributeValue::B(ref b) => { // Binary sort keys are stored as hex strings (D-M5), @@ -3242,12 +3255,44 @@ fn infer_sk_type_from_field(field: &str) -> ScalarAttributeType { } } -/// Increment a string to get the exclusive upper bound for `begins_with`. -fn increment_string(s: &str) -> String { - // Append the maximum Unicode code point - let mut result = s.to_string(); - result.push(char::MAX); - result +/// Compute the least string strictly greater than every string +/// beginning with `s`, used as the exclusive upper bound for +/// `sk BEGINS_WITH s`. +/// +/// Strategy: find the rightmost char in `s` that isn't `char::MAX`, +/// increment it, and truncate everything to its right. If every char +/// is `char::MAX` (an unlikely-but-real edge case), no upper bound +/// exists — return `None` so the caller can drop the `$lt` clause. +/// +/// The previous implementation appended `char::MAX` to `s` and used +/// `$lt`, which excluded any stored string equal to `s + char::MAX` +/// (or extending past it) — those still begin with `s` and DDB +/// matches them. D-m12. +fn next_string_prefix(s: &str) -> Option { + let chars: Vec = s.chars().collect(); + // Walk from the right, find the first char we can bump. + for i in (0..chars.len()).rev() { + if chars[i] < char::MAX { + let mut out = String::with_capacity(s.len()); + for c in &chars[..i] { + out.push(*c); + } + // char::from_u32 handles the surrogate gap by skipping + // to the next valid scalar. u32 → char via char::from_u32 + // returns None on the surrogate range D800..=DFFF, so + // walk past it. + let mut next = u32::from(chars[i]) + 1; + let bumped = loop { + if let Some(c) = char::from_u32(next) { + break c; + } + next += 1; + }; + out.push(bumped); + return Some(out); + } + } + None } /// Increment bytes to get the exclusive upper bound for `begins_with` on binary. @@ -3370,6 +3415,25 @@ mod tests { assert_eq!(returned, None); } + #[test] + fn next_string_prefix_ascii() { + // Basic ASCII: "abc" -> "abd" as the exclusive upper bound. + assert_eq!(next_string_prefix("abc").as_deref(), Some("abd")); + + // Trailing char::MAX skips back to a bumpable char. + // E.g. "abZ\u{10FFFF}" -> "ab[" + let s: String = ['a', 'b', 'Z', char::MAX].iter().collect(); + let expected: String = ['a', 'b', '['].iter().collect(); + assert_eq!(next_string_prefix(&s).as_deref(), Some(expected.as_str())); + + // All-char::MAX -> None (no bound; caller drops $lt clause). + let s: String = std::iter::repeat_n(char::MAX, 3).collect(); + assert!(next_string_prefix(&s).is_none()); + + // Empty string is also unbounded (no chars to bump). + assert!(next_string_prefix("").is_none()); + } + #[test] fn between_low_gt_high_binary() { assert!(sk_between_low_gt_high( From 7b76a4cccdf06ccbc97e4962bb39803c8fa1e130 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 17:22:43 -0700 Subject: [PATCH 45/83] fix(mongodb): lock in pushdown analyzer exclusions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The condition compiler in `condition.rs` has latent correctness bugs on numeric operands, set/list/map equality, mixed-type IN lists, and attribute_type with an unvalidated type tag. Option (c) from the D-M6 decision: keep the compiler, expand analyzer tests to lock the buggy shapes out of the pushdown path. - Tighten `attribute_type` in the analyzer: require the placeholder to resolve to a string that is exactly one of the ten DDB type tags (S, N, B, BOOL, NULL, L, M, SS, NS, BS). The compiler interpolates the tag verbatim into the mongo field path, so accepting an arbitrary string like `$ne` would produce a filter clause where "field" starts with a mongo operator. - Add four analyzer regression tests that lock in exclusion of every input shape D-M6 flagged as compile-time-buggy: numeric compares, set/list/map equality, IN, and attribute_type with an invalid tag. Plus a positive test that `attribute_type` with a valid tag still pushes. Compiler bodies stay unchanged — the analyzer is now the load- bearing correctness boundary, and its expansion is checked in with tests. --- crates/storage-mongodb/src/pushdown.rs | 113 +++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 6 deletions(-) diff --git a/crates/storage-mongodb/src/pushdown.rs b/crates/storage-mongodb/src/pushdown.rs index 4762566c..2f599b20 100644 --- a/crates/storage-mongodb/src/pushdown.rs +++ b/crates/storage-mongodb/src/pushdown.rs @@ -86,12 +86,33 @@ fn walk(expr: &Expr, maps: &ExpressionMaps) -> Pushable { if args.len() != 2 { return Pushable::No("attribute_type arity"); } - // The type argument must resolve to a String (the type - // tag: "S", "N", "B", "BOOL", "NULL", "L", "M", "SS", - // "NS", "BS"). Anything else is a parse-time error but - // we defensively check. - if !arg_resolves_to_scalar_type(&args[1], maps, AttrKind::S) { - return Pushable::No("attribute_type non-string tag"); + // The type argument must resolve to a String whose + // value is exactly one of the DDB type tags. The + // compiler in condition.rs inserts the tag verbatim + // into the mongo field path via + // `format!("{field}.{type_name}")`; without this + // whitelist a placeholder like `":t": {"S": "$ne"}` + // would produce a filter clause with a `$`-prefixed + // "field" that mongo would interpret as an operator. + // Keeping the whitelist in the analyzer means the + // compiler is only ever reached with a known-safe + // tag. RFC-0003 §6.1 / §8 (D-M6). + let Expr::Placeholder(name) = &args[1] else { + return Pushable::No("attribute_type tag not a placeholder"); + }; + let Ok(val) = maps.resolve_value(name) else { + return Pushable::No("attribute_type tag unresolvable"); + }; + match val { + AttributeValue::S(tag) => { + const VALID: &[&str] = &[ + "S", "N", "B", "BOOL", "NULL", "L", "M", "SS", "NS", "BS", + ]; + if !VALID.contains(&tag.as_str()) { + return Pushable::No("attribute_type tag not a DDB type name"); + } + } + _ => return Pushable::No("attribute_type non-string tag"), } Pushable::Yes } @@ -386,4 +407,84 @@ mod tests { // this test locks in that decision. assert!(!is_pushable(&expr, &maps).is_yes()); } + + // ── D-M6 exclusion tests ──────────────────────────────────────── + // + // The condition compiler in `condition.rs` has latent correctness + // bugs on numeric operands, set/list/map equality, mixed-type IN + // lists, and unvalidated attribute_type tags. The A5 analyzer is + // supposed to keep every one of those input shapes out of the + // pushdown path. These tests lock in that boundary so a future + // widening of the compiler doesn't accidentally admit a buggy + // shape without the analyzer being updated. + + #[test] + fn numeric_compare_is_not_pushable() { + // Numeric operands would compile to BSON string comparisons — + // the analyzer must reject. + let maps = maps_with(&[(":n", AttributeValue::N("42".into()))]); + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Lt, + right: Box::new(Expr::Placeholder(":n".into())), + }; + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn in_is_not_pushable() { + // IN with mixed-type list uses the first literal's type for + // all entries — analyzer must reject entirely. + let maps = maps_with(&[ + (":a", AttributeValue::S("x".into())), + (":b", AttributeValue::N("1".into())), + ]); + let expr = Expr::In { + operand: Box::new(path("a")), + list: vec![ + Expr::Placeholder(":a".into()), + Expr::Placeholder(":b".into()), + ], + }; + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn set_equality_is_not_pushable() { + // Eq/Ne on SS/NS/BS compiles to Bson::Null in the compiler. + // Analyzer classifies as `un-inferrable operand kind` and + // must reject. + let ss: std::collections::BTreeSet = + ["a".to_owned(), "b".to_owned()].into_iter().collect(); + let maps = maps_with(&[(":s", AttributeValue::SS(ss))]); + let expr = Expr::Compare { + left: Box::new(path("tags")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":s".into())), + }; + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn attribute_type_valid_tag_is_pushable() { + // "S" is a valid DDB type tag — analyzer admits. + let expr = Expr::Function { + name: "attribute_type".to_owned(), + args: vec![path("a"), Expr::Placeholder(":t".into())], + }; + let maps = maps_with(&[(":t", AttributeValue::S("S".into()))]); + assert!(is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn attribute_type_invalid_tag_is_not_pushable() { + // An arbitrary string as the tag is refused so the compiler + // never assembles `field.$evil` mongo path fragments. + let expr = Expr::Function { + name: "attribute_type".to_owned(), + args: vec![path("a"), Expr::Placeholder(":t".into())], + }; + let maps = maps_with(&[(":t", AttributeValue::S("$ne".into()))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } } From 21294f686de9fda351575b5d04448bd9ea70533d Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 8 Jul 2026 15:20:09 -0600 Subject: [PATCH 46/83] feat(): add mongodb storage backend rfc --- docs/rfcs/0000-mongodb-backend.md | 287 ++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 docs/rfcs/0000-mongodb-backend.md diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md new file mode 100644 index 00000000..37e41c69 --- /dev/null +++ b/docs/rfcs/0000-mongodb-backend.md @@ -0,0 +1,287 @@ +# RFC-206: MongoDB Storage Backend + +- Status: Draft +- Author: @diegotoledano95 +- Created: 2026-07-08 +- Tracking issue: #206 + +## Summary + +This RFC proposes adding MongoDB as a backend for ExtendDB. The goal is to let developers run DynamoDB-compatible workloads on MongoDB while preserving ExtendDB’s core value: a DynamoDB-compatible API over multiple storage backends. The implementation covers all mandatory traits defined in RFC-0002 and all optional traits, uses ExtendDB's existing `inventory`-based plugin registration system without modifying the server or engine layers, and is maintained by the MongoDB team who commit to ongoing ownership of the backend crate. + +## Motivation + +ExtendDB’s core premise is DynamoDB API compatibility over multiple storage backends. The initial reference PostgreSQL backend demonstrates the feasibility of this approach while opening the opportunity for other databases to participate. + +MongoDB is a natural fit as an additional database target: data model alignment; high read/write throughput through horizontal scalability; infrastructure fit. + +DynamoDB and MongoDB share the same data model approach - documents stored as schema-less JSON-like data. MongoDBs document model maps directly to the approach taken by DynamoDB with each item stored as a MongoDB BSON document with no impedance mismatch at the data model level. Unlike relational databases, the translation from JSON to BSON is direct without complicated relational mapping techniques required. + +Customers evaluating DynamoDB and MongoDB often consider scalability as a key requirement. ExtendDB’s deployment approach requiring high write throughput is matched by MongoDB’s replica set model via horizontal scaling. High read and write throughput across multiple nodes is a core tenant of MongoDB and aligns naturally with the scalability requirement for an ExtendDB customer. + +Organizations running ExtendDB, DynamoDB, and MongoDB have already evaluated the usefulness of a non-relational database approach. These shared customers do not want to run PostgreSQL or other relational databases solely for DynamoDB compatibility. Rather, taking advantage of the infrastructure they already run that aligns with the document model design and scalability requirements they require makes MongoDB a natural fit. + + +## Detailed design + +### Scope + +This RFC proposes: + +- A new optional `extenddb-storage-mongodb` crate at `crates/storage-mongodb/` +- A `mongodb` Cargo feature flag on the `extenddb` binary crate +- Backend registration through the existing `inventory`-based plugin system without changes to `crates/engine/`, `crates/server/`, `crates/auth/`, or `crates/core/` +- MongoDB-specific implementations of all mandatory and optional storage traits defined in RFC-0002 +- Setup documentation and sample configuration for MongoDB deployments + +This RFC does not propose: + +- Changes to the DynamoDB wire protocol or API response shapes +- MongoDB Atlas, Atlas Data API, or any hosted MongoDB service as a target +- Sharded cluster support (replica sets only) +- Changing the default backend from PostgreSQL +- Dual-write or online migration from PostgreSQL to MongoDB +- A generic document-store abstraction shared with future document database backends + +### Repository structure + +The backend lives at `crates/storage-mongodb/` in the main ExtendDB repository, following the mono-repo structure prescribed by RFC-0002. It is selected at build time via a `mongodb` Cargo feature flag on the `extenddb` binary crate. + + +Feature flag definition: `crates/bin/Cargo.toml` — `[features]` section defines `mongodb = ["extenddb-storage-mongodb"]` with the crate as an optional dependency. The `postgres` feature remains the default. An `all-backends` convenience flag is planned as part of gap resolution. + +### Plugin registration + +The backend registers itself with ExtendDB's `inventory`-based plugin system without modifying the server, engine, or auth layers. Four `inventory::submit!` calls in `lib.rs` register the backend for: bootstrapping (`extenddb init`), config parsing, settings store access, and server component construction (`extenddb serve`). + + +All four registration blocks: `crates/storage-mongodb/src/lib.rs`. The `ServerComponentsRegistration` block is the critical one — it is the factory function called when `extenddb serve --backend mongodb` is run. No changes were required in `crates/server/`, `crates/engine/`, or `crates/auth/`. + +### Database layout + +The backend uses two MongoDB databases: + +**`extenddb_catalog`** — metadata and management. Created on `extenddb init`. Contains 17 collections covering table definitions, index metadata, IAM users, groups, roles, access keys, policies, permissions boundaries, settings, metrics, login attempts, backups, and schema migration history. + +**`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named the same way with the index's ID. Two shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `idempotency_tokens` for transaction deduplication, and `counters` for sequence number generation. + + +Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()` creates all 17 catalog collections and their MongoDB indexes. Data database setup: `create_data_db()` in the same file creates `idempotency_tokens` with a 10-minute TTL index. Collection naming convention: `crates/storage-mongodb/src/data/mod.rs` — `data_collection_name()` and `index_collection_name()`. + +### Document structure for DynamoDB items + +Each DynamoDB item is stored as a MongoDB document with the following structure: + +``` +{ + _id: "partitionKeyValue#sortKeyValue", + pk: "partitionKeyValue", + sk_s: "sortKeyValue", // string sort keys + sk_n: Decimal128(...), // number sort keys, native MongoDB numeric type + sk_b: Binary(...), // binary sort keys + item_data: { ... full DynamoDB item in DynamoDB JSON format ... } +} +``` + +The `_id` field enables O(1) point lookups. The `pk` field is indexed separately to support partition scans (Query operations). Sort keys are stored in typed fields (`sk_s`, `sk_n`, `sk_b`) so MongoDB can apply native range comparisons with correct ordering — notably, numeric sort keys use MongoDB's `Decimal128` type rather than strings to ensure correct numeric ordering. The full item is stored in `item_data` using DynamoDB's own type-tagged format (`{"S": "hello"}`, `{"N": "42"}`, etc.), preserving all type information without lossy conversion. + +String sort key collections are created with `{ locale: "simple", strength: 3 }` collation, ensuring byte-for-byte ordering that matches DynamoDB's behavior rather than locale-aware Unicode ordering. + + +Document conversion functions: `crates/storage-mongodb/src/data/mod.rs` — `item_to_document()` (DynamoDB Item → BSON document) and `document_to_item()` (BSON document → DynamoDB Item). Sort key type handling including `Decimal128`: same file, `item_to_document()` `ScalarAttributeType::N` branch. Collation: `crates/storage-mongodb/src/table_engine.rs` — `CreateTable` implementation, index creation with `Collation` options. + +### Condition expression pushdown + +Condition expressions (`ConditionExpression` on PutItem, DeleteItem, UpdateItem) are compiled into MongoDB filter documents and executed server-side as part of atomic `findOneAndReplace` and `findOneAndDelete` operations. This means a conditional write is a single round-trip to MongoDB — no separate fetch, no application-level check, no race window between the check and the write. + +The compiler handles: `attribute_exists`, `attribute_not_exists`, `attribute_type`, `begins_with`, `contains`, `size`, `BETWEEN`, `IN`, `=`, `<>`, `<`, `<=`, `>`, `>=`, `AND`, `OR`, `NOT`. Because items are stored with DynamoDB type tags, compiled paths include the type suffix: `item_data.fieldName.S` for strings, `.N` for numbers. + + +Condition compiler: `crates/storage-mongodb/src/condition.rs` — `condition_to_filter()` is the entry point. Each DynamoDB function and operator has a corresponding compilation case. Unit tests in the same file demonstrate each compiled output. Usage in write operations: `crates/storage-mongodb/src/data_engine.rs` — `put_item_impl()`, `delete_item_impl()`, `update_item_impl()` each pass the compiled filter to MongoDB's `findOneAndReplace`/`findOneAndDelete`/`findOneAndUpdate`. + +### Query and Scan + +**Query** translates `KeyConditionExpression` to a MongoDB `find()` filter. Partition key equality maps to `{ pk: "" }`. Sort key conditions map to typed range filters on `sk_s`, `sk_n`, or `sk_b`. `ScanIndexForward: false` applies a descending sort. Pagination uses `ExclusiveStartKey` to add a `$gt` or `$lt` bound on the sort key, making each page fetch a single indexed range query. + +**Scan** performs a full collection scan with `.find({})`, paginated via sort-key-based cursor. Filter expressions are evaluated after retrieval. + +**Parallel scan** (`Segment` / `TotalSegments`) is handled in the application: each segment filters documents using `crc32(pk) % TotalSegments == Segment`. This means each segment scans the full collection. Pre-bucketing documents at write time would avoid this but adds overhead to every write for a feature that is rarely used in practice. The current tradeoff favors write-path simplicity. + +### Global Secondary Indexes (GSI) and Local Secondary Indexes (LSI) + +Each secondary index has its own MongoDB collection. On writes that modify indexed attributes, the backend updates the index collection in the same operation, maintaining synchronous GSI propagation. A `DashMap` in-memory cache on `MongoEngine` tracks which tables have GSIs, avoiding catalog lookups on every write to tables with no indexes. + + +GSI collection creation: `crates/storage-mongodb/src/table_engine.rs` — `CreateTable` implementation. GSI cache: `crates/storage-mongodb/src/lib.rs` — `MongoEngine` struct `gsi_cache` field. Index write propagation: `crates/storage-mongodb/src/data_engine.rs` — `put_item_impl()` and related write paths check the cache before updating index collections. + +### Transactions + +`TransactWriteItems` uses MongoDB multi-document ACID transactions — a client session is opened, all operations execute within it, and the session is committed or aborted atomically. This requires MongoDB to be running as a replica set (standalone MongoDB does not support multi-document transactions). `TransactGetItems` performs a consistent snapshot read. + +Idempotency tokens for `TransactWriteItems` are stored in the `idempotency_tokens` collection in `extenddb_data` with a 10-minute MongoDB TTL index, matching DynamoDB's 10-minute idempotency window. + + +Transaction implementation: `crates/storage-mongodb/src/data_engine.rs` — `transact_write_items_impl()`. Idempotency token storage: same file, idempotency check at the start of `transact_write_items_impl()`. Replica set requirement documentation: `docs/local-mongodb-setup.md` — replica set initialization section. + +### Write conflict handling + +**UpdateItem** uses optimistic concurrency. A `_v` version counter is stored on each document. The write path reads the current `_v`, applies the update expression in memory, sets `_v = current_version + 1`, then executes `replaceOne` filtered on both the primary key and the expected `_v`. If `matched_count == 0`, a concurrent writer incremented the version first. The operation retries with jittered exponential backoff (100 µs base, up to 50 attempts). Exhausted retries propagate the error to the caller. + +**PutItem and DeleteItem** use condition pushdown (see Condition expression pushdown). When `ReturnValuesOnConditionCheckFailure` is requested and the condition fails, a follow-up `find_one` fetches the existing item for the response. This matches DynamoDB's own best-effort semantics for the returned item on condition failure. + +**TransactWriteItems** runs all operations inside a single MongoDB ACID transaction with snapshot read concern and majority write concern. Transaction failures are not retried — the error propagates as `TransactionCanceled`. + +Write conflict handling: `crates/storage-mongodb/src/data_engine.rs` — `update_item_impl()`, version field handling and retry loop. + +### DynamoDB Streams + +DynamoDB Streams are implemented using explicit stream record storage in MongoDB collections, not MongoDB's native Change Streams feature. The explicit approach was adopted to maintain behavioral parity with the PostgreSQL backend and to retain full application control over sequence number generation, shard assignment, and record retention lifecycle — all of which the DynamoDB Streams API contract tightly specifies. + +Each table is assigned 4 shards at creation time. On each data write with streams enabled, the backend assigns the write to a shard by hashing the partition key with CRC32, generates a monotonically increasing 21-digit sequence number using MongoDB's atomic `findOneAndUpdate` with `$inc`, and writes a record to `stream_records`. `GetRecords` paginates using `{ "sequence_number": { "$gt": after } }` range queries with ascending sort. + + +Stream implementation: `crates/storage-mongodb/src/stream_engine.rs`. Shard initialization: `init_stream_shards()`. Sequence number generation: `next_sequence_number()` using `$inc` on a counters document. Shard assignment by CRC32 hash: `assign_shard()`. Inline stream write from data operations: `crates/storage-mongodb/src/data_engine.rs` — `write_stream_inline()`. + +### Time to Live (TTL) + +When TTL is enabled on a table, the backend creates a sparse MongoDB index on `item_data.{ttl_attribute}.N`. A background worker spawned at server startup sweeps expired items every 60 seconds in batches of 100. Each deletion uses `DataEngine::delete_item` with a condition expression re-checking expiry, preventing races. TTL deletions carry `UserIdentity { type: "Service", principalId: "dynamodb.amazonaws.com" }` on their stream records, matching DynamoDB's TTL stream record format. + + +TTL index creation: `crates/storage-mongodb/src/metadata_engine.rs` — `create_ttl_index()`. Background worker: `crates/storage-mongodb/src/ttl_worker.rs` — `ttl_cleanup_worker()`, `sweep_expired_items()`. Worker spawn: `crates/storage-mongodb/src/lib.rs` — `MongoRuntimeHooks::spawn_workers()`. UserIdentity on TTL stream records: `crates/storage-mongodb/src/ttl_worker.rs` — `sweep_expired_items()`, `ttl_identity` construction. + +### Control plane state transitions + +Table creation and deletion are asynchronous at the DynamoDB API level — `CreateTable` returns `CREATING` status, `DeleteTable` returns `DELETING`. A background `WorkerStore` implementation polls the `tables` catalog collection for entries whose `status_transition_at` timestamp has passed and completes the transition: flipping CREATING → ACTIVE, or for DELETING → dropped (drops the data collection, index collections, removes catalog entries and tags). + + +Worker implementation: `crates/storage-mongodb/src/worker_store.rs` — `process_control_plane_transitions()`. + +### Authentication and authorization + +ExtendDB's mandatory SigV4 authentication is fully supported. Access key secrets are stored AES-GCM encrypted in the `extenddb_catalog.access_keys` collection. The encryption key is a 256-bit random key generated during `extenddb init`, base64-encoded, and stored in `extenddb_catalog.settings` under `_id: "encryption_key"`. Admin passwords are bcrypt-hashed before storage in `extenddb_catalog.admin_users`. + +IAM policy evaluation fetches user-attached policies, group-attached policies (via `iam_group_members` → `iam_policies` join), role policies, permissions boundaries, and session policies from the catalog. + + +Encryption key generation and storage: `crates/storage-mongodb/src/bootstrapper.rs` — `bootstrap_encryption_key()`. Admin password hashing: same file, `bootstrap_admin_user()`. Access key decryption at request time: `crates/storage-mongodb/src/credential_store.rs`. IAM policy fetching: `crates/storage-mongodb/src/authorization_store.rs` — `fetch_user_policies()`, `fetch_user_group_policies()`, `fetch_role_policies()`, `fetch_session_data()`. + +### Backup + +`CreateBackup` uses MongoDB's server-side `$out` aggregation stage to copy a table's collection to a backup collection (`_backup_{backup_id}_{table_id}`) without transferring data through the application. `RestoreTableFromBackup` reads the backup collection and reconstructs the table. `DeleteBackup` drops the backup collection. + + +Backup implementation: `crates/storage-mongodb/src/backup_engine.rs`. + +### Operational requirements + +**Minimum MongoDB version: 8.0.** This is the minimum supported version for this backend. The MongoDB Rust driver 3.x is technically compatible with MongoDB 4.2+, but this backend targets 8.0 as the minimum supported server version. + +**Replica set required.** MongoDB must be configured as a replica set before running `extenddb init`. A standalone node does not support multi-document transactions (`TransactWriteItems`). A single-node replica set is sufficient for development and CI. Production deployments should use a 3-node replica set for high availability. + +**File descriptor limit.** Each MongoDB collection maps to one WiredTiger file. At 500 DynamoDB tables with 2 GSIs each (~1,500 collections), ensure `ulimit -n ≥ 65536` on the MongoDB host. See `docs/local-mongodb-setup.md` for platform-specific instructions. + +**Target scale.** This backend is designed for deployments of up to ~500 DynamoDB tables. At that scale, WiredTiger handles the collection count comfortably with default settings. Deployments significantly beyond this range have not been validated. + +Configuration is added under `[storage.mongodb]` in `extenddb.toml`: + +```toml +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +max_connections = 50 +max_catalog_connections = 20 +``` + +Initialization uses backend-selection flags on `extenddb init`: + +```text +extenddb init \ + --storage-backend mongodb \ + --storage-host 127.0.0.1 \ + --storage-port 27017 \ + --config extenddb.toml +``` + +Configuration struct: `crates/storage-mongodb/src/config.rs`. Sample configuration: `extenddb.sample.toml` — `[storage.mongodb]` section. Setup guide: `docs/local-mongodb-setup.md`. + +### Implementation summary + +| Crate modified | Change | +|---|---| +| `crates/storage-mongodb/` | New crate — full backend implementation | +| `crates/bin/Cargo.toml` | Added `mongodb` optional feature flag | +| `crates/bin/src/main.rs` | Added `#[cfg(feature = "mongodb")] extern crate` | +| `Cargo.toml` (workspace) | Added crate to members, added `mongodb`, `bson`, `dashmap` workspace dependencies | + +No changes to `crates/engine/`, `crates/server/`, `crates/storage/` (trait definitions), `crates/auth/`, or `crates/core/`. + + +Full diff: `mongodb-forks/extenddb` branch `extenddb-on-mongo` compared to `main`. The absence of changes in engine/server/auth/core crates can be verified directly in that diff. + +### Design decisions summary + +| Decision | Choice | Rationale | +|---|---|---| +| Conditional writes (PutItem, DeleteItem) | Filter pushdown into `findOneAndReplace` / `findOneAndDelete` | Single-document atomicity; no transaction overhead on the hot path | +| UpdateItem write conflict | Optimistic concurrency with `_v` version field + jittered backoff | Avoids transactions for single-item updates while preventing lost updates | +| GSI updates | Synchronous inline with `DashMap` cache | No Change Stream recovery complexity; GSI reads are strongly consistent | +| DynamoDB Streams | Inline writes to `stream_records` collection | Behavioral parity with PostgreSQL backend; explicit control over sequence numbers, shard assignment, and retention | +| Stream shards | 4 per table, CRC32 hash assignment | Predictable consumer parallelism; no catalog lookup at shard assignment time | +| Sort key numbers | Native BSON `Decimal128` | Correct ordering by value; no string-encoding tricks | +| Backups | Server-side `$out` aggregation stage | No client-side data transfer; no document size limitations | +| Parallel scan | Application-side `crc32(pk) % segments` filter | Avoids per-document write overhead of a pre-bucketed segment field | + +### Performance characteristics + +**Single-item writes (hot path).** Transaction-free. A PutItem with a condition expression is a single `findOneAndReplace` with a filter — one network round-trip, one WiredTiger document write. No locking, no multi-phase commit. + +**GSI write overhead.** For tables with no GSIs, the `gsi_cache` short-circuits to zero overhead — no catalog query, no additional I/O. For tables with GSIs, one catalog query fetches index definitions (cached for subsequent writes on the same table), plus one upsert or delete per index collection per write. + +**Stream write overhead.** When streams are enabled, each write adds one atomic `findOneAndUpdate` counter increment and one document insert into `stream_records`. + +**Query and Scan.** Direct index lookups on `{ pk, sk_* }`. Performance characteristics match any indexed MongoDB query. Parallel scans scan the full collection once per segment (see Query and Scan). + +**TransactWriteItems.** Multi-collection ACID transaction with snapshot read concern. Uncommon in practice — most DynamoDB workloads are single-item operations. + +### Testing + +Testing is organized in three layers. + +**Unit tests** cover pure logic without a live MongoDB instance: condition expression compilation (`condition.rs`), document encoding and decoding (`data/mod.rs`), sort key ordering, and sequence number generation. The MongoDB client is mocked at this layer. + +**Integration tests** run against a single-node replica set in Docker (`mongod --replSet rs0`). They cover the full table lifecycle, all item operations (including condition expressions), query and scan pagination, transactions, TTL worker behavior, stream record writes, GSI propagation, backup and restore, and all catalog and IAM operations. These execute as `cargo test -p extenddb-storage-mongodb`. + +**End-to-end tests** run the existing ExtendDB pytest suite (`tests/`) unchanged against a MongoDB-backed ExtendDB server. The pytest suite speaks the DynamoDB wire protocol and has no backend awareness — a passing run against MongoDB is equivalent to a passing run against PostgreSQL. This is the conformance test baseline required by RFC-0002. + +The CI job spins up a single-node MongoDB 8.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. + +## Drawbacks + +**Replica set requirement.** MongoDB must be run as a replica set for `TransactWriteItems` support. This adds operational complexity for users who currently run standalone MongoDB. Users who run standalone MongoDB will receive a runtime error on transactional operations. This is documented in setup guides and is a MongoDB architectural constraint, not an ExtendDB limitation. + +**Time to Live expiration throughput at scale.** DynamoDB's Time to Live deletion must emit stream records with a specific service identity. MongoDB's native Time to Live index operates at the storage engine level with no awareness of ExtendDB's stream system, so this implementation uses an application-level background worker that owns the full deletion lifecycle — finding expired items, deleting them, and emitting correctly attributed stream records. The worker runs every 60 seconds and processes expired items in batches of 100 per table. This is sufficient for ExtendDB's target deployment contexts. At very high expiration rates — tables where a large number of items expire per minute continuously — the worker will fall behind and the backlog will grow. This is a known limitation at scale, not a correctness issue, as DynamoDB's own contract only guarantees expiration within 48 hours rather than immediately (see `docs/differences-from-dynamodb.md`, TTL deletion row). + +## Alternatives + +### Use MongoDB Change Streams for DynamoDB Streams + +MongoDB has a native change data capture feature (Change Streams) that could back DynamoDB Streams. This approach was not fully evaluated. The implementation instead adopted the explicit `stream_records` collection approach used by the PostgreSQL backend, which gives ExtendDB full control over sequence number generation, shard assignment, record retention, and iterator behavior — all of which the DynamoDB Streams API contract tightly specifies. Reviewers are invited to weigh in on whether a comparative evaluation of the Change Streams approach should be documented before acceptance. + + +## Prior art + +**MongoDB document model and DynamoDB.** MongoDB's flexible document model has been noted as a natural fit for DynamoDB-style workloads in multiple independent analyses. Amazon DocumentDB (MongoDB-compatible) demonstrates AWS's own recognition of this overlap. The key difference in this implementation is that ExtendDB provides the full DynamoDB API layer — clients using the AWS SDK do not need to know they are talking to MongoDB. + +**Condition pushdown pattern.** Compiling application-level filter expressions into storage-native query operators is a well-established pattern in query engines (Apache Arrow DataFusion, Spark, Presto all implement predicate pushdown). The `condition.rs` compiler in this implementation applies the same principle at the storage backend level. + +--- + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is +a trademark of Amazon.com, Inc. From fe9637b72aeb33504695d5b31412fc335a48a730 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 18:04:14 -0700 Subject: [PATCH 47/83] docs(rfc): update RFC-206 to match implementation Rewrites three sections of docs/rfcs/0000-mongodb-backend.md where the RFC described an initially-planned design that differs from what the implementation does, plus small errata. Section rewrites: - Condition expression evaluation (was "Condition expression pushdown"). RFC previously described a single-round-trip design in which compiled MongoDB filters were pushed into findOneAndReplace / findOneAndDelete. Implementation evaluates conditions in-Rust against a loaded item inside a MongoDB client session that also wraps the write. The condition compiler exists as scaffolding for a future filter-pushdown optimization but is not on the correctness path. Section rewritten to describe the session-scoped approach, with the pushdown path documented as an alternative considered. - Backup. RFC previously described using MongoDB's server-side $out aggregation stage to per-backup collections named _backup_{backup_id}_{table_id}. Implementation writes items into a shared backup_items collection keyed by backup_arn, with backup metadata in extenddb_catalog.backups. Section rewritten; $out documented as a future optimization. - Performance characteristics. The "single-item writes: transaction-free" claim was too broad. Rewritten to reflect that the session wrap is what provides DynamoDB's atomicity contract, with a sessionless fast path planned as a follow-up optimization for the narrow case of tables with no streams and no GSIs. Errata: - Plugin registration: four -> five inventory::submit! calls (diagnostics store registration was omitted). - Database layout: 17 -> 18 catalog collections; enumerated the additional collections; noted iam_group_members and backup_items auto-create on first insert rather than in migrations. - Write conflict handling: OCC retry base 100us -> 50us (matches code at data_engine.rs:706). - Removed size from the list of supported compiled functions. - Removed the "all-backends" convenience flag from the roadmap. - Design decisions summary and implementation summary tables updated to reflect the rewrites (session-scoped conditional writes; shared backup_items collection; cmd_serve.rs added as a modified file). - Numeric sort keys: added a note that values exceeding Decimal128's 34 significant digits are rejected at write/query time (see docs/differences-from-dynamodb.md). - Collation: dropped the strength: 3 specifier; the code uses MongoDB's default (Tertiary) which matches DynamoDB behavior. No behavioral change to the proposed design. --- docs/rfcs/0000-mongodb-backend.md | 55 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index 37e41c69..1bd267d6 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -48,25 +48,25 @@ This RFC does not propose: The backend lives at `crates/storage-mongodb/` in the main ExtendDB repository, following the mono-repo structure prescribed by RFC-0002. It is selected at build time via a `mongodb` Cargo feature flag on the `extenddb` binary crate. -Feature flag definition: `crates/bin/Cargo.toml` — `[features]` section defines `mongodb = ["extenddb-storage-mongodb"]` with the crate as an optional dependency. The `postgres` feature remains the default. An `all-backends` convenience flag is planned as part of gap resolution. +Feature flag definition: `crates/bin/Cargo.toml` — `[features]` section defines `mongodb = ["extenddb-storage-mongodb"]` with the crate as an optional dependency. The `postgres` feature remains the default. Both features can be enabled together to compile a binary supporting both backends. ### Plugin registration -The backend registers itself with ExtendDB's `inventory`-based plugin system without modifying the server, engine, or auth layers. Four `inventory::submit!` calls in `lib.rs` register the backend for: bootstrapping (`extenddb init`), config parsing, settings store access, and server component construction (`extenddb serve`). +The backend registers itself with ExtendDB's `inventory`-based plugin system without modifying the server, engine, or auth layers. Five `inventory::submit!` calls in `lib.rs` register the backend for: bootstrapping (`extenddb init`), config parsing, settings store access, diagnostics store access, and server component construction (`extenddb serve`). -All four registration blocks: `crates/storage-mongodb/src/lib.rs`. The `ServerComponentsRegistration` block is the critical one — it is the factory function called when `extenddb serve --backend mongodb` is run. No changes were required in `crates/server/`, `crates/engine/`, or `crates/auth/`. +All five registration blocks: `crates/storage-mongodb/src/lib.rs`. The `ServerComponentsRegistration` block is the critical one — it is the factory function called when `extenddb serve --backend mongodb` is run. No changes were required in `crates/server/`, `crates/engine/`, or `crates/auth/`. ### Database layout The backend uses two MongoDB databases: -**`extenddb_catalog`** — metadata and management. Created on `extenddb init`. Contains 17 collections covering table definitions, index metadata, IAM users, groups, roles, access keys, policies, permissions boundaries, settings, metrics, login attempts, backups, and schema migration history. +**`extenddb_catalog`** — metadata and management. Created on `extenddb init`. Contains 18 collections covering: table definitions (`tables`), index metadata, accounts, tags, admin users, IAM users, groups, group memberships, roles, access keys, IAM sessions, policies, permissions boundaries, session data, settings, metrics, login attempts, continuous backup metadata, and schema migration history. `iam_group_members` and `backup_items` are used by the runtime but auto-created on first insert rather than during migrations. -**`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named the same way with the index's ID. Two shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `idempotency_tokens` for transaction deduplication, and `counters` for sequence number generation. +**`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named the same way with the index's ID. Shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `idempotency_tokens` for transaction deduplication, and `counters` for sequence number generation. -Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()` creates all 17 catalog collections and their MongoDB indexes. Data database setup: `create_data_db()` in the same file creates `idempotency_tokens` with a 10-minute TTL index. Collection naming convention: `crates/storage-mongodb/src/data/mod.rs` — `data_collection_name()` and `index_collection_name()`. +Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()` creates the migration-managed catalog collections and their MongoDB indexes; `schema_history` is created separately by `create_catalog_db()` in the same file. Data database setup: `create_data_db()` creates `idempotency_tokens` with a 10-minute TTL index. Collection naming convention: `crates/storage-mongodb/src/data/mod.rs` — `data_collection_name()` and `index_collection_name()`. ### Document structure for DynamoDB items @@ -83,21 +83,23 @@ Each DynamoDB item is stored as a MongoDB document with the following structure: } ``` -The `_id` field enables O(1) point lookups. The `pk` field is indexed separately to support partition scans (Query operations). Sort keys are stored in typed fields (`sk_s`, `sk_n`, `sk_b`) so MongoDB can apply native range comparisons with correct ordering — notably, numeric sort keys use MongoDB's `Decimal128` type rather than strings to ensure correct numeric ordering. The full item is stored in `item_data` using DynamoDB's own type-tagged format (`{"S": "hello"}`, `{"N": "42"}`, etc.), preserving all type information without lossy conversion. +The `_id` field enables O(1) point lookups. The `pk` field is indexed separately to support partition scans (Query operations). Sort keys are stored in typed fields (`sk_s`, `sk_n`, `sk_b`) so MongoDB can apply native range comparisons with correct ordering — notably, numeric sort keys use MongoDB's `Decimal128` type rather than strings to ensure correct numeric ordering. Values that exceed Decimal128's 34-significant-digit precision are rejected at write and query time with a ValidationException, rather than downcast; DynamoDB itself supports up to 38 significant digits. This is documented as a backend-specific behavioral difference (see `docs/differences-from-dynamodb.md`). The full item is stored in `item_data` using DynamoDB's own type-tagged format (`{"S": "hello"}`, `{"N": "42"}`, etc.), preserving type information for non-key attributes. -String sort key collections are created with `{ locale: "simple", strength: 3 }` collation, ensuring byte-for-byte ordering that matches DynamoDB's behavior rather than locale-aware Unicode ordering. +String sort key collections are created with `{ locale: "simple" }` collation, ensuring byte-for-byte ordering that matches DynamoDB's behavior rather than locale-aware Unicode ordering. Document conversion functions: `crates/storage-mongodb/src/data/mod.rs` — `item_to_document()` (DynamoDB Item → BSON document) and `document_to_item()` (BSON document → DynamoDB Item). Sort key type handling including `Decimal128`: same file, `item_to_document()` `ScalarAttributeType::N` branch. Collation: `crates/storage-mongodb/src/table_engine.rs` — `CreateTable` implementation, index creation with `Collation` options. -### Condition expression pushdown +### Condition expression evaluation -Condition expressions (`ConditionExpression` on PutItem, DeleteItem, UpdateItem) are compiled into MongoDB filter documents and executed server-side as part of atomic `findOneAndReplace` and `findOneAndDelete` operations. This means a conditional write is a single round-trip to MongoDB — no separate fetch, no application-level check, no race window between the check and the write. +Condition expressions (`ConditionExpression` on PutItem, DeleteItem, UpdateItem) are evaluated inside a MongoDB client session that also wraps the write. Within the session, the backend reads the current item, evaluates the condition in Rust against the loaded item, and issues the write — the read and write share the session's transactional atomicity, so a concurrent writer's changes are either visible to the condition or cause the write to observe its version guard (see Write conflict handling). This delivers DynamoDB's atomicity contract for conditional writes and matches the `ReturnValuesOnConditionCheckFailure = ALL_OLD` semantics naturally (the read result is available for the response). -The compiler handles: `attribute_exists`, `attribute_not_exists`, `attribute_type`, `begins_with`, `contains`, `size`, `BETWEEN`, `IN`, `=`, `<>`, `<`, `<=`, `>`, `>=`, `AND`, `OR`, `NOT`. Because items are stored with DynamoDB type tags, compiled paths include the type suffix: `item_data.fieldName.S` for strings, `.N` for numbers. +The condition compiler exists as scaffolding for a follow-up optimization: it translates DynamoDB expressions into MongoDB filter documents that could be pushed into `findOneAndReplace` / `findOneAndDelete` for a single-round-trip conditional write. The compiler handles: `attribute_exists`, `attribute_not_exists`, `attribute_type`, `begins_with`, `contains`, `BETWEEN`, `IN`, `=`, `<>`, `<`, `<=`, `>`, `>=`, `AND`, `OR`, `NOT`. Because items are stored with DynamoDB type tags, compiled paths include the type suffix: `item_data.fieldName.S` for strings, `.N` for numbers. The compiler is not on the load-bearing correctness path today; the session-scoped read-then-write is. +**Alternative considered — filter pushdown as the primary path.** Compiling filters into `findOneAndReplace` / `findOneAndDelete` would reduce a conditional write to a single round-trip. It is planned as a follow-up optimization once the compiler is exercised through the integration test suite. The current design was chosen because it gives us the loaded item for `ReturnValuesOnConditionCheckFailure` responses without a follow-up read, and because the compiler's behavior on edge-case expressions (multi-valued `size()`, `NOT` on missing paths, mixed-type set membership) is easier to validate incrementally under the session-scoped path. -Condition compiler: `crates/storage-mongodb/src/condition.rs` — `condition_to_filter()` is the entry point. Each DynamoDB function and operator has a corresponding compilation case. Unit tests in the same file demonstrate each compiled output. Usage in write operations: `crates/storage-mongodb/src/data_engine.rs` — `put_item_impl()`, `delete_item_impl()`, `update_item_impl()` each pass the compiled filter to MongoDB's `findOneAndReplace`/`findOneAndDelete`/`findOneAndUpdate`. + +Session-scoped condition path: `crates/storage-mongodb/src/data_engine.rs` — `put_item_impl()`, `delete_item_impl()`, `update_item_impl()`, and the four `OwnedTransactWriteOp` arms in `execute_write_op_in_session()` each do read → `expression::evaluate_condition` → write, all with the session bound to each driver call. Condition compiler (scaffolding for the future pushdown path): `crates/storage-mongodb/src/condition.rs` — `condition_to_filter()` is the entry point. Unit tests in the same file demonstrate each compiled output. ### Query and Scan @@ -125,9 +127,9 @@ Transaction implementation: `crates/storage-mongodb/src/data_engine.rs` — `tra ### Write conflict handling -**UpdateItem** uses optimistic concurrency. A `_v` version counter is stored on each document. The write path reads the current `_v`, applies the update expression in memory, sets `_v = current_version + 1`, then executes `replaceOne` filtered on both the primary key and the expected `_v`. If `matched_count == 0`, a concurrent writer incremented the version first. The operation retries with jittered exponential backoff (100 µs base, up to 50 attempts). Exhausted retries propagate the error to the caller. +**UpdateItem** uses optimistic concurrency. A `_v` version counter is stored on each document. The write path reads the current `_v`, applies the update expression in memory, sets `_v = current_version + 1`, then executes `replaceOne` filtered on both the primary key and the expected `_v`. If `matched_count == 0`, a concurrent writer incremented the version first. The operation retries with jittered exponential backoff (50 µs base, up to 50 attempts). Exhausted retries propagate the error to the caller. -**PutItem and DeleteItem** use condition pushdown (see Condition expression pushdown). When `ReturnValuesOnConditionCheckFailure` is requested and the condition fails, a follow-up `find_one` fetches the existing item for the response. This matches DynamoDB's own best-effort semantics for the returned item on condition failure. +**PutItem and DeleteItem** run their conditional read, condition evaluation, and write within a single MongoDB client session (see Condition expression evaluation). The item loaded to evaluate the condition is reused directly for the `ReturnValuesOnConditionCheckFailure = ALL_OLD` response — no follow-up read is issued. When no condition is present, the write is a straight `findOneAndReplace` / `findOneAndDelete`. **TransactWriteItems** runs all operations inside a single MongoDB ACID transaction with snapshot read concern and majority write concern. Transaction failures are not retried — the error propagates as `TransactionCanceled`. @@ -167,10 +169,12 @@ Encryption key generation and storage: `crates/storage-mongodb/src/bootstrapper. ### Backup -`CreateBackup` uses MongoDB's server-side `$out` aggregation stage to copy a table's collection to a backup collection (`_backup_{backup_id}_{table_id}`) without transferring data through the application. `RestoreTableFromBackup` reads the backup collection and reconstructs the table. `DeleteBackup` drops the backup collection. +`CreateBackup` iterates the source table's collection and inserts each item into a shared `backup_items` collection in the `extenddb_catalog` database, tagged with the `backup_arn`. Backup metadata (arn, table, timestamps, status) is stored in `extenddb_catalog.backups`. `RestoreTableFromBackup` reads `backup_items` filtered by `backup_arn` and reconstructs the table. `DeleteBackup` removes the entries for that arn from the shared collection and marks the metadata row deleted. + +**Alternative considered — server-side `$out` aggregation.** MongoDB's `$out` stage would copy a collection server-side without transferring data through the application. It was not adopted here because backup metadata (retention policies, tags, cross-collection references, account-scoped ARN lookups) does not compose with `$out`'s single-collection-target model, and because a shared `backup_items` collection avoids proliferating per-backup collection names in the WiredTiger file namespace at typical scale. A future revision could switch to `$out` for the copy phase while keeping the shared metadata schema. -Backup implementation: `crates/storage-mongodb/src/backup_engine.rs`. +Backup implementation: `crates/storage-mongodb/src/backup_engine.rs`. `backup_items` collection layout and `backup_arn` indexing are documented in the module header. ### Operational requirements @@ -212,6 +216,7 @@ Configuration struct: `crates/storage-mongodb/src/config.rs`. Sample configurati | `crates/storage-mongodb/` | New crate — full backend implementation | | `crates/bin/Cargo.toml` | Added `mongodb` optional feature flag | | `crates/bin/src/main.rs` | Added `#[cfg(feature = "mongodb")] extern crate` | +| `crates/bin/src/cmd_serve.rs` | Generalized the supported-backend gate from a hard-coded `"postgres"` check to a compile-time-conditional list built from the enabled feature flags (accepts `"mongodb"` when the feature is on) | | `Cargo.toml` (workspace) | Added crate to members, added `mongodb`, `bson`, `dashmap` workspace dependencies | No changes to `crates/engine/`, `crates/server/`, `crates/storage/` (trait definitions), `crates/auth/`, or `crates/core/`. @@ -223,22 +228,22 @@ Full diff: `mongodb-forks/extenddb` branch `extenddb-on-mongo` compared to `main | Decision | Choice | Rationale | |---|---|---| -| Conditional writes (PutItem, DeleteItem) | Filter pushdown into `findOneAndReplace` / `findOneAndDelete` | Single-document atomicity; no transaction overhead on the hot path | -| UpdateItem write conflict | Optimistic concurrency with `_v` version field + jittered backoff | Avoids transactions for single-item updates while preventing lost updates | -| GSI updates | Synchronous inline with `DashMap` cache | No Change Stream recovery complexity; GSI reads are strongly consistent | -| DynamoDB Streams | Inline writes to `stream_records` collection | Behavioral parity with PostgreSQL backend; explicit control over sequence numbers, shard assignment, and retention | +| Conditional writes (PutItem, DeleteItem) | Read + evaluate + write within one MongoDB client session | Atomicity for the DynamoDB contract; loaded item is reused for `ReturnValuesOnConditionCheckFailure = ALL_OLD` without a follow-up read. Filter pushdown is planned as an optimization (see Condition expression evaluation). | +| UpdateItem write conflict | Optimistic concurrency with `_v` version field + jittered backoff | Avoids multi-document transactions for single-item updates while preventing lost updates | +| GSI updates | Synchronous inline within the base write's session, with `DashMap` cache short-circuit for tables with no GSIs | No Change Stream recovery complexity; GSI reads are strongly consistent | +| DynamoDB Streams | Inline writes to `stream_records` collection within the base write's session | Behavioral parity with PostgreSQL backend; explicit control over sequence numbers, shard assignment, and retention | | Stream shards | 4 per table, CRC32 hash assignment | Predictable consumer parallelism; no catalog lookup at shard assignment time | -| Sort key numbers | Native BSON `Decimal128` | Correct ordering by value; no string-encoding tricks | -| Backups | Server-side `$out` aggregation stage | No client-side data transfer; no document size limitations | +| Sort key numbers | Native BSON `Decimal128` | Correct ordering by value; no string-encoding tricks. Values exceeding Decimal128 precision (34 digits) are rejected — see `docs/differences-from-dynamodb.md`. | +| Backups | Per-item inserts to shared `backup_items` collection keyed by `backup_arn` | Composes cleanly with backup metadata (tags, retention, ARN-scoped restore/delete); avoids collection-name proliferation. Server-side `$out` documented as a future optimization. | | Parallel scan | Application-side `crc32(pk) % segments` filter | Avoids per-document write overhead of a pre-bucketed segment field | ### Performance characteristics -**Single-item writes (hot path).** Transaction-free. A PutItem with a condition expression is a single `findOneAndReplace` with a filter — one network round-trip, one WiredTiger document write. No locking, no multi-phase commit. +**Single-item writes.** A conditional PutItem, DeleteItem, or UpdateItem is executed within a MongoDB client session that covers the condition read, the write, and any dependent writes (stream record insert, GSI collection updates). This adds one session start/commit pair over a raw driver call — ~sub-millisecond on a local replica set. The session wrap is what gives DynamoDB's atomicity contract on conditional writes; it is not overhead in the DynamoDB-compatibility sense, it is the compatibility. A sessionless fast path for the narrow case of tables with no streams and no GSIs is planned as a follow-up optimization. -**GSI write overhead.** For tables with no GSIs, the `gsi_cache` short-circuits to zero overhead — no catalog query, no additional I/O. For tables with GSIs, one catalog query fetches index definitions (cached for subsequent writes on the same table), plus one upsert or delete per index collection per write. +**GSI write overhead.** For tables with no GSIs, the `gsi_cache` short-circuits to zero overhead — no catalog query, no additional I/O. For tables with GSIs, one catalog query fetches index definitions (cached for subsequent writes on the same table), plus one upsert or delete per index collection per write, all within the same session as the base write. -**Stream write overhead.** When streams are enabled, each write adds one atomic `findOneAndUpdate` counter increment and one document insert into `stream_records`. +**Stream write overhead.** When streams are enabled, each write adds one atomic `findOneAndUpdate` counter increment and one document insert into `stream_records`, both within the base write's session. **Query and Scan.** Direct index lookups on `{ pk, sk_* }`. Performance characteristics match any indexed MongoDB query. Parallel scans scan the full collection once per segment (see Query and Scan). From 61488e369a69f20eb257137e5be59c5bb26316f6 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 18:08:38 -0700 Subject: [PATCH 48/83] docs: rewrite RFC-206 and design doc to describe current backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites both documents from scratch to describe the MongoDB backend as it exists today, dropping earlier-planned architecture that never made it into the shipping code (single-round-trip filter pushdown as the primary path, sk_b as BSON Binary, index docs without base-key disambiguation, synchronous inline GSI create, etc.). RFC-206 (`docs/rfcs/0000-mongodb-backend.md`): - Document structure section covers netstring `_id`, typed sort-key fields including hex-encoded binary, `_v` OCC counter, and the index-doc base_pk/base_sk_? disambiguation with 4-tuple _id. - Condition-evaluation section describes the session-scoped read + evaluate + write flow as the correctness path, with the analyzer-gated pushdown as an opt-in fast path. - Query/Scan covers hex-based binary begins_with, sort-key range preservation on pagination, and the compound index cursor. - GSI/LSI section covers the CREATING → ACTIVE state machine and the async gsi_backfill_worker. - Transactions section covers WriteConflict retry with TransactionConflict-on-exhaustion. - Streams section covers per-shard session-scoped sequence counters, shard_id embedding table_id, 24 h TTL retention, and idempotent stream-enable. - Operational requirements adds primary-only readPreference enforcement. Design doc (`docs/design/13-storage-mongodb.md`): - Catalog schemas (§3) now show TableClass / SSE / OnDemandThroughput persistence, indexes with `backfill_cursor` + `CREATING`. - Data schemas (§4) document netstring `_id`, `_v`, index docs with base-key fields and per-index compound query index, stream_records (TTL + compound query index), stream_shards (unique on shard_id derived from table_id), counters keyed per-shard, idempotency_tokens (540 s TTL + unique compound on account_id+token), and `_backup_{backup_id}` collections. - §5 rewritten around: session-scoped conditional writes, analyzer-gated pushdown, base-key disambiguation with compound cursor pagination, async GSI backfill, session-scoped per-shard sequence counters, WriteConflict retry, hex sort keys, and the next_string_prefix upper-bound. - §12 replaced with a feature-coverage inventory grouped by trait and background worker. - §14 adds primary readPreference enforcement. --- docs/design/13-storage-mongodb.md | 1247 ++++++++++++++++++----------- docs/rfcs/0000-mongodb-backend.md | 201 ++--- 2 files changed, 900 insertions(+), 548 deletions(-) diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index f3218db2..ab874fdd 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -3,31 +3,42 @@ ## 1. Overview The MongoDB backend (`extenddb-storage-mongodb`) implements the same trait surface as -`extenddb-storage-postgres`: the 6 engine traits (`TableEngine`, `DataEngine`, +`extenddb-storage-postgres`: the six engine traits (`TableEngine`, `DataEngine`, `MetadataEngine`, `StreamEngine`, `BackupEngine`, `WorkerStore`) and the catalog traits (`ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, `RateLimitStore`, `AuthorizationStore`). -**Driver:** `mongodb` (official Rust driver, async, supports multi-document ACID +**Driver:** `mongodb` (official Rust driver, async, multi-document ACID transactions on replica sets). -**Minimum MongoDB version:** 6.0 (for multi-document transactions and snapshot -reads). +**Minimum MongoDB version:** 6.0 (multi-document transactions, snapshot reads). + +**Read preference:** `primary` only. `MongoEngine::new` rejects connection strings +that request `secondary`, `secondaryPreferred`, `primaryPreferred`, or `nearest` — +DynamoDB's `ConsistentRead=true` contract requires linearizable reads, which only +Primary provides. Silently routing reads to replicas would return stale data with +no signal to the caller. ## 2. Database Layout -Two databases, mirroring the PostgreSQL backend's catalog/data separation: +Two databases, mirroring the PostgreSQL backend's catalog / data separation: | Database | Purpose | |----------------------|------------------------------------------------------| -| `extenddb_catalog` | Table metadata, IAM, settings, metrics | -| `extenddb_data` | Per-table item collections, idempotency tokens | +| `extenddb_catalog` | Table metadata, IAM, settings, metrics, backup metadata | +| `extenddb_data` | Per-table item collections, per-index collections, streams, idempotency tokens, backup snapshots | -DynamoDB Streams are implemented via inline stream record writes during data -operations. GSI updates are propagated synchronously inline during writes. +DynamoDB Streams are implemented via inline stream-record writes during data +operations. GSI updates are propagated synchronously inline for existing indexes +and asynchronously via a background worker during UpdateTable-driven GSI +creation. ## 3. Catalog Database Collections +Collections created by `run_catalog_migrations` in `bootstrapper.rs`. Two +additional collections (`iam_group_members`, `backup_items` — the latter is +no longer used) are auto-created on first insert and not in the migration list. + ### 3.1 `accounts` ```json { "_id": "", "account_name": "...", "created_at": ISODate } @@ -48,12 +59,15 @@ Unique index on `account_name`. "table_size_bytes": NumberLong, "item_count": NumberLong, "table_arn": "...", - "table_id": "...", + "table_id": "", "ttl_attribute": null, + "ttl_index_ready": false, "deletion_protection_enabled": false, "status_transition_at": null, "stream_label": null, - "ttl_index_ready": false + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null } ``` Unique index on `table_id`. Partial index on `status_transition_at` where not null. @@ -62,14 +76,17 @@ Unique index on `table_id`. Partial index on `status_transition_at` where not nu ```json { "_id": { "table_id": "...", "index_name": "..." }, - "index_id": "...", + "index_id": "", "index_type": "GSI|LSI", "key_schema": [...], "projection": { ... }, - "index_status": "ACTIVE", - "provisioned_throughput": { ... } + "index_status": "ACTIVE|CREATING", + "provisioned_throughput": { ... }, + "backfill_cursor": // present while index_status = CREATING } ``` +`backfill_cursor` is written by `gsi_backfill_worker` after every batch. The +field is unset when the index flips to `ACTIVE`. ### 3.4 `tags` ```json @@ -80,10 +97,12 @@ Unique index on `table_id`. Partial index on `status_transition_at` where not nu ```json { "_id": "", "value": "..." } ``` +Bootstrapped keys include `catalog_version`, `encryption_key` (base64-encoded +256-bit AES-GCM key), `data_database_name`, and `data_connection_string`. ### 3.6 `admin_users` ```json -{ "_id": "", "password_hash": "...", "created_at": ISODate } +{ "_id": "", "password_hash": "", "created_at": ISODate } ``` ### 3.7 `iam_users` @@ -101,7 +120,7 @@ Unique index on `user_arn`. ### 3.8 `access_keys` ```json { - "_id": "", + "access_key_id": "", "secret_key_encrypted": BinData, "account_id": "...", "user_name": "...", @@ -109,7 +128,9 @@ Unique index on `user_arn`. "created_at": ISODate } ``` -Index on `(account_id, user_name)`. +Index on `(account_id, user_name)`. `secret_key_encrypted` is AES-256-GCM +ciphertext of the secret access key using the `settings.encryption_key`; +`access_key_id` is bound as additional authenticated data. ### 3.9 `iam_groups` ```json @@ -139,7 +160,7 @@ Unique index on `role_arn`. ```json { "_id": "", - "access_key_id": "...", + "access_key_id": "ASIA...", "secret_key_encrypted": BinData, "account_id": "...", "role_name": "...", @@ -150,12 +171,15 @@ Unique index on `role_arn`. "created_at": ISODate } ``` -Unique index on `access_key_id`. TTL index on `expires_at`. +Unique index on `access_key_id`. TTL index on `expires_at` (`expireAfterSeconds: 0`). ### 3.12 `iam_policies` ```json { - "_id": { "account_id": "...", "principal_type": "...", "principal_name": "...", "policy_name": "..." }, + "account_id": "...", + "principal_type": "user|role|group", + "principal_name": "...", + "policy_name": "...", "policy_document": { ... }, "created_at": ISODate } @@ -164,7 +188,9 @@ Unique index on `access_key_id`. TTL index on `expires_at`. ### 3.13 `iam_permissions_boundaries` ```json { - "_id": { "account_id": "...", "principal_type": "...", "principal_name": "..." }, + "account_id": "...", + "principal_type": "user|role", + "principal_name": "...", "policy_document": { ... } } ``` @@ -179,49 +205,46 @@ Unique index on `access_key_id`. TTL index on `expires_at`. "max": -Infinity } ``` -Index on `bucket` for pruning. +Index on `_id.bucket` for pruning. ### 3.15 `login_attempts` ```json -{ - "principal": "...", - "attempted_at": ISODate, - "success": false, - "source_ip": "..." -} +{ "principal": "...", "attempted_at": ISODate, "success": false, "source_ip": "..." } ``` Compound index on `(principal, attempted_at)`. -Partial index on `(source_ip, attempted_at)` where source_ip exists. -### 3.16 `backups` (metadata only) +### 3.16 `backups` ```json { "_id": "", + "backup_id": "", "backup_name": "...", + "backup_status": "AVAILABLE|DELETED", + "backup_type": "USER", "table_id": "...", "table_name": "...", + "table_arn": "...", "account_id": "...", - "backup_status": "AVAILABLE", - "backup_type": "USER", "backup_size_bytes": NumberLong, "item_count": NumberLong, "key_schema": [...], "attribute_definitions": [...], "billing_mode": "PAY_PER_REQUEST", - "provisioned_throughput": null, - "stream_specification": null, - "backup_collection": "_backup_{backup_id}", - "created_at": ISODate + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "created_at": ISODate, + "table_creation_date_time": } ``` -Index on `(account_id, table_name)`. - -Backup item data is stored in a cloned collection (see Section 5.7). +Index on `(account_id, table_name)`. The physical backup collection lives in +`extenddb_data` as `_backup_{backup_id}` — see §4.5. ### 3.17 `continuous_backups` ```json { - "_id": { "account_id": "...", "table_name": "..." }, + "account_id": "...", + "table_name": "...", "pitr_enabled": false, "earliest_restorable": null, "latest_restorable": null @@ -237,318 +260,554 @@ Backup item data is stored in a cloned collection (see Section 5.7). ### 4.1 Per-Table Item Collections: `_ddb_{table_id}` -Each DynamoDB virtual table maps to a MongoDB collection. +Each DynamoDB virtual table maps to a MongoDB collection named +`_ddb_{table_id}` (table_id is a UUID assigned at CreateTable). The +collection-name derivation shields the physical layer from caller-visible +name changes and from characters that are unsafe as MongoDB collection +names. **Document structure:** ```json { - "_id": "#", + "_id": "", "pk": "...", "sk_s": "...", "sk_n": Decimal128, - "sk_b": BinData, + "sk_b": "", + "_v": NumberLong, "item_data": { ... } } ``` Fields: -- `_id` — deterministic compound key for upserts -- `pk` — partition key value (string-encoded) -- `sk_s` — sort key (string type), null if not applicable -- `sk_n` — sort key (numeric type, native BSON Decimal128), null if not applicable -- `sk_b` — sort key (binary type), null if not applicable -- `item_data` — full DynamoDB item serialized as BSON + +- `_id` — netstring-encoded composite key `:,:,`. + Netstring framing gives an unambiguous boundary between `pk` and `sk` + regardless of their contents. A naive `{pk}#{sk}` delimiter collides + on `pk="a#b",sk="c"` vs `pk="a",sk="b#c"`. PK-only tables use raw `pk` + text as `_id`. +- `pk` — partition key text (matches `composite_pk_to_text` from + `extenddb_storage::util`). +- `sk_s` / `sk_n` / `sk_b` — typed sort key, absent when the schema has + no sort key. See §5.2 for the type-specific encoding. +- `_v` — OCC version counter used by `UpdateItem`'s versioned filter + guard. Absent on freshly-inserted rows (treated as 0); bumped by every + update, including the native fast path. +- `item_data` — full DynamoDB item serialized as BSON via the + `AttributeValue` JSON representation. Non-key attribute values retain + their DynamoDB type tags (`{"S": "hello"}`, `{"N": "42"}`, ...). **Indexes:** -- `{ pk: 1, sk_s: 1 }` or `{ pk: 1, sk_n: 1 }` or `{ pk: 1, sk_b: 1 }` depending - on sort key type, or just `{ pk: 1 }` for PK-only tables -**Sort key ordering:** -- **String (`sk_s`):** Collection uses `collation: { locale: "simple" }` for - byte-order sorting (matches DynamoDB's UTF-8 byte-order comparison). -- **Numeric (`sk_n`):** Native BSON Decimal128. MongoDB sorts numbers by value — - no encoding tricks needed. DynamoDB supports 38 significant digits; Decimal128 - provides 34, which covers all practical use cases. +- `{ pk: 1 }` (PK-only tables) or `{ pk: 1, sk_?: 1 }`, unique. + String sort-key indexes use `{ locale: "simple" }` collation for + byte-order comparisons. ### 4.2 Per-Index Collections: `_ddb_{index_id}` -Same structure as item collections. GSI/LSI data is projected and stored here. -Written synchronously inline during data operations (PutItem, UpdateItem, DeleteItem). +Each GSI/LSI has its own collection. The document schema extends §4.1 +with the base table's key attributes as first-class fields: -### 4.3 `idempotency_tokens` ```json { - "_id": "", - "fingerprint": "...", - "created_at": ISODate + "_id": "", + "pk": "...", // index partition key + "sk_s|sk_n|sk_b": ..., // index sort key + "base_pk": "...", // base-table partition key text + "base_sk_s|_n|_b": ..., // base-table sort key, typed + "item_data": { ... } // projected item per the index's Projection } ``` -TTL index on `created_at` (10 minutes) — MongoDB automatically cleans up expired -tokens. -## 5. Key Design Decisions - -### 5.1 Transactions +Two reasons for the extra base-key material: + +1. **Unique identity per base item.** GSI keys are non-unique — multiple + base items can share `(index_pk, index_sk)`. If `_id` were only + derived from index keys, two base items with the same GSI-key values + would upsert to the same document; one would silently overwrite the + other. Including the base keys in `_id` and the entry-delete filter + (`index_entry_filter`) makes each entry addressable independently. +2. **Compound pagination cursor.** Index queries and scans sort and + paginate on `(index_sk?, base_pk, base_sk?)` — see §5.3. + Having the base-key components as fields (not buried under + `item_data..S`) lets the sort and cursor filters use per-field + indexes. + +Every index collection carries a compound index on +`(pk, sk_?, base_pk, base_sk_?)` created by `create_index_data_collection`. +Simple collation is used whenever the tuple contains a string component. + +### 4.3 `stream_records` +```json +{ + "sequence_number": "<21-digit zero-padded>", + "shard_id": "shardId-{table_id}-{index:012}", + "table_id": "...", + "event_name": "INSERT|MODIFY|REMOVE", + "record_data": { ... full StreamRecord as BSON ... }, + "created_at": ISODate +} +``` -MongoDB multi-document ACID transactions are used **only** for: +- TTL index on `created_at` with `expireAfterSeconds = 24 * 3600` — + primary retention enforcement. +- Compound index on `(shard_id, sequence_number)` — powers `GetRecords` + (`shard_id` equality + `sequence_number > cursor` range with ascending + sort). Without it, every consumer poll runs a full-collection scan. -1. **TransactWriteItems** — all operations in a single transaction. -2. **TransactGetItems** — snapshot read using a session with `snapshot` read concern. +### 4.4 `stream_shards` +```json +{ + "shard_id": "shardId-{table_id}-{index:012}", + "table_id": "...", + "starting_sequence_number": "<21-digit>", + "ending_sequence_number": null, + "created_at": ISODate +} +``` -Everything else is transaction-free: -- Single-item conditional writes use filter pushdown (Section 5.2) -- GSI updates are done synchronously inline during the write operation (Section 5.4) -- Stream records are written inline during data operations (Section 5.5) +Unique index on `shard_id`. Four shards per stream-enabled table. -### 5.2 Condition Evaluation — Filter Pushdown +`shard_id` embeds `table_id` (a UUID) rather than `table_name`. Table +names are only unique per-account; a name-derived scheme would let one +account's `GetRecords(shard_id)` observe another account's records on +same-named tables. `table_id` resets on `DeleteTable + CreateTable`, so +recreated tables get fresh shard_ids and leftover records from the +deleted table cannot resurface. -DynamoDB condition expressions are compiled to MongoDB query filters and pushed into -the write operation itself. This exploits MongoDB's single-document atomicity: a -`findOneAndReplace`/`findOneAndUpdate`/`findOneAndDelete` with a filter is atomic -without an explicit transaction. +### 4.5 `counters` +```json +{ "_id": "stream_seq:", "value": NumberLong } +``` -**Flow:** -1. Compile `ConditionExpression` AST → MongoDB filter document -2. Combine with primary key filter: `{ pk: X, sk_s: Y, ...condition_filter... }` -3. Execute as `findOneAndReplace` (PutItem), `findOneAndUpdate` (UpdateItem), or - `findOneAndDelete` (DeleteItem) -4. If result is `None` and the item exists → condition failed → - `StorageError::ConditionFailed` +One document per shard. `$inc` on `value` inside a session yields the +next sequence number. Per-shard counters (not a single global counter) +preserve DynamoDB Streams' contract that sequence numbers are strictly +monotonic within a shard and independent across shards. -**Condition-to-filter translation:** +### 4.6 `idempotency_tokens` +```json +{ + "account_id": "...", + "token": "...", + "fingerprint": "...", + "created_at": ISODate +} +``` -| DynamoDB condition | MongoDB filter | -|---|---| -| `attribute_exists(foo)` | `{ "item_data.foo": { $exists: true } }` | -| `attribute_not_exists(foo)` | `{ "item_data.foo": { $exists: false } }` | -| `foo = :val` | `{ "item_data.foo.S": val }` (typed) | -| `foo <> :val` | `{ "item_data.foo.S": { $ne: val } }` | -| `foo < :val` | `{ "item_data.foo.N": { $lt: val } }` | -| `foo > :val` | `{ "item_data.foo.N": { $gt: val } }` | -| `begins_with(foo, :p)` | `{ "item_data.foo.S": { $regex: "^

" } }` | -| `contains(foo, :v)` | `{ "item_data.foo.S": { $regex: "" } }` | -| `size(foo) = :n` | `{ $expr: { $eq: [{ $size: "$item_data.foo.L" }, n] } }` | -| `cond1 AND cond2` | `{ $and: [filter1, filter2] }` | -| `cond1 OR cond2` | `{ $or: [filter1, filter2] }` | -| `NOT cond` | `{ $nor: [filter] }` | +- TTL index on `created_at` with `expireAfterSeconds = 540`. +- **Unique compound index on `(account_id, token)`.** -**Implementation:** `condition_to_filter(expr: &Expr, maps: &ExpressionMaps) -> bson::Document` -walks the expression AST and emits a MongoDB filter. +The TTL is 540s (9 min), tighter than DDB's 10-min window. MongoDB's TTL +monitor runs on a ~60s cadence, so worst-case retention with TTL = 540s +is ≤10 min. The data-plane read path (`transact_write_items_impl`) also +filters existing rows by `created_at` age < 600 000 ms so retention is +correct regardless of TTL-monitor timing. -**Common patterns (all transaction-free):** +The unique index closes a race window: two concurrent `TransactWriteItems` +calls with the same token both take snapshot reads that miss the other's +uncommitted insert; without the constraint, both would commit and the +operation would execute twice. With it, the second inserter fails +`E11000` and the write path resolves the winner by re-reading (still +subject to the age filter — if the winner has just expired, the retry +does a fresh insert). -| DynamoDB pattern | MongoDB operation | -|---|---| -| PutItem + `attribute_not_exists(pk)` | `updateOne({ pk, sk, "item_data.pk": {$exists: false} }, $setOnInsert, upsert)` | -| UpdateItem + `version = :v` | `findOneAndUpdate({ pk, sk, "item_data.version.N": v }, $set)` | -| DeleteItem + `status = :val` | `findOneAndDelete({ pk, sk, "item_data.status.S": val })` | -| PutItem (unconditional) | `replaceOne({ pk, sk }, doc, upsert: true)` | +### 4.7 `_backup_{backup_id}` -**Returning the old item:** +One collection per user-created backup. Populated by a server-side +`[{ $out: "_backup_{backup_id}" }]` aggregation pipeline on the source +data collection, so items are copied server-side without transferring +through the driver. Restored the same way, in reverse. `DeleteBackup` +drops the collection. -`findOneAndReplace`/`findOneAndDelete` atomically returns the pre-modification -document when `return_old = true`. No transaction needed. +## 5. Key Design Decisions -For `ConditionFailed` with `ReturnValuesOnConditionCheckFailure`, a follow-up -`find_one` fetches the existing item. This is acceptable — DynamoDB has the same -best-effort semantics for the returned item. +### 5.1 Session-scoped conditional writes + +`PutItem`, `DeleteItem`, and `UpdateItem` — when they carry a +`ConditionExpression`, a `StreamCapture`, or write to a table with GSIs — +run inside a MongoDB client session bound to a multi-document transaction +with snapshot read concern and majority write concern. Within the session: + +1. `find_one` the current document. +2. Evaluate the DynamoDB condition in Rust + (`extenddb_core::expression::evaluate_condition`) against the loaded + item. +3. Write (`find_one_and_replace` / `delete_one` / versioned `replace_one`). +4. Synchronize GSIs (`sync_indexes_in_session`). +5. Emit any stream record (`write_stream_inline_in_session`), including + per-shard sequence-number `$inc` — also in the same session. +6. Commit. + +All five happen on the same session, so a concurrent conflicting writer +manifests as a WriteConflict at commit — which the caller retries — not +as a stale-read anomaly. The pre-image loaded in step 1 is reused for +`ReturnValuesOnConditionCheckFailure = ALL_OLD` and for `OldImage` on any +attached stream capture; no follow-up read is needed. + +Update-as-insert (the pre-image was `None`) emits an `INSERT` stream +event with no `OldImage`, not a `MODIFY` with a fabricated key-only stub. + +`UpdateItem` also always fetches the pre-image regardless of the caller's +`ReturnValues` setting — the pre-image is required to compute correct +GSI deltas when the update changes or removes an indexed attribute, and +skipping it leaves stale entries in index collections forever. + +**Native fast path.** For unconditional updates on tables with no streams +and no GSIs (fresh cache says `Some(false)`), the backend collapses the +transaction to a single `find_one_and_update` outside any session using +compiled MongoDB atomic operators (`$set` / `$unset` / plus +`$inc: {_v: 1}`). The `_v` bump is unconditional on this path: without +it, a concurrent session-scoped update running against a stale snapshot +could pass its versioned filter and lost-update over the fast-path +write. + +Implementation: `data_engine.rs::put_item_impl`, `delete_item_impl`, +`update_item_impl`, and `execute_transact_write_op_in_session` for the +TWI arms. + +### 5.2 Filter-pushdown fast path (analyzer-gated) + +An optional pushdown fast path skips the session for conditional writes +that a static analyzer certifies as safe. The path is gated on: + +- `condition` is present. +- `stream` is `None`. +- `gsi_cache_get_fresh(table_id) == Some(false)` (i.e. the cache is + fresh AND says the table has no GSIs). +- `pushdown::is_pushable(cond, maps) == Pushable::Yes`. + +Under those guards, single-document `find_one_and_replace` / +`find_one_and_delete` provides atomicity — no session, no GSI sync, no +stream write. The compiled filter is merged with the primary-key filter +under `$and`. + +The **compiler** (`condition.rs`) is intentionally broader than +production usage: it translates +`attribute_exists`, `attribute_not_exists`, `attribute_type`, +`begins_with`, `contains`, `BETWEEN`, `IN`, `=`, `<>`, `<`, `<=`, `>`, +`>=`, `AND`, `OR`, `NOT`, and `size` into BSON filters. Some of those +translations are correct only for certain operand types. + +The **analyzer** (`pushdown.rs::is_pushable`) is the load-bearing +correctness boundary. It certifies a whole-condition subset that is +provably in agreement with `evaluate_condition`: + +- Existence functions (`attribute_exists`, `attribute_not_exists`) — always + pushable. +- `attribute_type(path, :t)` — pushable when `:t` resolves to a placeholder + whose value is one of the 10 valid DDB type tags + (`S`, `N`, `B`, `BOOL`, `NULL`, `L`, `M`, `SS`, `NS`, `BS`). Without + the whitelist a malicious `:t` could produce a `$`-prefixed pseudo-field. +- `begins_with(path, :S)` — string-only. +- `contains(path, :S)` — string-only. +- `path :S` for any comparator — string operands are stored + verbatim, lex order matches wire order. +- `path = :B` / `path <> :B` — binary equality only. Ordering + comparators on binary are refused because the compiler stores B as + base64 strings inside `item_data`, and base64 lex order diverges from + bytewise order across mismatched lengths. +- `path = / <> :BOOL` and `path = / <> :NULL` — value-only. +- `AND` / `OR` — pushable iff both children are pushable (all-or-nothing; + cherry-picking would confuse composition semantics). +- `NOT attribute_exists(path)` / `NOT attribute_not_exists(path)` — the + only pushable `NOT` forms. Anywhere else, MongoDB's `$nor` semantics + on missing paths diverge from DDB's three-valued logic. + +Not pushable: any operand of type `N` (numbers stored as strings; `"10" +> "9"` is false lex-wise), `size` (MongoDB has no UTF-16 code-unit +count), `BETWEEN` / `IN` (pending proptest coverage; the compiler emits +them, the analyzer refuses them), and `NOT` around anything else. + +Property tests (`tests/pushdown_parity.rs`) generate random items and +expressions, compile the filter, and check that a pure-Rust BSON +interpreter and `evaluate_condition` agree on match/no-match — the +regression harness that lets the analyzer's certification be extended +safely. ### 5.3 Query and Scan -**Query:** Translates `KeyCondition` to a MongoDB `find()` filter: -- Partition key equality: `{ pk: "" }` -- Sort key conditions: - - `=` → `{ sk_s: value }` - - `<` → `{ sk_s: { $lt: value } }` - - `begins_with` → `{ sk_s: { $gte: prefix, $lt: prefix_upper } }` - - `BETWEEN` → `{ sk_s: { $gte: low, $lte: high } }` - -Sort direction: `.sort({ sk_s: 1 })` for forward, `.sort({ sk_s: -1 })` for reverse. - -Pagination: `exclusive_start_key` translates to an additional `$gt`/`$lt` filter on -the sort key (or partition key for scans). - -**Scan:** Full collection scan with `.find({})`, paginated via sort-key-based cursor. - -**Parallel scan:** Segments are handled by filtering in application -(`crc32(pk) % total_segments == segment`). Each segment scans the full collection. -This is a known tradeoff — the only way to avoid redundant scans is a pre-bucketed -field on every document, which adds write-path overhead for a feature that is rarely -used in practice. - -### 5.4 GSI Propagation (Synchronous Inline) - -GSI updates are performed synchronously inline during each write operation. There is -no background worker, no Change Stream consumer, and no resume token tracking for GSI -propagation. - -**How it works:** - -On each write (PutItem, UpdateItem, DeleteItem), after writing to the base table -collection, the `sync_indexes` method: - -1. Checks the in-memory `gsi_cache` (`DashMap`) keyed by `table_id`. - If the cache entry is `false`, skip the catalog query entirely (fast path for - tables with no GSIs). -2. If the cache misses or is `true`, query the `indexes` collection in the catalog - database for all indexes belonging to this `table_id`. -3. For each GSI found: - - If an old item exists and has the index keys: delete the old entry from - `_ddb_{index_id}` - - If a new item exists and has the index keys: project the relevant attributes - (respecting the GSI's `Projection` setting) and upsert into `_ddb_{index_id}` -4. Update the cache: `gsi_cache.insert(table_id, found_any)`. - -**Cache invalidation:** -- On table delete (`delete_table`): `gsi_cache.remove(table_id)` -- On GSI create (`update_table`): `gsi_cache.insert(table_id, true)` -- On GSI delete (`update_table`): `gsi_cache.remove(table_id)` (will be re-populated - on next write) - -**Consistency model:** -- GSI reads are strongly consistent (index is updated before write returns to client) -- This is stricter than DynamoDB's eventual consistency model for GSIs, which is - acceptable (stronger guarantees never break application code) - -**Rationale:** Synchronous inline propagation avoids the complexity of Change Stream -recovery, resume token management, and eventual consistency bugs. The overhead is one -catalog query per write for tables with GSIs (cached to zero for tables without GSIs). - -### 5.5 DynamoDB Streams (Inline Record Storage) - -DynamoDB Streams are implemented by writing stream records inline during data -operations, using the same storage model as the PostgreSQL backend. Stream records -are stored in MongoDB collections (`stream_records` and `stream_shards` in the data -database) with explicit sequence numbers and shard assignment. This approach provides -behavioral parity with the PostgreSQL backend rather than relying on MongoDB Change -Streams. - -**Data model:** - -- `stream_shards` — one document per shard (4 shards per stream-enabled table), - keyed by `shard_id` + `table_id` -- `stream_records` — one document per event, containing `sequence_number`, `shard_id`, - `table_id`, `event_name`, `record_data` (full `StreamRecord` serialized as BSON), - and `created_at` - -**Write path:** - -When `StreamCapture` is provided to a data operation (PutItem, UpdateItem, DeleteItem), -the `write_stream_inline` helper: - -1. Determines the event type (INSERT/MODIFY/REMOVE) from old/new item presence -2. Builds key images and old/new images based on `StreamViewType` -3. Assigns a shard using `crc32(partition_key) % shard_count` -4. Obtains a sequence number via atomic `findOneAndUpdate` on a counter document -5. Writes the stream record to the `stream_records` collection - -**Shard assignment:** `crc32(pk) % SHARDS_PER_STREAM` (currently 4 shards per table). - -**Sequence numbers:** Global monotonic counter stored in `counters` collection, using -`findOneAndUpdate` with `$inc` for atomic increment. Format: zero-padded 21 digits. - -**`StreamEngine` trait mapping:** - -| Trait method | Implementation | -|----------------------------------|------------------------------------------------------------------| -| `write_stream_record` | Insert record document into `stream_records` collection | -| `get_stream_records` | Query `stream_records` by `shard_id`, ordered by `sequence_number` | -| `describe_stream` | Query `tables` + `stream_shards`, return shard list | -| `list_streams` | Query `tables` where `stream_label` is not null | -| `cleanup_expired_stream_records` | Delete records older than retention cutoff | -| `assign_shard` | `crc32(pk) % shard_count` over shards for the table | -| `next_sequence_number` | Atomic `$inc` on counter document in `counters` collection | -| `validate_shard` | Check table+stream exist and shard_id belongs to the stream | -| `latest_sequence_number` | Query last record in shard by descending `sequence_number` | - -**Retention:** `cleanup_expired_stream_records` deletes records with `created_at` -older than the configured retention period. - -### 5.6 TTL Handling - -MongoDB's built-in TTL indexes handle automatic cleanup for: -- `idempotency_tokens` — expire after 10 minutes -- `iam_sessions` — expire at `expires_at` - -For DynamoDB-level TTL (user-configured `TimeToLive`), the application-level TTL -worker is still needed because TTL deletion must emit stream records with a specific -`UserIdentity`. When TTL is enabled on a table, a sparse index is created on the TTL -attribute path for efficient expired-item lookup: +**Query key mapping.** +- Partition-key equality: `{ pk: }`. +- Sort-key conditions map to typed filters on `sk_s` / `sk_n` / `sk_b`. +- `BETWEEN` with `low > high` is rejected at the storage boundary with a + `ValidationException`. +- `begins_with(:S)` emits `{ sk_s: { $gte: prefix, $lt: next_string_prefix(prefix) } }`. + `next_string_prefix` computes the exclusive upper bound by incrementing + the rightmost non-`char::MAX` code point (skipping the surrogate gap + via `char::from_u32` retry); if the entire prefix is `char::MAX` it + returns `None` and the caller emits only the `$gte` bound. The + earlier `prefix + char::MAX` scheme excluded stored strings equal to + `s + char::MAX` (or extending past it) that DDB matches. +- `begins_with(:B)` emits the same range shape on the hex-encoded sort + key: `{ sk_b: { $gte: hex(prefix), $lt: hex(increment_bytes(prefix)) } }`. + +**Pagination.** `ExclusiveStartKey` **merges** into the existing sort-key +predicate rather than replacing it. Base-table Query paginates on a +single `$gt` / `$lt` sort-key comparison. Naively inserting +`filter.insert(sk, {$gt: cursor})` drops the caller's original +`BETWEEN` / `begins_with` bound and returns items outside it on page +2+. The merge covers three shapes: + +- No existing sk predicate → insert cursor bound. +- Existing operator map (`{ $gte: X, $lt: Y }`) → merge the cursor bound + into the map. +- Existing equality (`sk = X`) → wrap both under `$and`. + +**Index Query and Scan cursors.** Index-key values are non-unique, so +pagination cannot rely on `(pk, sk)` alone — items with duplicate index +keys would form an unstable page boundary. Instead, index queries +paginate on the compound tuple `(index_sk?, base_pk, base_sk?)` +expressed as a lexicographic `$or`: -```rust -db.collection("_ddb_{table_id}") - .create_index(IndexModel::builder() - .keys(doc! { format!("item_data.{ttl_attribute}.N"): 1 }) - .options(IndexOptions::builder().sparse(true).build()) - .build()) ``` - -### 5.7 Backups - -`CreateBackup` clones the source collection server-side using `$out`: - -```rust -// CreateBackup — server-side collection clone -data_db.collection("_ddb_{table_id}") - .aggregate([doc! { "$out": "_backup_{backup_id}" }]) - .await?; - -// RestoreTableFromBackup — clone back to new table -data_db.collection("_backup_{backup_id}") - .aggregate([doc! { "$out": "_ddb_{new_table_id}" }]) - .await?; - -// DeleteBackup — drop the backup collection -data_db.collection("_backup_{backup_id}").drop().await?; +(a > A) OR (a == A AND b > B) OR (a == A AND b == B AND c > C) ``` -No document size limits, handles tables of any size, no client-side data transfer. - -### 5.8 Write Conflict Handling - -**UpdateItem (optimistic concurrency):** - -`UpdateItem` uses a read-modify-write pattern with a `_v` version field for conflict -detection: - -1. Read the existing document and note its `_v` (version) value (defaults to 0 if - absent) -2. Apply update expressions in memory to produce the new item -3. Set `_v = current_version + 1` on the new document -4. Execute `replaceOne` with a filter matching both the primary key AND the expected - `_v` value -5. If `matched_count == 0`, a concurrent writer incremented the version first — - retry with jittered exponential backoff (base 100us, up to 50 attempts) - -This avoids multi-document transactions for single-item updates while preventing -lost updates. - -**Conditional writes (PutItem, DeleteItem):** - -These use a find-then-write pattern. For PutItem, the condition is evaluated -client-side against the fetched document, then `findOneAndReplace` (or `insert_one` -for new items) is used. Duplicate key errors on insert are caught and mapped to -`ConditionFailed`. - -**Explicit transactions (`TransactWriteItems`):** - -All operations in a `TransactWriteItems` call execute within a single MongoDB -multi-document transaction with snapshot read concern and majority write concern. -If the transaction fails, it is not retried — the error propagates as -`StorageError::TransactionCanceled`. - -### 5.9 Account ID Validation - -Defense against MongoDB operator injection: -- Reject `$` (operator injection) -- Reject `.` (field path traversal) -- Reject null bytes -- Reject non-ASCII - -### 5.10 Catalog Version Check - -Read `catalog_version` from the `settings` collection and compare against the -compiled-in constant. Same pattern as PostgreSQL. +reversed to `$lt` for descending scans. Index Scan paginates on +`(pk, sk?, base_pk, base_sk?)` (index Scan lacks the partition-key +equality that Query has). Sort direction is applied to the whole +tuple so pagination is deterministic across items sharing an index-key +value. `LastEvaluatedKey` carries both the index-key and base-key +components so the next page's `ExclusiveStartKey` can rehydrate the +cursor. + +**Scan** uses lazy cursor iteration and stops when either `limit + 1` +in-segment items are accumulated or the cursor exhausts. It does not +impose a server-side hard limit. `Parallel Scan` filters items in the +application via `crc32(pk) % TotalSegments == Segment`. A hard +`(limit + 1) * TotalSegments` limit combined with post-fetch filtering +silently drops items under hot-key skew — an entire limit window can +land in one segment, terminating the scan with the others empty. +MongoDB batches under the hood (~101 docs), so lazy iteration is +efficient even without a hard limit — at most one extra network batch +beyond what is returned. + +Implementation: `data_engine.rs::query_impl`, `scan_impl`, +`build_sk_filter`, `next_string_prefix`, `increment_bytes`. + +### 5.4 GSI propagation (synchronous inline + async backfill) + +**Live writes** synchronize GSIs in the same session as the base write. +`sync_indexes_in_session` walks the `indexes` catalog for the table_id, +and for each index: + +1. If the old item had the index-key attributes, project it into the + index shape (`project_item`, respecting the `Projection` setting) and + run `delete_one` filtered on both index-key AND base-key components + (`index_entry_filter`). Filtering on index keys alone would delete + every base item's entry sharing those keys. +2. If the new item has the index-key attributes, project and upsert into + the index collection (`index_document` + `replace_one` with + `upsert: true`). + +The `gsi_cache` on `MongoEngine` (`DashMap`) +short-circuits the catalog walk when we know the table has no indexes. +Cache entries expire after `GSI_CACHE_TTL` (60s) so out-of-band GSI +changes on other ExtendDB instances converge within the window. + +**Async backfill.** `UpdateTable` GSI-create inserts the catalog +document with `index_status: "CREATING"` and pre-creates the mongo +index-collection + its compound query index (so live reads on the +CREATING index don't run collection scans). A background +`gsi_backfill_worker` (spawned in `MongoRuntimeHooks::spawn_workers`) +runs every 5 seconds: + +1. `find { index_status: "CREATING", index_type: "GSI" }` on the + `indexes` catalog collection. +2. For each job, read the base collection in batches of 500 items + (`backfill_gsi_batch`) starting from the row's persistent + `backfill_cursor` field. +3. Upsert projected items into the index collection. +4. After every batch, persist `backfill_cursor` back to the catalog + document so a mid-backfill server restart resumes where it left off. +5. When a batch returns fewer docs than the batch size (base fully + scanned), flip the catalog row to + `index_status: "ACTIVE"` and unset `backfill_cursor`. + +Live writes during the backfill window continue to hit `sync_indexes_in_session`, +which writes to CREATING indexes too — index-catalog membership, not +status, is what gates the write path. All writes are upserts on the +same `_id` shape, so a base item touched by both paths converges +regardless of interleaving. + +**Index-key input validation.** `validate_index_keys_for_item` rejects +wrong-type or empty index-key attributes on the item **before** any +write work (post-apply for `UpdateItem`). Without this, +`index_document` would silently skip the typed `sk_?` field when it +sees a type mismatch, leaving the resulting index row un-locatable for +subsequent deletes. Inside `TransactWriteItems`, the failure surfaces +as a per-item `CancellationReason::ValidationError` rather than a +top-level `ValidationException`. + +### 5.5 DynamoDB Streams + +Stream records are written inline during data operations, using the same +storage model as the PostgreSQL backend. This design gives ExtendDB full +control over sequence numbers, shard assignment, and retention — all of +which the DynamoDB Streams API contract tightly specifies. Native +MongoDB Change Streams are not used. + +**Shard model.** Four shards per stream-enabled table, created at +`CreateTable` (or on the first `UpdateTable` stream-enable). Shard IDs +embed the table's UUID: `shardId-{table_id}-{index:012}`. Table names +are only unique per account, so a name-derived scheme would allow +cross-tenant shard-id collisions on same-named tables. `table_id` is +per-instance; a `DeleteTable + CreateTable` sequence produces fresh +shard_ids, and `cleanup_stream_state_for_table` in `delete_table_impl` +removes the deleted table's shards, records, and counters so nothing +resurfaces on recreation. + +**Write path** (`write_stream_inline_in_session`): resolve the event +type from `(old_item, new_item)` presence, build key + old-image + +new-image per `StreamViewType`, hash the pk with CRC32 to select a +shard, draw the next sequence number, insert the record. Both shard +resolution and sequence-number assignment run inside the same session +as the data write. + +**Session-scoped sequence numbers.** `next_sequence_number_in_session` +does `find_one_and_update` with `$inc` on the per-shard counter +document — inside the write session. Without this, a fast writer B +could draw seq=6 and commit before a slow writer A (which drew seq=5) +commits; a consumer polling between B's commit and A's commit sees +seq=6 and advances past it, so when A finally commits, seq=5 lands +behind the cursor and is never returned. Session-scoped assignment +also serializes concurrent writers on the same shard: two `$inc`s +racing under snapshot isolation conflict at commit, and the loser +retries. + +**Per-shard counters.** Counter documents are keyed by +`_id: "stream_seq:"`. A single global counter would couple +the sequence spaces of unrelated shards, so a writer pushing records +into shard B would advance the counter shard A reads back — producing +non-contiguous sequence numbers on shard A's `GetRecords` pages. + +**Event names.** `event_name_ddb_str` emits DynamoDB wire casing +(`INSERT`, `MODIFY`, `REMOVE`). When `UpdateItem` creates an item +(upsert with no pre-image), the stream layer emits an `INSERT`, not a +`MODIFY` with a fabricated key-only `OldImage`. + +**Retention.** 24 hours, enforced by a TTL index on +`stream_records.created_at`. A `stream_record_cleanup_worker` (hourly) +provides defense in depth. + +**`GetRecords` path.** `{ shard_id: , sequence_number: { $gt: after } }` +with ascending sort, backed by the compound index +`(shard_id, sequence_number)`. + +**Non-session `write_stream_record`.** The `StreamEngine::write_stream_record` +trait method is a stub that returns an explicit error — the mongo backend +has no callers for it, and enrolling in the wrong or no session would let +a stream record commit while its base-table write rolls back. + +**`UpdateTable` stream-enable is idempotent.** If shards already exist +for the table, it reuses them and preserves the existing `stream_label`; +only a first-time enable rotates it. A repeat `UpdateTable` +`{ StreamEnabled: true }` would otherwise duplicate the shard set and +invalidate stream ARNs previously handed out to consumers. + +**`stream_label` format.** `YYYY-MM-DDThh:mm:ss` (second precision, no +timezone). Byte-for-byte compatible with the PostgreSQL backend so an +ARN issued by one backend is parseable by tooling that only ever saw +the other. See `format_stream_label` in `table_engine.rs`. + +### 5.6 Write conflict handling + +**Transient-conflict detection.** `is_transient_write_conflict` returns +true for any of: the `TransientTransactionError` label, the +`UnknownTransactionCommitResult` label, or a raw `WriteConflict` (code +112). Under snapshot isolation these all mean "your write lost to a +concurrent writer; retry the whole transaction." + +**Retry loop.** Session-scoped writes (Put / Delete / Update / TWI) wrap +the transaction body in a `for attempt in 0..TRANSIENT_RETRY_ATTEMPTS` +loop (50 attempts). Each attempt starts a fresh transaction, runs the +body, and either commits, aborts and retries (transient), or aborts and +returns (fatal). Retries sleep with jittered exponential backoff +(`backoff_sleep`, base 50 µs). + +**UpdateItem's OCC guard on top.** Even inside the transaction snapshot, +`UpdateItem` uses a `_v` version filter. The transaction guarantees the +snapshot the update was computed from; the versioned replace_one +guarantees the write only commits if the row's `_v` still matches what +we read. If `matched_count == 0` the attempt returns `Stale` (a distinct +signal from `Transient`) and the loop restarts. The native fast path +always emits `$inc: {_v: 1}` so a concurrent slow-path update racing +against a stale snapshot fails its filter and retries. + +**Exhaustion behavior.** Single-item retry exhaustion returns +`StorageError::Internal` (rare in practice; the retry ceiling is high). +`TransactWriteItems` exhaustion surfaces as +`StorageError::TransactionCanceled` with a synthetic per-op +`TransactionConflict` reason so wire consumers see the DDB-canonical +error string instead of a bare HTTP 500. + +**Conditional insert races.** A conditional PutItem on a nonexistent key +that raced against a concurrent inserter can manifest as either an +E11000 duplicate-key (unique-index race) or a WriteConflict (snapshot +race). The write path maps E11000 to `ConditionFailed` after +re-reading the winner outside the session; WriteConflict falls through +the normal retry loop, and the retry re-reads and sees the winner via +the existing-doc branch. + +### 5.7 TTL + +Two TTL surfaces: + +**Storage-native TTL indexes** — configured at bootstrap: +- `idempotency_tokens.created_at` — 540s (§4.6 for the rationale). +- `stream_records.created_at` — 24 h. +- `iam_sessions.expires_at` — `expireAfterSeconds: 0`. + +**Application-level DynamoDB TTL** — user-configured `TimeToLive` +attribute per table. MongoDB's native TTL runs at the storage engine +and cannot emit ExtendDB stream records with the required `Service` +user identity, so the backend maintains its own worker. + +`update_ttl` sets `ttl_attribute` on the table doc. `create_ttl_index` +creates a sparse index on `item_data.{ttl_attribute}.N` and flips +`ttl_index_ready: true`. The `ttl_cleanup_worker` (60s cadence) walks +tables with `ttl_index_ready`, finds expired items in batches of 100 +per table, and issues `DataEngine::delete_item` with a re-check +condition (`attribute_exists(ttl) AND ttl <= now`) to prevent races +with concurrent writes. The delete carries a `StreamCapture` with +`UserIdentity { identity_type: "Service", principal_id: "dynamodb.amazonaws.com" }` +so the stream record matches DynamoDB's format. + +### 5.8 Backups + +`CreateBackup` snapshots the source table by running a server-side +aggregation pipeline `[{ $out: "_backup_" }]` on the data +collection. `$out` writes the target collection server-side without +per-item traffic between the driver and the server. The destination +name is derived from a UUID because the caller-visible `backup_arn` +contains characters (`:`, `/`) that MongoDB does not allow in +collection names. + +`RestoreTableFromBackup` recreates the target table via +`CreateTable` (preserving `TableClass` / `SSESpecification` / +`OnDemandThroughput` from the backup metadata), then clones the backup +collection into the new data collection with the same `$out` pipeline +in reverse. + +`DeleteBackup` drops the physical collection using `backup_id` from +the metadata document and marks the metadata row `DELETED`. + +Implementation: `backup_engine.rs`. + +### 5.9 Account ID validation + +Injection defense on all account-scoped operations (`validate_account_id` +in `lib.rs`). Reject `$` (operator injection), `.` (field-path +traversal), null bytes, and non-ASCII. Runs before any query +construction. + +### 5.10 Catalog version check + +`read_catalog_version` returns the `settings.catalog_version` value; +`expected_catalog_version` returns the compiled-in `0.0.2`. The bin +layer compares them on `extenddb serve` startup — same pattern as the +PostgreSQL backend. ## 6. Crate Structure @@ -556,26 +815,28 @@ compiled-in constant. Same pattern as PostgreSQL. crates/storage-mongodb/ ├── Cargo.toml └── src/ - ├── lib.rs # MongoEngine struct, inventory registrations - ├── config.rs # Configuration parsing - ├── bootstrapper.rs # Database initialization (init/destroy) - ├── table_engine.rs # CreateTable, DeleteTable, DescribeTable, UpdateTable - ├── data_engine.rs # PutItem, GetItem, DeleteItem, UpdateItem, Query, Scan, Transactions - ├── data/mod.rs # Document <-> Item conversion helpers - ├── condition.rs # DynamoDB condition expressions -> MongoDB filters - ├── stream_engine.rs # DynamoDB Streams (shard management, sequence numbers) - ├── metadata_engine.rs # TTL, tags, table size tracking - ├── ttl_worker.rs # Background TTL cleanup - ├── backup_engine.rs # Backup/restore via collection cloning - ├── management_store.rs # IAM management, settings, metrics, rate limiting - ├── authorization_store.rs # Policy evaluation, boundaries, sessions - ├── credential_store.rs # Access key lookup with AES-GCM decryption - ├── catalog_store.rs # Catalog and diagnostics - ├── admin_store.rs # Admin operations - └── worker_store.rs # Control plane state transitions -``` - -## 7. MongoEngine Struct + ├── lib.rs # MongoEngine, GSI cache, inventory registrations + ├── config.rs # MongoStorageConfig + ├── operations.rs # OperationsEngine (CLI): connection parsing, redaction + ├── bootstrapper.rs # init / destroy / migrations + ├── table_engine.rs # CreateTable, UpdateTable, DeleteTable, DescribeTable + ├── data_engine.rs # PutItem, GetItem, DeleteItem, UpdateItem, Query, Scan, Transactions, pushdown fast path + ├── data/mod.rs # composite_id, item_to_document, index_document, binary_sk_to_hex + ├── condition.rs # DDB condition Expr → MongoDB filter compiler + ├── pushdown.rs # is_pushable analyzer — pushdown correctness boundary + ├── stream_engine.rs # Shard management, sequence numbers, GetRecords + ├── metadata_engine.rs # TTL configuration, tags, table size bookkeeping + ├── ttl_worker.rs # TTL sweep, stream record cleanup, GSI backfill workers + ├── backup_engine.rs # $out-based backup and restore + ├── management_store.rs # IAM CRUD, settings, metrics, rate limiting + ├── authorization_store.rs # Policy fetching for auth decisions + ├── credential_store.rs # Access-key lookup + AES-GCM decryption + ├── catalog_store.rs # SettingsStore / DiagnosticsStore glue + ├── admin_store.rs # Admin operations (currently thin) + └── worker_store.rs # Control-plane state transitions (CREATING/DELETING) +``` + +## 7. `MongoEngine` Struct ```rust pub struct MongoEngine { @@ -584,170 +845,250 @@ pub struct MongoEngine { data_db: mongodb::Database, region: String, max_connections: u32, - /// Cache of `table_id` -> `has_gsi`. Avoids catalog queries on every write - /// for tables with no GSIs. - gsi_cache: dashmap::DashMap, + gsi_cache: dashmap::DashMap, } + +const GSI_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); ``` -MongoDB's driver manages connection pooling internally (configurable via -`ClientOptions`). A single `Client` is shared; `Database` handles are lightweight -references. The `gsi_cache` provides a fast path to skip GSI catalog lookups for -tables known to have no indexes. +`MongoEngine::new` parses the connection string, rejects non-primary +read preferences, then constructs the client with `max_pool_size = +max_connections`. `catalog_db` and `data_db` are lightweight handles +against the single shared client. + +`gsi_cache` entries carry the observation time so a stale entry +(`elapsed() > GSI_CACHE_TTL`) is treated as a miss and re-read from the +catalog. This keeps writes correct when a GSI is added or dropped on +another ExtendDB instance sharing the catalog. ## 8. Configuration ```toml [storage.mongodb] -connection_string = "mongodb://localhost:27017" -pool_size = 20 +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +max_connections = 50 +max_catalog_connections = 20 ``` ```rust -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MongoStorageConfig { - #[serde(default = "default_connection_string")] pub connection_string: String, - #[serde(default = "default_pool_size")] - pub pool_size: u32, + #[serde(default = "default_max_connections")] + pub max_connections: u32, + #[serde(default = "default_max_catalog_connections")] + pub max_catalog_connections: u32, } ``` +The connection string must use `readPreference=primary` (the driver +default). Persisting the connection string in +`settings.data_connection_string` uses the raw string as-is; the bin +layer redacts the password from it for display via the +`OperationsEngine::redact_connection_string` hook. + ## 9. Bootstrapper Flow **`extenddb init`:** -1. Connect to MongoDB (databases created implicitly on first write) -2. Create catalog collections with indexes -3. Seed `settings` with `catalog_version` -4. Generate and store encryption key -5. Create default account -6. Create admin user -7. Create data database `idempotency_tokens` collection with TTL index -8. Record data database name in catalog settings - -**`extenddb destroy`:** -1. Drop data database -2. Drop catalog database + +1. Materialize both databases with sentinel collections (MongoDB creates + databases lazily on first write, so we explicitly create + `schema_history` in the catalog db and `idempotency_tokens` in the + data db). +2. `run_catalog_migrations` — create the 17 catalog collections and + their indexes; seed `settings.catalog_version = 0.0.2`; record + the migration in `schema_history`. +3. `create_data_db` — create `idempotency_tokens` (TTL + unique index), + `stream_shards` (unique on shard_id), `stream_records` (TTL + compound + query index). +4. `bootstrap_encryption_key` — generate a random 256-bit key, + base64-encode, insert into `settings` (idempotent — E11000 races are + silently ignored). +5. `bootstrap_default_account` — insert `default` account if none + exists. +6. `bootstrap_admin_user` — bcrypt-hash the password, insert into + `admin_users`. +7. `record_data_connection` — record `data_database_name` and + `data_connection_string` in `settings`. + +**`extenddb destroy`:** drop both databases. ## 10. Inventory Registrations -```rust -inventory::submit! { BackendRegistration { name: "mongodb", .. } } -inventory::submit! { OperationsEngineRegistration { name: "mongodb", .. } } -inventory::submit! { StorageConfigRegistration { backend: "mongodb", .. } } -inventory::submit! { ServerComponentsRegistration { backend: "mongodb", .. } } -inventory::submit! { SettingsStoreRegistration { backend: "mongodb", .. } } -``` +Six `inventory::submit!` blocks in `lib.rs`: + +- `BackendRegistration` — factory for `MongoBootstrapper`, called by + `extenddb init`. +- `OperationsEngineRegistration` — CLI operations (connection parsing, + redaction, identifier validation, sensitive-key detection). +- `StorageConfigRegistration` — TOML deserializer for + `[storage.mongodb]`. +- `SettingsStoreRegistration` — factory for `MongoCatalogStore` acting + as `SettingsStore`. +- `DiagnosticsStoreRegistration` — factory for `MongoCatalogStore` + acting as `DiagnosticsStore`. +- `ServerComponentsRegistration` — factory called by `extenddb serve` + that constructs `MongoEngine`, `MongoCatalogStore`, + `MongoCredentialStore`, and `MongoRuntimeHooks` (which spawns the + TTL, stream-cleanup, and GSI-backfill workers). + +No changes to `crates/server/`, `crates/engine/`, or `crates/auth/` +are required — everything flows through the plugin registration +system. ## 11. Dependencies ```toml [dependencies] -mongodb = { version = "3", features = ["tokio-runtime"] } -bson = "2" -dashmap = "6" -tokio = { workspace = true, features = ["sync"] } -futures = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } -time = { workspace = true } -uuid = { workspace = true } -base64 = { workspace = true } -rand = { workspace = true } -bcrypt = { workspace = true } -aes-gcm = { workspace = true } -async-trait = { workspace = true } -zeroize = { workspace = true } -inventory = { workspace = true } -extenddb-core = { workspace = true } -extenddb-storage = { workspace = true } -extenddb-auth = { workspace = true } -crc32fast = { workspace = true } -``` - -## 12. Implementation Phases - -### Phase 1: Core (MVP) -- Complete -- `MongoEngine` struct and connection setup -- `TableEngine` (create/delete/describe/list/update) -- `DataEngine` (put/get/delete/update/query/scan) with condition filter compiler -- `Bootstrapper` (init, destroy) -- `StorageConfig` and inventory registrations -- Unit tests against a local MongoDB replica set - -### Phase 2: Management & Auth -- Complete -- `CatalogStore` (ManagementStore, AdminStore, SettingsStore, MetricsStore, - RateLimitStore) -- `AuthorizationStore` -- `MongoCredentialStore` -- Web console and management API working - -### Phase 3: Streams & Transactions -- Complete -- `StreamEngine` (inline stream record writes, shard management, sequence numbers) -- `TransactGetItems` / `TransactWriteItems` -- Idempotency tokens - -### Phase 4: Advanced Features -- Complete -- `BackupEngine` (collection cloning via `$out`) -- `WorkerStore` (control plane state transitions) -- Synchronous inline GSI propagation with `DashMap` cache -- TTL worker (application-level DynamoDB TTL) -- `MetadataEngine` (full TTL lifecycle) - -### Phase 5: Testing & Production Readiness -- Complete -- Full pytest integration suite passes against MongoDB backend -- Performance benchmarking vs PostgreSQL backend -- Documentation +mongodb.workspace = true # 3.x, async, tokio-runtime +bson.workspace = true +dashmap.workspace = true # GSI existence cache +tokio.workspace = true +async-trait.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +toml.workspace = true +tracing.workspace = true +time.workspace = true +uuid.workspace = true +base64.workspace = true +rand.workspace = true +bcrypt.workspace = true +aes-gcm.workspace = true +zeroize.workspace = true +thiserror.workspace = true +anyhow.workspace = true +inventory.workspace = true +extenddb-core.workspace = true +extenddb-storage.workspace = true +extenddb-auth.workspace = true +crc32fast.workspace = true + +[dev-dependencies] +proptest = "1" # pushdown parity harness +``` + +## 12. Feature coverage + +The backend implements every trait in `extenddb-storage`: + +**Data plane** +- `TableEngine` — Create/Delete/Describe/List/UpdateTable, including + GSI create with async backfill, LSI create, and idempotent + stream-enable on UpdateTable. +- `DataEngine` — PutItem, GetItem, DeleteItem, UpdateItem, Query, + Scan (including parallel scan), TransactGetItems, TransactWriteItems, + BatchGetItem, BatchWriteItem. Condition expressions run session- + scoped with an analyzer-gated pushdown fast path for a certified + subset. +- `StreamEngine` — session-scoped per-shard sequence numbers, four + shards per stream, CRC32 pk routing, TRIM_HORIZON / LATEST / + AT_SEQUENCE_NUMBER / AFTER_SEQUENCE_NUMBER iterators, 24h retention. +- `MetadataEngine` — TTL lifecycle, tags, table-size tracking. +- `BackupEngine` — CreateBackup, RestoreTableFromBackup, DeleteBackup + via server-side `$out` aggregation. + +**Control plane and catalog** +- `Bootstrapper` — init, destroy, migrate, verify. Creates the + catalog and data databases, seeds encryption key and admin user, + applies index schema. +- `WorkerStore` — CREATING → ACTIVE / DELETING → dropped transitions, + polled every scan interval. +- `ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, + `RateLimitStore` — the catalog trait surface. +- `AuthorizationStore` — user/group/role/permissions-boundary/session + policy lookup for IAM evaluation. +- `MongoCredentialStore` — SigV4 credential resolution with + AES-GCM-decrypted secret keys. + +**Background workers** — spawned from `MongoRuntimeHooks::spawn_workers`: +- `ttl_cleanup_worker` — sweep expired items every 60 s, emit + service-attributed stream records for the deletes. +- `stream_record_cleanup_worker` — hourly defense-in-depth for the + 24 h retention TTL index. +- `gsi_backfill_worker` — drain `indexes` rows in `CREATING` state, + scan the base collection with a persistent cursor, flip to ACTIVE. ## 13. Testing Strategy -- **Unit tests:** Mock the MongoDB client for pure logic tests -- **Integration tests:** Single-node replica set in Docker (`mongod --replSet rs0`) -- **Existing pytest suite:** Passes unchanged (speaks DynamoDB wire protocol) -- **CI:** GitHub Actions job with MongoDB replica set, runs - `cargo test -p extenddb-storage-mongodb` +- **Unit tests:** netstring `_id` encoding, hex sort-key ordering, + condition compiler, shard-id derivation, next_string_prefix, + Decimal128 rejection, index-doc key disambiguation. +- **Property tests:** `tests/pushdown_parity.rs` — random items and + expressions checked for agreement between the compiled BSON filter + and `evaluate_condition`. +- **Integration tests:** Single-node replica set in Docker + (`mongod --replSet rs0`), full trait coverage. +- **Existing pytest suite:** Passes unchanged (backend-agnostic wire + protocol tests). +- **CI:** GitHub Actions job with MongoDB 6.0 replica set, runs + `cargo test -p extenddb-storage-mongodb` then `devtools/run-tests + --extenddb --pytest --external`. ## 14. Deployment Requirements -- MongoDB **6.0+** in **replica set** mode (required for multi-document transactions) -- Single-node replica set is fine for development/testing -- For production: 3-node replica set -- Target scale: < 500 DynamoDB tables. At 500 tables with 2 GSIs each (~1500 - collections), WiredTiger handles this comfortably with default settings. - Ensure `ulimit -n` ≥ 65536. +- MongoDB **6.0+** in **replica set** mode. Standalone nodes reject + multi-document transactions. +- **`readPreference=primary`** on the connection string. Non-primary + is rejected at engine startup. +- Single-node replica set is fine for development / CI. Production: + 3-node replica set. +- Target scale: < 500 DynamoDB tables. At 500 tables with 2 GSIs each + (~1,500 collections), WiredTiger handles the count comfortably with + default settings. Ensure `ulimit -n ≥ 65536`. ## 15. Design Decisions Summary | Decision | Choice | Rationale | |----------|--------|-----------| -| Conditional writes | Filter pushdown (no transaction) | Single-document atomicity, no tx overhead on hot path | -| GSI updates | Synchronous inline | Simplicity, no Change Stream recovery complexity, strongly consistent | -| DynamoDB Streams | Inline record writes to MongoDB collections | Behavioral parity with PostgreSQL backend, explicit sequence numbers | -| Stream shards | 4 per table, CRC32 hash assignment | Predictable parallelism for consumers | -| Sort key numbers | Native BSON Decimal128 | Correct ordering by value, zero encoding overhead | -| Backups | `$out` collection clone | Server-side, no size limits | -| Parallel scan | Filter in application | Rarely used, not worth write-path overhead of `_seg` field | -| Write conflict (UpdateItem) | Optimistic concurrency with `_v` field + jittered backoff | Avoids transactions for single-item updates | +| Conditional writes | Read + evaluate + write inside a MongoDB transaction session | Snapshot atomicity delivers DDB's contract; pre-image reused for `ReturnValuesOnConditionCheckFailure = ALL_OLD` and `OldImage`. | +| Filter pushdown | Analyzer-gated fast path; `is_pushable` certifies a subset | Compiler in `condition.rs` handles broader syntax than production uses; the analyzer is the correctness boundary. | +| GSI live sync | Synchronous inline within the base write's session, gated by a 60s-TTL cache | Strongly-consistent GSI reads; no Change Stream recovery. | +| GSI async backfill | `CREATING` → `ACTIVE` via `gsi_backfill_worker` with persistent `backfill_cursor` | Matches DDB's async UpdateTable contract; restart-safe. | +| Index-doc identity | Composite `_id = netstring(idx_pk, idx_sk, base_pk, base_sk)` + `base_pk` / `base_sk_?` as first-class fields | Base-key disambiguation for non-unique index keys; compound cursor pagination without touching `item_data`. | +| Composite `_id` | Netstring `:,...` | Delimiter-free framing between pk and sk. | +| Binary sort keys | Stored as lowercase hex strings | BSON Binary comparison is length-first-then-content, diverging from DDB unsigned-lex byte order across mismatched lengths. | +| Sort key numbers | Native BSON `Decimal128`; values exceeding 34 sig-digits rejected | Correct numeric ordering; no silent precision loss. | +| DynamoDB Streams | Inline record writes to `stream_records` in the base write's session; per-shard sequence counters | Behavioral parity with PostgreSQL backend; per-shard monotonicity is a contract. | +| Sequence assignment | Inside the write session | Prevents ordering holes where a fast writer commits a higher seq before a slower earlier one does. | +| Stream shard ID | `shardId-{table_id}-{i:012}` | Cross-tenant isolation on same-named tables. | +| Stream retention | 24h TTL index on `stream_records.created_at` + hourly cleanup worker | Primary enforcement at storage; worker is defense in depth. | +| WriteConflict handling | Retry with jittered exponential backoff (50 attempts); TWI exhaustion → `TransactionCanceled` with synthetic `TransactionConflict` reasons | Bounded tail latency; DDB-canonical error surface. | +| UpdateItem concurrency | Snapshot txn + `_v` version filter + retry; native fast-path always `$inc: {_v: 1}` | Prevents lost updates; fast path stays safe against a concurrent slow path. | +| Idempotency retention | Unique `(account_id, token)` index + 540s TTL + 600 ms data-plane age filter | Race safety under snapshot isolation; ≤10-min worst-case retention regardless of TTL-monitor cadence. | +| Backups | Per-backup collection via server-side `$out` aggregation | No per-item driver traffic; metadata schema decoupled from collection name. | +| Parallel scan | Application-side `crc32(pk) % segments` + lazy cursor | Rare feature; server-side bucketing would tax every write. Lazy iteration prevents item-drops under hot-key skew. | +| Read preference | `primary` enforced at engine startup | `ConsistentRead=true` requires linearizable reads. | ## 16. Performance Characteristics -**Hot path (single-item writes):** Transaction-free. A PutItem with condition is a -single `findOneAndReplace` with a filter — one network roundtrip, one WiredTiger -document write. No locking, no multi-phase commit. - -**GSI overhead on write path:** One catalog query per write for tables with GSIs -(to fetch index definitions), plus one upsert/delete per GSI. For tables with no -GSIs, the `gsi_cache` short-circuits to zero overhead (no catalog query, no I/O). - -**Stream overhead on write path:** When streaming is enabled, one counter increment -(atomic `findOneAndUpdate`) plus one document insert to `stream_records` per write -operation. - -**Query/Scan:** Direct index lookups on `{ pk, sk_* }`. Same performance -characteristics as any indexed MongoDB query. - -**TransactWriteItems:** Multi-collection transaction. Rare in practice (most -workloads are single-item operations). Limited to 100 operations per DynamoDB -API spec. +**Hot path — single-item conditional write.** One MongoDB transaction +session covers pre-image read, condition eval, base write, GSI sync, +stream insert (per-shard counter `$inc` + document insert). On a local +replica set, session overhead is ~sub-ms over a raw driver call. The +session is what buys the DDB atomicity contract — it is the +compatibility, not overhead. The pushdown fast path collapses this to +a single `find_one_and_*` for the certified subset on tables with no +GSIs / streams. + +**Unconditional single-item update, no GSIs, no streams.** Native +fast path: one `find_one_and_update` with compiled `$set` / `$unset` / +`$inc` outside any session. The `$inc: {_v: 1}` keeps the fast path +safe against a concurrent slow-path update. + +**GSI write overhead.** No GSIs: zero (cached). Has GSIs: one catalog +`find` (cached for subsequent writes on the same table until +`GSI_CACHE_TTL` elapses) + one upsert or delete per index per write, +all inside the base write's session. + +**Stream write overhead.** One counter `$inc` + one `stream_records` +insert per write, inside the base write's session. + +**Query / Scan.** Index lookups on `(pk, sk_?)` for base tables and +`(pk, sk_?, base_pk, base_sk_?)` for index queries. `GetRecords` +uses the compound `(shard_id, sequence_number)` index. + +**TransactWriteItems.** Multi-collection ACID transaction with +snapshot read concern and majority write concern; up to 100 operations +per the DDB spec. Retried on transient conflicts with jittered backoff. diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index 1bd267d6..a1b5d7c5 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -7,20 +7,19 @@ ## Summary -This RFC proposes adding MongoDB as a backend for ExtendDB. The goal is to let developers run DynamoDB-compatible workloads on MongoDB while preserving ExtendDB’s core value: a DynamoDB-compatible API over multiple storage backends. The implementation covers all mandatory traits defined in RFC-0002 and all optional traits, uses ExtendDB's existing `inventory`-based plugin registration system without modifying the server or engine layers, and is maintained by the MongoDB team who commit to ongoing ownership of the backend crate. +This RFC proposes adding MongoDB as a backend for ExtendDB. The goal is to let developers run DynamoDB-compatible workloads on MongoDB while preserving ExtendDB's core value: a DynamoDB-compatible API over multiple storage backends. The implementation covers all mandatory traits defined in RFC-0002 and all optional traits, uses ExtendDB's existing `inventory`-based plugin registration system without modifying the server or engine layers, and is maintained by the MongoDB team who commit to ongoing ownership of the backend crate. ## Motivation -ExtendDB’s core premise is DynamoDB API compatibility over multiple storage backends. The initial reference PostgreSQL backend demonstrates the feasibility of this approach while opening the opportunity for other databases to participate. +ExtendDB's core premise is DynamoDB API compatibility over multiple storage backends. The initial reference PostgreSQL backend demonstrates the feasibility of this approach while opening the opportunity for other databases to participate. -MongoDB is a natural fit as an additional database target: data model alignment; high read/write throughput through horizontal scalability; infrastructure fit. +MongoDB is a natural fit as an additional database target: data model alignment; high read/write throughput through horizontal scalability; infrastructure fit. -DynamoDB and MongoDB share the same data model approach - documents stored as schema-less JSON-like data. MongoDBs document model maps directly to the approach taken by DynamoDB with each item stored as a MongoDB BSON document with no impedance mismatch at the data model level. Unlike relational databases, the translation from JSON to BSON is direct without complicated relational mapping techniques required. +DynamoDB and MongoDB share the same data model approach — documents stored as schema-less JSON-like data. MongoDB's document model maps directly to the approach taken by DynamoDB with each item stored as a MongoDB BSON document with no impedance mismatch at the data model level. Unlike relational databases, the translation from JSON to BSON is direct without complicated relational mapping techniques required. -Customers evaluating DynamoDB and MongoDB often consider scalability as a key requirement. ExtendDB’s deployment approach requiring high write throughput is matched by MongoDB’s replica set model via horizontal scaling. High read and write throughput across multiple nodes is a core tenant of MongoDB and aligns naturally with the scalability requirement for an ExtendDB customer. - -Organizations running ExtendDB, DynamoDB, and MongoDB have already evaluated the usefulness of a non-relational database approach. These shared customers do not want to run PostgreSQL or other relational databases solely for DynamoDB compatibility. Rather, taking advantage of the infrastructure they already run that aligns with the document model design and scalability requirements they require makes MongoDB a natural fit. +Customers evaluating DynamoDB and MongoDB often consider scalability as a key requirement. ExtendDB's deployment approach requiring high write throughput is matched by MongoDB's replica set model via horizontal scaling. High read and write throughput across multiple nodes is a core tenet of MongoDB and aligns naturally with the scalability requirement for an ExtendDB customer. +Organizations running ExtendDB, DynamoDB, and MongoDB have already evaluated the usefulness of a non-relational database approach. These shared customers do not want to run PostgreSQL or other relational databases solely for DynamoDB compatibility. Rather, taking advantage of the infrastructure they already run that aligns with the document model design and scalability requirements they require makes MongoDB a natural fit. ## Detailed design @@ -47,140 +46,157 @@ This RFC does not propose: The backend lives at `crates/storage-mongodb/` in the main ExtendDB repository, following the mono-repo structure prescribed by RFC-0002. It is selected at build time via a `mongodb` Cargo feature flag on the `extenddb` binary crate. - Feature flag definition: `crates/bin/Cargo.toml` — `[features]` section defines `mongodb = ["extenddb-storage-mongodb"]` with the crate as an optional dependency. The `postgres` feature remains the default. Both features can be enabled together to compile a binary supporting both backends. ### Plugin registration -The backend registers itself with ExtendDB's `inventory`-based plugin system without modifying the server, engine, or auth layers. Five `inventory::submit!` calls in `lib.rs` register the backend for: bootstrapping (`extenddb init`), config parsing, settings store access, diagnostics store access, and server component construction (`extenddb serve`). - +The backend registers itself with ExtendDB's `inventory`-based plugin system without modifying the server, engine, or auth layers. Six `inventory::submit!` calls in `lib.rs` register the backend for: operations engine (CLI commands), bootstrapping (`extenddb init`), config parsing, settings store access, diagnostics store access, and server component construction (`extenddb serve`). -All five registration blocks: `crates/storage-mongodb/src/lib.rs`. The `ServerComponentsRegistration` block is the critical one — it is the factory function called when `extenddb serve --backend mongodb` is run. No changes were required in `crates/server/`, `crates/engine/`, or `crates/auth/`. +All registration blocks live in `crates/storage-mongodb/src/lib.rs`. The `ServerComponentsRegistration` block is the critical one — it is the factory function called when `extenddb serve --backend mongodb` is run. No changes are required in `crates/server/`, `crates/engine/`, or `crates/auth/`. ### Database layout The backend uses two MongoDB databases: -**`extenddb_catalog`** — metadata and management. Created on `extenddb init`. Contains 18 collections covering: table definitions (`tables`), index metadata, accounts, tags, admin users, IAM users, groups, group memberships, roles, access keys, IAM sessions, policies, permissions boundaries, session data, settings, metrics, login attempts, continuous backup metadata, and schema migration history. `iam_group_members` and `backup_items` are used by the runtime but auto-created on first insert rather than during migrations. +**`extenddb_catalog`** — metadata and management. Created on `extenddb init`. Contains collections for table definitions (`tables`), index metadata (`indexes`), accounts, tags, admin users, IAM users, groups, roles, access keys, IAM sessions, policies, permissions boundaries, settings, metrics, login attempts, backup metadata, continuous backup state, and schema migration history. -**`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named the same way with the index's ID. Shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `idempotency_tokens` for transaction deduplication, and `counters` for sequence number generation. +**`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named `_ddb_{index_id}`. Shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `counters` for per-shard sequence-number counters, `idempotency_tokens` for transaction deduplication, and one `_backup_{backup_id}` collection per user-created backup. - -Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()` creates the migration-managed catalog collections and their MongoDB indexes; `schema_history` is created separately by `create_catalog_db()` in the same file. Data database setup: `create_data_db()` creates `idempotency_tokens` with a 10-minute TTL index. Collection naming convention: `crates/storage-mongodb/src/data/mod.rs` — `data_collection_name()` and `index_collection_name()`. +Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()`. Data-database setup (`idempotency_tokens`, `stream_shards`, `stream_records` and their indexes): `create_data_db()` in the same file. Collection naming: `data/mod.rs` — `data_collection_name()` and `index_collection_name()`. ### Document structure for DynamoDB items -Each DynamoDB item is stored as a MongoDB document with the following structure: +Each DynamoDB item is stored as a MongoDB document: ``` { - _id: "partitionKeyValue#sortKeyValue", + _id: "", pk: "partitionKeyValue", sk_s: "sortKeyValue", // string sort keys - sk_n: Decimal128(...), // number sort keys, native MongoDB numeric type - sk_b: Binary(...), // binary sort keys + sk_n: Decimal128(...), // number sort keys, native BSON Decimal128 + sk_b: "aabb...", // binary sort keys, lowercase hex string + _v: NumberLong, // OCC version counter (present on updated docs) item_data: { ... full DynamoDB item in DynamoDB JSON format ... } } ``` -The `_id` field enables O(1) point lookups. The `pk` field is indexed separately to support partition scans (Query operations). Sort keys are stored in typed fields (`sk_s`, `sk_n`, `sk_b`) so MongoDB can apply native range comparisons with correct ordering — notably, numeric sort keys use MongoDB's `Decimal128` type rather than strings to ensure correct numeric ordering. Values that exceed Decimal128's 34-significant-digit precision are rejected at write and query time with a ValidationException, rather than downcast; DynamoDB itself supports up to 38 significant digits. This is documented as a backend-specific behavioral difference (see `docs/differences-from-dynamodb.md`). The full item is stored in `item_data` using DynamoDB's own type-tagged format (`{"S": "hello"}`, `{"N": "42"}`, etc.), preserving type information for non-key attributes. +The `_id` is a netstring-encoded composite key of the form `:,:,`. Netstring framing prevents the collision an ad-hoc `"{pk}#{sk}"` scheme suffers from when either component contains the delimiter (e.g. `pk="a#b", sk="c"` and `pk="a", sk="b#c"`). PK-only tables use the raw pk text as `_id`. -String sort key collections are created with `{ locale: "simple" }` collation, ensuring byte-for-byte ordering that matches DynamoDB's behavior rather than locale-aware Unicode ordering. +Typed sort-key fields (`sk_s`, `sk_n`, `sk_b`) let MongoDB apply native range comparisons with correct ordering: +- **String** sort keys use collection-level `{ locale: "simple" }` collation so range comparisons are byte-order, matching DynamoDB. +- **Numeric** sort keys use BSON `Decimal128`. Values whose precision exceeds Decimal128 (34 significant digits) are rejected at write and query time with a ValidationException rather than silently downcast. DynamoDB itself supports 38 significant digits; this is documented in `docs/differences-from-dynamodb.md`. +- **Binary** sort keys are stored as lowercase hex strings. BSON's native Binary comparison is length-first-then-content, which diverges from DynamoDB's unsigned-lex byte order (DynamoDB says `[0x01,0xFF] < [0x02]`; BSON Binary reverses that). Hex-encoded strings preserve DynamoDB byte order under MongoDB's default lexicographic string comparison and let `begins_with` use a plain `$gte` / `$lt` range filter. -Document conversion functions: `crates/storage-mongodb/src/data/mod.rs` — `item_to_document()` (DynamoDB Item → BSON document) and `document_to_item()` (BSON document → DynamoDB Item). Sort key type handling including `Decimal128`: same file, `item_to_document()` `ScalarAttributeType::N` branch. Collation: `crates/storage-mongodb/src/table_engine.rs` — `CreateTable` implementation, index creation with `Collation` options. +The full item is stored in `item_data` using DynamoDB's own type-tagged format (`{"S": "hello"}`, `{"N": "42"}`), preserving type information for non-key attributes. Item conversion helpers live in `data/mod.rs` (`item_to_document`, `document_to_item`, `composite_id`, `binary_sk_to_hex`). -### Condition expression evaluation +**Secondary-index documents** carry a superset of these fields. In addition to the index-key components (`pk`, `sk_?`), each index document also stores the base-table key attributes as first-class fields: `base_pk` (text) and `base_sk_s|n|b` (typed). The `_id` is a 4-tuple netstring `[idx_pk, idx_sk, base_pk, base_sk]`. GSI keys are non-unique across base items, so encoding the base key into `_id` gives each index entry a unique identity keyed to the base item it describes; without this, two base items sharing an index-key value would upsert to the same document and one would silently overwrite the other. The base-key fields also let index pagination form a compound cursor `(index_sk, base_pk, base_sk)` without traversing the JSON `item_data` payload. See `index_document()` and `index_entry_filter()` in `data/mod.rs`. -Condition expressions (`ConditionExpression` on PutItem, DeleteItem, UpdateItem) are evaluated inside a MongoDB client session that also wraps the write. Within the session, the backend reads the current item, evaluates the condition in Rust against the loaded item, and issues the write — the read and write share the session's transactional atomicity, so a concurrent writer's changes are either visible to the condition or cause the write to observe its version guard (see Write conflict handling). This delivers DynamoDB's atomicity contract for conditional writes and matches the `ReturnValuesOnConditionCheckFailure = ALL_OLD` semantics naturally (the read result is available for the response). +### Condition expression evaluation -The condition compiler exists as scaffolding for a follow-up optimization: it translates DynamoDB expressions into MongoDB filter documents that could be pushed into `findOneAndReplace` / `findOneAndDelete` for a single-round-trip conditional write. The compiler handles: `attribute_exists`, `attribute_not_exists`, `attribute_type`, `begins_with`, `contains`, `BETWEEN`, `IN`, `=`, `<>`, `<`, `<=`, `>`, `>=`, `AND`, `OR`, `NOT`. Because items are stored with DynamoDB type tags, compiled paths include the type suffix: `item_data.fieldName.S` for strings, `.N` for numbers. The compiler is not on the load-bearing correctness path today; the session-scoped read-then-write is. +Conditional writes (`ConditionExpression` on PutItem, DeleteItem, UpdateItem) run the condition read, evaluation, and write inside a MongoDB client session bound to a multi-document transaction. Within the session, the backend reads the current item, evaluates the DynamoDB condition in Rust against the loaded item (`extenddb_core::expression::evaluate_condition`), and issues the write on the same session. Read and write share the transaction's snapshot isolation, so a concurrent writer's changes either become visible to the condition (in which case the caller sees the same outcome as if the writes were serial) or trigger a WriteConflict at commit (which the write path retries — see Write conflict handling). -**Alternative considered — filter pushdown as the primary path.** Compiling filters into `findOneAndReplace` / `findOneAndDelete` would reduce a conditional write to a single round-trip. It is planned as a follow-up optimization once the compiler is exercised through the integration test suite. The current design was chosen because it gives us the loaded item for `ReturnValuesOnConditionCheckFailure` responses without a follow-up read, and because the compiler's behavior on edge-case expressions (multi-valued `size()`, `NOT` on missing paths, mixed-type set membership) is easier to validate incrementally under the session-scoped path. +This delivers DynamoDB's atomicity contract for conditional writes and matches `ReturnValuesOnConditionCheckFailure = ALL_OLD` semantics naturally: the item loaded to evaluate the condition is reused directly in the failure response. +An optional filter-pushdown fast path (`pushdown.rs` + `condition.rs`) skips the session for a restricted subset of conditions on tables that have no GSIs and no stream capture. A compile-time analyzer (`is_pushable`) certifies that a condition's compiled MongoDB filter agrees with `evaluate_condition` on every item; when it says yes, the backend collapses read + check + write into a single `find_one_and_replace` / `find_one_and_delete` or replace with a merged key+condition filter. The analyzer is the correctness boundary: the compiler in `condition.rs` covers a broader syntax (numeric compare, sets, `IN`, `BETWEEN`, `size`, arbitrary `NOT`) than the analyzer certifies, and only the analyzer-approved subset ever reaches production. Anything else falls through to the session-scoped path, which is always authoritative. -Session-scoped condition path: `crates/storage-mongodb/src/data_engine.rs` — `put_item_impl()`, `delete_item_impl()`, `update_item_impl()`, and the four `OwnedTransactWriteOp` arms in `execute_write_op_in_session()` each do read → `expression::evaluate_condition` → write, all with the session bound to each driver call. Condition compiler (scaffolding for the future pushdown path): `crates/storage-mongodb/src/condition.rs` — `condition_to_filter()` is the entry point. Unit tests in the same file demonstrate each compiled output. +Session-scoped condition path: `data_engine.rs` — `put_item_impl`, `delete_item_impl`, `update_item_impl`, and each `OwnedTransactWriteOp` arm in `execute_transact_write_op_in_session`. Pushdown fast path: `delete_item_pushdown` and `update_item_pushdown` in the same file, gated on `is_pushable(cond, maps) == Yes && stream.is_none() && gsi_cache_get_fresh(table_id) == Some(false)`. ### Query and Scan -**Query** translates `KeyConditionExpression` to a MongoDB `find()` filter. Partition key equality maps to `{ pk: "" }`. Sort key conditions map to typed range filters on `sk_s`, `sk_n`, or `sk_b`. `ScanIndexForward: false` applies a descending sort. Pagination uses `ExclusiveStartKey` to add a `$gt` or `$lt` bound on the sort key, making each page fetch a single indexed range query. +**Query** translates `KeyConditionExpression` to a MongoDB `find()` filter. Partition key equality maps to `{ pk: "" }`. Sort key conditions map to typed range filters on `sk_s`, `sk_n`, or `sk_b`. `BETWEEN` with `low > high` is rejected upfront with a ValidationException. `begins_with` on strings emits `{ $gte: prefix, $lt: next_string_prefix(prefix) }`, where the upper bound is the least string strictly greater than any prefix-starting string (built by incrementing the rightmost non-`char::MAX` code point). `begins_with` on binary emits the same range shape on the hex-encoded sort key. `ScanIndexForward: false` applies a descending sort. -**Scan** performs a full collection scan with `.find({})`, paginated via sort-key-based cursor. Filter expressions are evaluated after retrieval. +Pagination via `ExclusiveStartKey` **merges** the resume bound into the existing sort-key predicate rather than replacing it. Naively inserting `{sk: {$gt: cursor}}` drops the caller's original `BETWEEN` / `begins_with` bound and returns items outside it on page 2 and beyond. The merge covers three cases: no existing sk predicate (insert), existing operator map (merge into it), and existing equality (fall back to `$and`). See `query_impl` in `data_engine.rs`. -**Parallel scan** (`Segment` / `TotalSegments`) is handled in the application: each segment filters documents using `crc32(pk) % TotalSegments == Segment`. This means each segment scans the full collection. Pre-bucketing documents at write time would avoid this but adds overhead to every write for a feature that is rarely used in practice. The current tradeoff favors write-path simplicity. +**Index queries** paginate over a compound tuple `(index_sk?, base_pk, base_sk?)`. Index-key values are non-unique — duplicates fall through to the base-key tie-breaker. The cursor is expressed as a lexicographic `$or` of the form `(a > A) OR (a == A AND b > B) OR (a == A AND b == B AND c > C)` (reversed for descending). Sort direction is applied to the same compound tuple so ordering is deterministic across groups of items sharing index keys. `LastEvaluatedKey` carries both index-key and base-key components so the next page's `ExclusiveStartKey` resolves the compound cursor. -### Global Secondary Indexes (GSI) and Local Secondary Indexes (LSI) +**Scan** performs a full collection scan with lazy cursor iteration. Base-table scans paginate on `_id` (unique after netstring encoding). Index scans paginate on the same compound cursor as index Query. Filter expressions are evaluated after retrieval. -Each secondary index has its own MongoDB collection. On writes that modify indexed attributes, the backend updates the index collection in the same operation, maintaining synchronous GSI propagation. A `DashMap` in-memory cache on `MongoEngine` tracks which tables have GSIs, avoiding catalog lookups on every write to tables with no indexes. +**Parallel scan** (`Segment` / `TotalSegments`) filters items in the application via `crc32(pk) % TotalSegments == Segment`. Each segment scans the full collection. The scan loop streams the cursor and terminates when either `limit + 1` in-segment items are accumulated or the cursor exhausts, without imposing a server-side hard limit — a hard `limit * total_segments` cap combined with post-fetch segment filtering silently drops items under any hot-key skew. Pre-bucketing documents at write time would avoid the per-segment full scan but adds overhead to every write for a feature that is rarely used in practice. +### Global and Local Secondary Indexes -GSI collection creation: `crates/storage-mongodb/src/table_engine.rs` — `CreateTable` implementation. GSI cache: `crates/storage-mongodb/src/lib.rs` — `MongoEngine` struct `gsi_cache` field. Index write propagation: `crates/storage-mongodb/src/data_engine.rs` — `put_item_impl()` and related write paths check the cache before updating index collections. +Each secondary index has its own MongoDB collection with a compound index on `(pk, sk_?, base_pk, base_sk_?)` (created by `create_index_data_collection` in `table_engine.rs`). String-sorted index columns use `simple` collation matching the query path. On writes that modify indexed attributes, the backend synchronizes the index collection in the same session as the base write: `sync_indexes_in_session` deletes the old projected entry (filtered on the full index-key + base-key tuple so duplicate index keys don't cross-delete) and upserts the new one. + +A `DashMap` in-memory cache on `MongoEngine` short-circuits the catalog lookup for tables known to have no indexes. Cache entries carry an insertion timestamp and expire after 60 seconds (`GSI_CACHE_TTL`), so out-of-band GSI changes on another ExtendDB instance converge within the TTL window. + +**Async GSI backfill.** `UpdateTable`'s GSI-create path writes the catalog document with `index_status: "CREATING"` and pre-creates the mongo collection and its query index; a background `gsi_backfill_worker` (in `ttl_worker.rs`) discovers `CREATING` rows, iterates the base collection in batches keyed by a persistent `backfill_cursor` field on the index document, upserts projected items into the index collection via `backfill_gsi_batch`, and flips the status to `ACTIVE` when the base is fully scanned. The cursor is persisted between batches so a mid-backfill server restart resumes where it left off. Live writes during the backfill window continue to route through `sync_indexes_in_session`, which writes to CREATING indexes too — all writes are upserts on the same `_id` shape, so a base item touched by both paths converges regardless of interleaving. + +Before every put and update, `validate_index_keys_for_item` rejects wrong-type or empty index-key attributes as a top-level `ValidationException` (or a per-item `CancellationReason` inside `TransactWriteItems`). Without this check, a mismatched-type index-key attribute would be silently dropped from the index doc, leaving the row un-locatable for subsequent deletes. ### Transactions -`TransactWriteItems` uses MongoDB multi-document ACID transactions — a client session is opened, all operations execute within it, and the session is committed or aborted atomically. This requires MongoDB to be running as a replica set (standalone MongoDB does not support multi-document transactions). `TransactGetItems` performs a consistent snapshot read. +`TransactWriteItems` runs all operations inside a single MongoDB multi-document ACID transaction with snapshot read concern and majority write concern. Each operation's condition evaluation, base-row write, GSI synchronization, and stream record insert happens on the same `ClientSession` (`sync_indexes_in_session` + `write_stream_inline_in_session` are called inline from each `OwnedTransactWriteOp` arm). Without this, a transactional write to a GSI-bearing or streams-enabled table would commit the base row while silently dropping its dependent side effects. -Idempotency tokens for `TransactWriteItems` are stored in the `idempotency_tokens` collection in `extenddb_data` with a 10-minute MongoDB TTL index, matching DynamoDB's 10-minute idempotency window. +Idempotency tokens live in the `idempotency_tokens` collection in `extenddb_data`. A unique compound index on `(account_id, token)` catches races between concurrent transacts under snapshot isolation — an inserter that races through the pre-check gets an `E11000` on insert, resolved by re-reading the winner and returning `IdempotentReplay` (fingerprints match) or `IdempotentMismatch` (fingerprints differ). Retention is enforced by a 540-second MongoDB TTL index plus a data-plane age filter (`created_at` within 600 000 ms) so worst-case retention stays ≤10 minutes regardless of the TTL monitor's ~60s cadence. +`TransactGetItems` performs a consistent snapshot read using a `ClientSession` with snapshot read concern. -Transaction implementation: `crates/storage-mongodb/src/data_engine.rs` — `transact_write_items_impl()`. Idempotency token storage: same file, idempotency check at the start of `transact_write_items_impl()`. Replica set requirement documentation: `docs/local-mongodb-setup.md` — replica set initialization section. +Transaction implementation: `data_engine.rs` — `transact_write_items_impl`, `transact_get_items_impl`, `execute_transact_write_op_in_session`. ### Write conflict handling -**UpdateItem** uses optimistic concurrency. A `_v` version counter is stored on each document. The write path reads the current `_v`, applies the update expression in memory, sets `_v = current_version + 1`, then executes `replaceOne` filtered on both the primary key and the expected `_v`. If `matched_count == 0`, a concurrent writer incremented the version first. The operation retries with jittered exponential backoff (50 µs base, up to 50 attempts). Exhausted retries propagate the error to the caller. +**Session-scoped writes** (PutItem, DeleteItem, UpdateItem, TransactWriteItems) detect transient MongoDB conflicts via `is_transient_write_conflict`, which returns true for any of: the `TransientTransactionError` label, the `UnknownTransactionCommitResult` label, or a raw `WriteConflict` (code 112). Conflicts trigger a retry loop with jittered exponential backoff (`backoff_sleep`, base 50 µs) up to `TRANSIENT_RETRY_ATTEMPTS` (50). Exhausted retries on single-item operations return a `StorageError::Internal`; on `TransactWriteItems`, they surface as a `TransactionCanceled` with a synthetic per-op `TransactionConflict` cancellation reason so wire consumers see the DDB-canonical error string instead of a bare HTTP 500. -**PutItem and DeleteItem** run their conditional read, condition evaluation, and write within a single MongoDB client session (see Condition expression evaluation). The item loaded to evaluate the condition is reused directly for the `ReturnValuesOnConditionCheckFailure = ALL_OLD` response — no follow-up read is issued. When no condition is present, the write is a straight `findOneAndReplace` / `findOneAndDelete`. +**UpdateItem** additionally uses an optimistic-concurrency version guard on top of the transaction. A `_v` counter is stored on each document. The write path reads the current `_v` under the transaction snapshot, applies the update expression in memory, sets `_v = current_version + 1`, then executes `replace_one` filtered on both the primary key AND the expected `_v`. If `matched_count == 0`, a concurrent writer committed a higher version between the snapshot read and the replace; the attempt aborts and the outer retry loop re-reads. The version guard doubles up when a native-fast-path update (unconditional, no stream, no GSI) is possible — that path uses a single `find_one_and_update` with `$inc: {_v: 1}` outside a transaction, and always bumps the counter so a concurrent session-scoped update against a stale snapshot fails its versioned filter and retries. -**TransactWriteItems** runs all operations inside a single MongoDB ACID transaction with snapshot read concern and majority write concern. Transaction failures are not retried — the error propagates as `TransactionCanceled`. - -Write conflict handling: `crates/storage-mongodb/src/data_engine.rs` — `update_item_impl()`, version field handling and retry loop. +**PutItem** with an existence guard on a new document maps duplicate-key errors (`E11000`) to `ConditionFailed` after re-reading the winner. This is the runtime signature of a conditional-put race the transaction snapshot didn't see. ### DynamoDB Streams -DynamoDB Streams are implemented using explicit stream record storage in MongoDB collections, not MongoDB's native Change Streams feature. The explicit approach was adopted to maintain behavioral parity with the PostgreSQL backend and to retain full application control over sequence number generation, shard assignment, and record retention lifecycle — all of which the DynamoDB Streams API contract tightly specifies. +DynamoDB Streams are implemented using explicit stream record storage in MongoDB collections, not MongoDB's native Change Streams feature. The explicit approach maintains behavioral parity with the PostgreSQL backend and retains full application control over sequence-number generation, shard assignment, and record retention — all of which the DynamoDB Streams API contract tightly specifies. -Each table is assigned 4 shards at creation time. On each data write with streams enabled, the backend assigns the write to a shard by hashing the partition key with CRC32, generates a monotonically increasing 21-digit sequence number using MongoDB's atomic `findOneAndUpdate` with `$inc`, and writes a record to `stream_records`. `GetRecords` paginates using `{ "sequence_number": { "$gt": after } }` range queries with ascending sort. +Each stream-enabled table is assigned 4 shards at creation time. Shard identifiers embed the table's globally-unique `table_id` UUID rather than the caller-visible `table_name` (`build_shard_id` in `stream_engine.rs`): `shardId-{table_id}-{i:012}`. Table names are only unique per-account, so a name-derived shard_id would let one account's `GetRecords` read another's records on same-named tables. `table_id` is per-instance, so a `DeleteTable + CreateTable` sequence produces fresh shard_ids; leftover stream records from the deleted table are cleaned up in `delete_table_impl` (`cleanup_stream_state_for_table`). A unique index on `stream_shards.shard_id` rules out duplicate insertions structurally. +On each data write with streams enabled, `write_stream_inline_in_session` runs inside the same session as the base write: -Stream implementation: `crates/storage-mongodb/src/stream_engine.rs`. Shard initialization: `init_stream_shards()`. Sequence number generation: `next_sequence_number()` using `$inc` on a counters document. Shard assignment by CRC32 hash: `assign_shard()`. Inline stream write from data operations: `crates/storage-mongodb/src/data_engine.rs` — `write_stream_inline()`. +1. Resolve the shard for the item's partition key by reading the table's shard set under the session (`assign_shard_in_session`) and hashing the pk with CRC32. +2. Draw the next sequence number by `$inc`-ing the per-shard counter at `_id: "stream_seq:"` in the `counters` collection — also under the session. +3. Insert the stream record into `stream_records`. -### Time to Live (TTL) +Per-shard counters preserve DynamoDB Streams' contract that sequence numbers are strictly monotonic within a shard and independent across shards. A single global counter would couple unrelated shards' sequence spaces. Session-scoped assignment closes an ordering hole: without it, a fast writer B can draw seq=6 and commit before a slow writer A (which drew seq=5) commits, and a consumer polling between B's commit and A's commit would advance past seq=6 and never see seq=5. With the counter increment inside the write transaction, two writers racing on the same shard conflict at commit time and the loser retries. -When TTL is enabled on a table, the backend creates a sparse MongoDB index on `item_data.{ttl_attribute}.N`. A background worker spawned at server startup sweeps expired items every 60 seconds in batches of 100. Each deletion uses `DataEngine::delete_item` with a condition expression re-checking expiry, preventing races. TTL deletions carry `UserIdentity { type: "Service", principalId: "dynamodb.amazonaws.com" }` on their stream records, matching DynamoDB's TTL stream record format. +Stream event names use DynamoDB wire casing (`INSERT`, `MODIFY`, `REMOVE`) via `event_name_ddb_str`. When `UpdateItem` creates an item that didn't exist (upsert case), the stream layer emits `INSERT`, not `MODIFY` with a fabricated key-only `OldImage`. +`GetRecords` paginates using `{ sequence_number: { $gt: after } }` range queries with ascending sort, backed by a compound index on `(shard_id, sequence_number)`. Retention is 24 hours: a TTL index on `stream_records.created_at` (24 h) drives primary enforcement; a background `stream_record_cleanup_worker` runs hourly as defense in depth. `UpdateTable` stream-enable is idempotent: if shards already exist for the table, it reuses them and preserves the existing `stream_label` rather than rotating it (which would invalidate ARNs previously handed out to consumers). `stream_label` uses `YYYY-MM-DDThh:mm:ss` (second precision, no timezone), byte-for-byte compatible with the PostgreSQL backend. -TTL index creation: `crates/storage-mongodb/src/metadata_engine.rs` — `create_ttl_index()`. Background worker: `crates/storage-mongodb/src/ttl_worker.rs` — `ttl_cleanup_worker()`, `sweep_expired_items()`. Worker spawn: `crates/storage-mongodb/src/lib.rs` — `MongoRuntimeHooks::spawn_workers()`. UserIdentity on TTL stream records: `crates/storage-mongodb/src/ttl_worker.rs` — `sweep_expired_items()`, `ttl_identity` construction. +The `StreamEngine::write_stream_record` trait method is not used on this backend; it returns an explicit error so a caller who invokes it doesn't get a subtly-wrong write outside any transaction session. -### Control plane state transitions +### Time to Live (TTL) + +When TTL is enabled on a table, the backend creates a sparse MongoDB index on `item_data.{ttl_attribute}.N` and marks `ttl_index_ready: true` on the table doc. A background TTL worker (spawned at server startup by `MongoRuntimeHooks::spawn_workers`) sweeps expired items every 60 seconds in batches of 100 per table. Each deletion goes through `DataEngine::delete_item` with a condition expression re-checking expiry, preventing races with concurrent writes. TTL deletions carry `UserIdentity { type: "Service", principalId: "dynamodb.amazonaws.com" }` on their stream records, matching DynamoDB's TTL stream record format. -Table creation and deletion are asynchronous at the DynamoDB API level — `CreateTable` returns `CREATING` status, `DeleteTable` returns `DELETING`. A background `WorkerStore` implementation polls the `tables` catalog collection for entries whose `status_transition_at` timestamp has passed and completes the transition: flipping CREATING → ACTIVE, or for DELETING → dropped (drops the data collection, index collections, removes catalog entries and tags). +TTL index creation: `metadata_engine.rs` — `create_ttl_index`. Background worker and stream/GSI companions: `ttl_worker.rs` — `ttl_cleanup_worker`, `stream_record_cleanup_worker`, `gsi_backfill_worker`. Worker spawn: `lib.rs` — `MongoRuntimeHooks::spawn_workers`. +### Control plane state transitions -Worker implementation: `crates/storage-mongodb/src/worker_store.rs` — `process_control_plane_transitions()`. +Table creation and deletion are asynchronous at the DynamoDB API level — `CreateTable` returns `CREATING` status, `DeleteTable` returns `DELETING`. A background `WorkerStore` implementation (`worker_store.rs`) polls the `tables` catalog collection for entries whose `status_transition_at` timestamp has passed and completes the transition: flipping CREATING → ACTIVE, or for DELETING → dropping the data collection, dropping every associated index collection, cleaning up stream shards + records + counters, and removing catalog entries and tags. ### Authentication and authorization -ExtendDB's mandatory SigV4 authentication is fully supported. Access key secrets are stored AES-GCM encrypted in the `extenddb_catalog.access_keys` collection. The encryption key is a 256-bit random key generated during `extenddb init`, base64-encoded, and stored in `extenddb_catalog.settings` under `_id: "encryption_key"`. Admin passwords are bcrypt-hashed before storage in `extenddb_catalog.admin_users`. +ExtendDB's mandatory SigV4 authentication is fully supported. Access-key secrets are stored AES-GCM encrypted in `extenddb_catalog.access_keys`. The encryption key is a 256-bit random key generated during `extenddb init`, base64-encoded, and stored in `extenddb_catalog.settings` under `_id: "encryption_key"`. Admin passwords are bcrypt-hashed before storage in `extenddb_catalog.admin_users`. -IAM policy evaluation fetches user-attached policies, group-attached policies (via `iam_group_members` → `iam_policies` join), role policies, permissions boundaries, and session policies from the catalog. +IAM policy evaluation fetches user-attached policies, group-attached policies (via `iam_groups.members` → `iam_policies` join), role policies, permissions boundaries, and session policies from the catalog. +`MongoEngine::new` rejects connection strings that specify a non-primary read preference (`secondary`, `secondaryPreferred`, `nearest`, `primaryPreferred`). DynamoDB's `ConsistentRead=true` requires linearizable reads; only MongoDB's Primary read preference provides that. A connection string that routes reads to a replica would silently return stale data — a fidelity violation the caller has no way to detect. The check fails at engine construction so misconfiguration surfaces at `extenddb serve` startup, not at request time. -Encryption key generation and storage: `crates/storage-mongodb/src/bootstrapper.rs` — `bootstrap_encryption_key()`. Admin password hashing: same file, `bootstrap_admin_user()`. Access key decryption at request time: `crates/storage-mongodb/src/credential_store.rs`. IAM policy fetching: `crates/storage-mongodb/src/authorization_store.rs` — `fetch_user_policies()`, `fetch_user_group_policies()`, `fetch_role_policies()`, `fetch_session_data()`. +Encryption key bootstrap: `bootstrapper.rs::bootstrap_encryption_key`. Admin password hashing: same file, `bootstrap_admin_user`. Access-key decryption: `credential_store.rs`. IAM policy fetching: `authorization_store.rs`. ### Backup -`CreateBackup` iterates the source table's collection and inserts each item into a shared `backup_items` collection in the `extenddb_catalog` database, tagged with the `backup_arn`. Backup metadata (arn, table, timestamps, status) is stored in `extenddb_catalog.backups`. `RestoreTableFromBackup` reads `backup_items` filtered by `backup_arn` and reconstructs the table. `DeleteBackup` removes the entries for that arn from the shared collection and marks the metadata row deleted. +`CreateBackup` snapshots the source table by running a server-side aggregation pipeline `[{ $out: "_backup_" }]` on the data collection. MongoDB copies items server-side without transferring them through the driver, and the destination is a per-backup collection in `extenddb_data` whose name derives from a UUID (never the caller-visible ARN, which contains characters MongoDB doesn't allow in collection names). Backup metadata (arn, backup_id, table, timestamps, key schema, table class, SSE, on-demand throughput, status) is stored in `extenddb_catalog.backups`. -**Alternative considered — server-side `$out` aggregation.** MongoDB's `$out` stage would copy a collection server-side without transferring data through the application. It was not adopted here because backup metadata (retention policies, tags, cross-collection references, account-scoped ARN lookups) does not compose with `$out`'s single-collection-target model, and because a shared `backup_items` collection avoids proliferating per-backup collection names in the WiredTiger file namespace at typical scale. A future revision could switch to `$out` for the copy phase while keeping the shared metadata schema. +`RestoreTableFromBackup` recreates the table via the normal CreateTable path (preserving TableClass, SSESpecification, OnDemandThroughput from the backup metadata) and clones the backup collection into the new data collection with the same `$out` stage. `DeleteBackup` drops the backup collection and marks the metadata row `DELETED`. - -Backup implementation: `crates/storage-mongodb/src/backup_engine.rs`. `backup_items` collection layout and `backup_arn` indexing are documented in the module header. +Backup implementation: `backup_engine.rs`. ### Operational requirements -**Minimum MongoDB version: 8.0.** This is the minimum supported version for this backend. The MongoDB Rust driver 3.x is technically compatible with MongoDB 4.2+, but this backend targets 8.0 as the minimum supported server version. +**Minimum MongoDB version: 6.0.** Required for multi-document ACID transactions and snapshot reads. The MongoDB Rust driver 3.x is technically compatible with earlier server versions; this backend targets 6.0 as the minimum supported. + +**Replica set required.** MongoDB must be configured as a replica set before running `extenddb init`. A standalone node does not support multi-document transactions. A single-node replica set is sufficient for development and CI; production deployments should use a 3-node replica set for high availability. -**Replica set required.** MongoDB must be configured as a replica set before running `extenddb init`. A standalone node does not support multi-document transactions (`TransactWriteItems`). A single-node replica set is sufficient for development and CI. Production deployments should use a 3-node replica set for high availability. +**Primary read preference.** Connection strings must use `readPreference=primary` (the driver default). Non-primary preferences are rejected at engine startup. **File descriptor limit.** Each MongoDB collection maps to one WiredTiger file. At 500 DynamoDB tables with 2 GSIs each (~1,500 collections), ensure `ulimit -n ≥ 65536` on the MongoDB host. See `docs/local-mongodb-setup.md` for platform-specific instructions. @@ -197,16 +213,6 @@ max_connections = 50 max_catalog_connections = 20 ``` -Initialization uses backend-selection flags on `extenddb init`: - -```text -extenddb init \ - --storage-backend mongodb \ - --storage-host 127.0.0.1 \ - --storage-port 27017 \ - --config extenddb.toml -``` - Configuration struct: `crates/storage-mongodb/src/config.rs`. Sample configuration: `extenddb.sample.toml` — `[storage.mongodb]` section. Setup guide: `docs/local-mongodb-setup.md`. ### Implementation summary @@ -216,69 +222,74 @@ Configuration struct: `crates/storage-mongodb/src/config.rs`. Sample configurati | `crates/storage-mongodb/` | New crate — full backend implementation | | `crates/bin/Cargo.toml` | Added `mongodb` optional feature flag | | `crates/bin/src/main.rs` | Added `#[cfg(feature = "mongodb")] extern crate` | -| `crates/bin/src/cmd_serve.rs` | Generalized the supported-backend gate from a hard-coded `"postgres"` check to a compile-time-conditional list built from the enabled feature flags (accepts `"mongodb"` when the feature is on) | -| `Cargo.toml` (workspace) | Added crate to members, added `mongodb`, `bson`, `dashmap` workspace dependencies | +| `crates/bin/src/cmd_serve.rs` | Generalized the supported-backend gate from a hard-coded `"postgres"` check to a compile-time list built from enabled features | +| `Cargo.toml` (workspace) | Added crate to members; added `mongodb`, `bson`, `dashmap` workspace dependencies | No changes to `crates/engine/`, `crates/server/`, `crates/storage/` (trait definitions), `crates/auth/`, or `crates/core/`. - -Full diff: `mongodb-forks/extenddb` branch `extenddb-on-mongo` compared to `main`. The absence of changes in engine/server/auth/core crates can be verified directly in that diff. - ### Design decisions summary | Decision | Choice | Rationale | |---|---|---| -| Conditional writes (PutItem, DeleteItem) | Read + evaluate + write within one MongoDB client session | Atomicity for the DynamoDB contract; loaded item is reused for `ReturnValuesOnConditionCheckFailure = ALL_OLD` without a follow-up read. Filter pushdown is planned as an optimization (see Condition expression evaluation). | -| UpdateItem write conflict | Optimistic concurrency with `_v` version field + jittered backoff | Avoids multi-document transactions for single-item updates while preventing lost updates | -| GSI updates | Synchronous inline within the base write's session, with `DashMap` cache short-circuit for tables with no GSIs | No Change Stream recovery complexity; GSI reads are strongly consistent | -| DynamoDB Streams | Inline writes to `stream_records` collection within the base write's session | Behavioral parity with PostgreSQL backend; explicit control over sequence numbers, shard assignment, and retention | -| Stream shards | 4 per table, CRC32 hash assignment | Predictable consumer parallelism; no catalog lookup at shard assignment time | -| Sort key numbers | Native BSON `Decimal128` | Correct ordering by value; no string-encoding tricks. Values exceeding Decimal128 precision (34 digits) are rejected — see `docs/differences-from-dynamodb.md`. | -| Backups | Per-item inserts to shared `backup_items` collection keyed by `backup_arn` | Composes cleanly with backup metadata (tags, retention, ARN-scoped restore/delete); avoids collection-name proliferation. Server-side `$out` documented as a future optimization. | -| Parallel scan | Application-side `crc32(pk) % segments` filter | Avoids per-document write overhead of a pre-bucketed segment field | +| Conditional writes | Read + evaluate + write inside a MongoDB transaction session | Snapshot atomicity gives DynamoDB's contract; loaded item reused for `ReturnValuesOnConditionCheckFailure = ALL_OLD` without a follow-up read. Analyzer-gated pushdown fast path skips the session for a certified subset on tables with no GSIs / streams. | +| UpdateItem concurrency | `_v` version guard inside snapshot txn + WriteConflict retry with jittered exponential backoff | Prevents lost updates; retry ceiling (50) bounds tail latency under sustained contention. | +| WriteConflict handling | Detect via `TransientTransactionError` label, `UnknownTransactionCommitResult` label, or raw code 112; retry with backoff | Converts a raw HTTP 500 into a retryable operation; TWI exhaustion surfaces as `TransactionCanceled` with per-op `TransactionConflict` reasons. | +| GSI updates | Synchronous inline within the base write's session, with 60-second TTL cache short-circuit for tables with no GSIs; async worker-driven backfill on UpdateTable | No Change Stream recovery; GSI reads are strongly consistent; UpdateTable matches DDB's async CREATING → ACTIVE contract. | +| GSI/LSI index docs | Composite `_id` includes both index and base keys; `base_pk` / `base_sk_?` stored as first-class fields | GSI keys are non-unique; base-key disambiguation prevents cross-item overwrite. Base keys as fields let index pagination form a compound cursor without traversing item_data. | +| Composite `_id` | Netstring-encoded (`:,...`) | Unambiguous boundary between pk and sk regardless of content. | +| Binary sort keys | Stored as lowercase hex strings | MongoDB's BSON Binary sort order diverges from DDB's unsigned-lex byte order across mismatched lengths. Hex-encoded strings preserve DDB order under default string comparison and make `begins_with` a plain range filter. | +| Sort key numbers | Native BSON `Decimal128` | Correct ordering by value. Values exceeding Decimal128's 34-digit precision are rejected. | +| DynamoDB Streams | Inline writes to `stream_records` inside the base write's session; per-shard sequence counters | Behavioral parity with PostgreSQL backend; sequence-number monotonicity within a shard is a contract. Session-scoped assignment prevents ordering holes under concurrent writes. | +| Stream shard ID | `shardId-{table_id}-{i:012}` | Table names are only account-unique; `table_id` UUID prevents cross-tenant shard address collisions. | +| Stream retention | TTL index on `stream_records.created_at` (24h) + hourly worker as defense in depth | Primary enforcement is at the storage layer; worker covers TTL-monitor lag or missing index. | +| Idempotency tokens | Unique compound index on `(account_id, token)` + 540s TTL + 600 ms data-plane age filter | Race safety under snapshot isolation; worst-case retention stays ≤10 min regardless of TTL-monitor cadence. | +| Backups | Per-backup collection via server-side `$out` aggregation | No per-item traffic between driver and server; backup metadata schema decouples from collection naming; ARN characters are unsafe as collection names. | +| Parallel scan | Application-side `crc32(pk) % segments` filter with lazy cursor iteration | Avoids per-document write overhead; lazy iteration prevents item-drops on hot-key skew that a hard server-side limit would cause. | +| Non-primary read preference | Rejected at engine startup | `ConsistentRead=true` requires linearizable reads; only Primary provides that. | ### Performance characteristics -**Single-item writes.** A conditional PutItem, DeleteItem, or UpdateItem is executed within a MongoDB client session that covers the condition read, the write, and any dependent writes (stream record insert, GSI collection updates). This adds one session start/commit pair over a raw driver call — ~sub-millisecond on a local replica set. The session wrap is what gives DynamoDB's atomicity contract on conditional writes; it is not overhead in the DynamoDB-compatibility sense, it is the compatibility. A sessionless fast path for the narrow case of tables with no streams and no GSIs is planned as a follow-up optimization. +**Single-item conditional writes.** One transaction session covers the pre-image read, condition evaluation, base write, GSI synchronization, and stream record insert. On a local replica set this adds ~sub-millisecond of session-start/commit overhead over a raw driver call. The session wrap is what gives DynamoDB's atomicity contract on conditional writes — it is the compatibility, not overhead. The analyzer-gated pushdown fast path collapses this to a single `find_one_and_*` call for the narrow case of certified pushable conditions on tables with no GSIs and no streams. + +**Unconditional single-item updates on GSI-free / stream-free tables.** A native-fast-path `find_one_and_update` runs outside any transaction. It always includes `$inc: {_v: 1}` so a concurrent slow-path update cannot pass its versioned filter against a stale snapshot. -**GSI write overhead.** For tables with no GSIs, the `gsi_cache` short-circuits to zero overhead — no catalog query, no additional I/O. For tables with GSIs, one catalog query fetches index definitions (cached for subsequent writes on the same table), plus one upsert or delete per index collection per write, all within the same session as the base write. +**GSI write overhead.** For tables with no GSIs, the `gsi_cache` short-circuits to zero overhead (no catalog query, no I/O) — refreshed at most once per `GSI_CACHE_TTL` window per table. For tables with GSIs, one catalog query fetches the index definitions (cached for subsequent writes) and one upsert or delete runs per index collection per write, all within the base write's session. -**Stream write overhead.** When streams are enabled, each write adds one atomic `findOneAndUpdate` counter increment and one document insert into `stream_records`, both within the base write's session. +**Stream write overhead.** When streams are enabled, each write adds one atomic per-shard counter `$inc` and one document insert into `stream_records`, both within the base write's session. -**Query and Scan.** Direct index lookups on `{ pk, sk_* }`. Performance characteristics match any indexed MongoDB query. Parallel scans scan the full collection once per segment (see Query and Scan). +**Query and Scan.** Direct index lookups on `(pk, sk_?)` for base tables; compound `(pk, sk_?, base_pk, base_sk_?)` lookups for index queries. `GetRecords` uses the compound `(shard_id, sequence_number)` index. -**TransactWriteItems.** Multi-collection ACID transaction with snapshot read concern. Uncommon in practice — most DynamoDB workloads are single-item operations. +**TransactWriteItems.** Multi-collection ACID transaction; up to 100 operations per the DDB spec. Uncommon in practice — most workloads are single-item operations. ### Testing Testing is organized in three layers. -**Unit tests** cover pure logic without a live MongoDB instance: condition expression compilation (`condition.rs`), document encoding and decoding (`data/mod.rs`), sort key ordering, and sequence number generation. The MongoDB client is mocked at this layer. +**Unit tests** cover pure logic without a live MongoDB instance: netstring composite `_id` encoding, hex sort-key ordering, condition filter compilation, pushdown-analyzer decisions, sequence-number formatting, stream shard-id derivation. Property tests (`crates/storage-mongodb/tests/pushdown_parity.rs`) exercise the parity between the pushdown compiler and the in-Rust `evaluate_condition` reference over randomly generated items and expressions. -**Integration tests** run against a single-node replica set in Docker (`mongod --replSet rs0`). They cover the full table lifecycle, all item operations (including condition expressions), query and scan pagination, transactions, TTL worker behavior, stream record writes, GSI propagation, backup and restore, and all catalog and IAM operations. These execute as `cargo test -p extenddb-storage-mongodb`. +**Integration tests** run against a single-node replica set (`mongod --replSet rs0`) covering the full table lifecycle, all item operations (conditional and unconditional), query and scan pagination (base and index), transactions, TTL worker behavior, stream record writes and consumer pagination, GSI propagation and async backfill, backup and restore, and all catalog and IAM operations. These execute as `cargo test -p extenddb-storage-mongodb`. **End-to-end tests** run the existing ExtendDB pytest suite (`tests/`) unchanged against a MongoDB-backed ExtendDB server. The pytest suite speaks the DynamoDB wire protocol and has no backend awareness — a passing run against MongoDB is equivalent to a passing run against PostgreSQL. This is the conformance test baseline required by RFC-0002. -The CI job spins up a single-node MongoDB 8.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. +CI spins up a single-node MongoDB 6.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. ## Drawbacks -**Replica set requirement.** MongoDB must be run as a replica set for `TransactWriteItems` support. This adds operational complexity for users who currently run standalone MongoDB. Users who run standalone MongoDB will receive a runtime error on transactional operations. This is documented in setup guides and is a MongoDB architectural constraint, not an ExtendDB limitation. +**Replica set requirement.** MongoDB must be run as a replica set for multi-document transactions. Users who run standalone MongoDB will receive a runtime error on transactional operations. This is a MongoDB architectural constraint, not an ExtendDB limitation, and is documented in setup guides. -**Time to Live expiration throughput at scale.** DynamoDB's Time to Live deletion must emit stream records with a specific service identity. MongoDB's native Time to Live index operates at the storage engine level with no awareness of ExtendDB's stream system, so this implementation uses an application-level background worker that owns the full deletion lifecycle — finding expired items, deleting them, and emitting correctly attributed stream records. The worker runs every 60 seconds and processes expired items in batches of 100 per table. This is sufficient for ExtendDB's target deployment contexts. At very high expiration rates — tables where a large number of items expire per minute continuously — the worker will fall behind and the backlog will grow. This is a known limitation at scale, not a correctness issue, as DynamoDB's own contract only guarantees expiration within 48 hours rather than immediately (see `docs/differences-from-dynamodb.md`, TTL deletion row). +**TTL throughput at scale.** DynamoDB TTL deletions must emit stream records with a specific service identity. MongoDB's native TTL indexes operate at the storage-engine level with no awareness of ExtendDB's stream system, so this implementation uses an application-level background worker that owns the full deletion lifecycle. The worker runs every 60 seconds and processes 100 expired items per table per pass. This is sufficient for ExtendDB's target deployment contexts. At very high sustained expiration rates the worker will fall behind, and the backlog will grow. This is a known scale limitation, not a correctness issue — DynamoDB's own contract only guarantees expiration within 48 hours, not immediately (see `docs/differences-from-dynamodb.md`, TTL row). ## Alternatives ### Use MongoDB Change Streams for DynamoDB Streams -MongoDB has a native change data capture feature (Change Streams) that could back DynamoDB Streams. This approach was not fully evaluated. The implementation instead adopted the explicit `stream_records` collection approach used by the PostgreSQL backend, which gives ExtendDB full control over sequence number generation, shard assignment, record retention, and iterator behavior — all of which the DynamoDB Streams API contract tightly specifies. Reviewers are invited to weigh in on whether a comparative evaluation of the Change Streams approach should be documented before acceptance. - +MongoDB has a native change-data-capture feature (Change Streams) that could back DynamoDB Streams. The implementation instead adopted the explicit `stream_records` collection approach used by the PostgreSQL backend, which gives ExtendDB full control over sequence-number generation, shard assignment, record retention, and iterator behavior — all of which the DynamoDB Streams API contract tightly specifies. Reviewers are invited to weigh in on whether a comparative evaluation of the Change Streams approach should be documented before acceptance. ## Prior art **MongoDB document model and DynamoDB.** MongoDB's flexible document model has been noted as a natural fit for DynamoDB-style workloads in multiple independent analyses. Amazon DocumentDB (MongoDB-compatible) demonstrates AWS's own recognition of this overlap. The key difference in this implementation is that ExtendDB provides the full DynamoDB API layer — clients using the AWS SDK do not need to know they are talking to MongoDB. -**Condition pushdown pattern.** Compiling application-level filter expressions into storage-native query operators is a well-established pattern in query engines (Apache Arrow DataFusion, Spark, Presto all implement predicate pushdown). The `condition.rs` compiler in this implementation applies the same principle at the storage backend level. +**Condition pushdown pattern.** Compiling application-level filter expressions into storage-native query operators is a well-established pattern in query engines (Apache Arrow DataFusion, Spark, Presto all implement predicate pushdown). The pushdown-analyzer / compiler split in this implementation applies the same principle at the storage backend level, with the analyzer serving as the correctness boundary between the two. --- From 21e7dcc0706664b5c98947afb3f743260c31cda3 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 18:41:16 -0700 Subject: [PATCH 49/83] chore(mongodb): remove crate-wide allow(unused) and dead code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-0003 §6.2 forbids blanket `#[allow(unused)]` suppressions because they conceal unimplemented trait methods and let latent bugs ship dormant. The crate had `#![allow(unused)]` at the top of `lib.rs`; removing it surfaced 19 real dead-code items. Cleanup: - Delete `MongoEngine::sync_indexes` (non-session variant). Every write path routes through `sync_indexes_in_session` since D-C2; the standalone variant was unused and independently missed the D-C2 through D-C6 fixes (it had the old wrong index-key filter, the `let _ = delete_one` swallow, and no compound-cursor support). Removing it eliminates a shadow implementation that could have been called by mistake. - Delete `data::index_collection_name` — dupe of `data_collection_name`. - Delete `MongoCatalogStore::client` accessor — nothing outside the struct used it. - Delete `MongoEngine.max_connections` field — set at construction and never read (mongo driver's `max_pool_size` already carries the value into the client options). - Delete an unused `use futures::TryStreamExt` inside `sync_indexes_in_session`; the method uses `cursor.next(session)` which comes from a different trait. - Fix the unused-var warning on `item_count` in `backup_engine.rs`. - Strip 14 unused imports via `cargo fix`. --- .../src/authorization_store.rs | 1 - crates/storage-mongodb/src/backup_engine.rs | 2 +- crates/storage-mongodb/src/catalog_store.rs | 4 - crates/storage-mongodb/src/data/mod.rs | 14 +-- crates/storage-mongodb/src/data_engine.rs | 111 +----------------- crates/storage-mongodb/src/lib.rs | 9 +- crates/storage-mongodb/src/table_engine.rs | 4 +- docs/rfcs/0000-mongodb-backend.md | 2 +- 8 files changed, 14 insertions(+), 133 deletions(-) diff --git a/crates/storage-mongodb/src/authorization_store.rs b/crates/storage-mongodb/src/authorization_store.rs index 412c6f65..6f712808 100644 --- a/crates/storage-mongodb/src/authorization_store.rs +++ b/crates/storage-mongodb/src/authorization_store.rs @@ -6,7 +6,6 @@ use futures::TryStreamExt; use futures::future::BoxFuture; use mongodb::bson::{self, Document, doc}; -use mongodb::options::FindOptions; use extenddb_storage::authorization_store::{AuthorizationStore, SessionData}; use extenddb_storage::management_store::{OpError, OpResult}; diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index 23de03c6..d9d987e3 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -95,7 +95,7 @@ impl BackupEngine for MongoEngine { .unwrap_or("PAY_PER_REQUEST") .to_owned(); let table_size = table_doc.get_i64("table_size_bytes").unwrap_or(0); - let item_count = table_doc.get_i64("item_count").unwrap_or(0); + let _item_count = table_doc.get_i64("item_count").unwrap_or(0); // Preserve TableClass / SSESpecification / OnDemandThroughput so // RestoreTableFromBackup can recreate the table with the same diff --git a/crates/storage-mongodb/src/catalog_store.rs b/crates/storage-mongodb/src/catalog_store.rs index 625cacb1..1fafc0dd 100644 --- a/crates/storage-mongodb/src/catalog_store.rs +++ b/crates/storage-mongodb/src/catalog_store.rs @@ -39,10 +39,6 @@ impl MongoCatalogStore { &self.catalog_db } - /// Get a reference to the `MongoDB` client. - pub(crate) fn client(&self) -> &mongodb::Client { - &self.client - } } // Implement CatalogStore supertrait diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index 31c5afdb..186c55f6 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -5,16 +5,17 @@ //! //! Contains document conversion, collection naming, and key extraction utilities. -use std::collections::BTreeMap; -use bson::{Bson, Document, doc}; +use bson::{Document, doc}; use extenddb_core::types::{ - AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, + AttributeDefinition, AttributeValue, Item, KeySchemaElement, ScalarAttributeType, }; +#[cfg(test)] +use extenddb_core::types::KeyType; use extenddb_storage::error::StorageError; use extenddb_storage::util::{ - composite_pk_to_text, encode_netstring_composite, pk_to_text, sk_info, + composite_pk_to_text, encode_netstring_composite, sk_info, }; /// Returns the `MongoDB` collection name for a `DynamoDB` table. @@ -34,11 +35,6 @@ pub fn composite_id(pk_text: &str, sk_text: &str) -> String { encode_netstring_composite(&[pk_text.to_owned(), sk_text.to_owned()]) } -/// Returns the `MongoDB` collection name for a secondary index. -pub fn index_collection_name(index_id: &str) -> String { - format!("_ddb_{index_id}") -} - /// Convert a `DynamoDB` Item to a `MongoDB` BSON document for storage. /// /// Document structure: `{ _id, pk, sk_s/sk_n/sk_b, item_data }` diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index e93f2560..df16cf84 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -5,13 +5,13 @@ use bson::{Document, doc}; use futures::future::BoxFuture; -use mongodb::options::{FindOneAndDeleteOptions, FindOneAndReplaceOptions, ReturnDocument}; +use mongodb::options::{FindOneAndReplaceOptions, ReturnDocument}; use extenddb_core::expression::{ self, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, UpdateAction, }; use extenddb_core::types::{ - AttributeValue, Item, KeySchemaElement, KeyType, ReturnValuesOnConditionCheckFailure, + AttributeValue, Item, KeySchemaElement, ReturnValuesOnConditionCheckFailure, ScalarAttributeType, StreamEventName, StreamRecord, StreamRecordData, TableKeyInfo, extract_key, item_size_bytes, }; @@ -20,7 +20,7 @@ use extenddb_storage::util::{ composite_pk_to_text, encode_netstring_composite, pk_to_text, sk_info, }; use extenddb_storage::{ - DataEngine, IdempotencyKey, ItemPairResult, QueryResult, StreamCapture, StreamEngine, + DataEngine, IdempotencyKey, ItemPairResult, QueryResult, StreamCapture, TransactGetOp, TransactWriteOp, }; @@ -32,7 +32,7 @@ use crate::data::{ }; use crate::pushdown::{Pushable, is_pushable}; -use extenddb_core::types::{AttributeDefinition, Projection, ProjectionType}; +use extenddb_core::types::{Projection, ProjectionType}; impl DataEngine for MongoEngine { fn put_item( @@ -1554,107 +1554,6 @@ impl MongoEngine { .map_err(|e| StorageError::Validation(e.to_string())) } - async fn sync_indexes( - &self, - key_info: &TableKeyInfo, - old_item: Option<&Item>, - new_item: Option<&Item>, - ) -> Result<(), StorageError> { - use futures::TryStreamExt; - - // Fast path: skip catalog query if we know this table has no GSIs. - // The cache entry is valid for GSI_CACHE_TTL, giving eventual - // convergence when a GSI is added on another ExtendDB instance. - if let Some(false) = self.gsi_cache_get_fresh(&key_info.table_id) { - return Ok(()); - } - - let indexes_coll = self.catalog_db.collection::("indexes"); - let mut cursor = indexes_coll - .find(doc! { "_id.table_id": &key_info.table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let mut found_any = false; - while let Some(idx_doc) = cursor - .try_next() - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - { - found_any = true; - let index_id = match idx_doc.get_str("index_id") { - Ok(id) => id.to_string(), - Err(_) => continue, - }; - let idx_key_schema: Vec = match idx_doc.get("key_schema") { - Some(ks) => bson::from_bson(ks.clone()).unwrap_or_default(), - None => continue, - }; - let projection: Projection = match idx_doc.get("projection") { - Some(p) => bson::from_bson(p.clone()).unwrap_or(Projection { - projection_type: ProjectionType::All, - non_key_attributes: None, - }), - None => Projection { - projection_type: ProjectionType::All, - non_key_attributes: None, - }, - }; - - let idx_coll_name = data_collection_name(&index_id); - let idx_coll = self.data_db.collection::(&idx_coll_name); - - // Delete old index entry. The filter must match on both the - // index-key AND the base-key components, because GSIs allow - // duplicate index-key values across base items. See D-C1 / - // RFC-0003 §2.1. - if let Some(old) = old_item - && item_has_index_keys(old, &idx_key_schema) - { - let projected_old = - project_item(old, &idx_key_schema, &key_info.key_schema, &projection); - let old_filter = index_entry_filter( - &projected_old, - &idx_key_schema, - &key_info.key_schema, - &key_info.attribute_definitions, - )?; - let _ = idx_coll.delete_one(old_filter).await; - } - - // Insert new index entry - if let Some(new) = new_item - && item_has_index_keys(new, &idx_key_schema) - { - let projected = - project_item(new, &idx_key_schema, &key_info.key_schema, &projection); - let idx_doc = index_document( - &projected, - &idx_key_schema, - &key_info.key_schema, - &key_info.attribute_definitions, - )?; - let filter = index_entry_filter( - &projected, - &idx_key_schema, - &key_info.key_schema, - &key_info.attribute_definitions, - )?; - let opts = mongodb::options::ReplaceOptions::builder() - .upsert(true) - .build(); - idx_coll - .replace_one(filter, idx_doc) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - } - - self.gsi_cache_set(&key_info.table_id, found_any); - Ok(()) - } - async fn sync_indexes_in_session( &self, key_info: &TableKeyInfo, @@ -1662,8 +1561,6 @@ impl MongoEngine { new_item: Option<&Item>, session: &mut mongodb::ClientSession, ) -> Result<(), StorageError> { - use futures::TryStreamExt; - if let Some(false) = self.gsi_cache_get_fresh(&key_info.table_id) { return Ok(()); } diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 2fb69d0e..0a4851fd 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -4,10 +4,7 @@ //! `MongoDB` storage backend for extenddb. //! //! Implements the storage traits from `extenddb-storage` using `MongoDB` -//! as the backing store. Phase 1 covers `TableEngine`, `DataEngine`, -//! Bootstrapper, and `StorageConfig` with condition filter pushdown. - -#![allow(unused)] +//! as the backing store. mod admin_store; mod authorization_store; @@ -36,7 +33,6 @@ pub use credential_store::MongoCredentialStore; use std::sync::Arc; use extenddb_storage::error::StorageError; -use futures::future::BoxFuture; // ============================================================================ // OperationsEngineRegistration @@ -122,7 +118,6 @@ inventory::submit! { // ServerComponentsRegistration // ============================================================================ -use extenddb_auth::BuiltinAuthProvider; use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; use extenddb_storage::server_components::{ BackendError, ServerComponents, ServerComponentsRegistration, @@ -240,7 +235,6 @@ pub struct MongoEngine { pub(crate) catalog_db: mongodb::Database, data_db: mongodb::Database, region: String, - max_connections: u32, /// Cache of `table_id` -> (`has_gsi`, insertion time). Avoids catalog /// queries on every write for tables with no GSIs. Entries older than /// [`GSI_CACHE_TTL`] are treated as misses and re-read from the catalog, @@ -294,7 +288,6 @@ impl MongoEngine { catalog_db, data_db, region: region.to_owned(), - max_connections, gsi_cache: dashmap::DashMap::new(), }) } diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 28bd4ce6..b7e3ee91 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -6,11 +6,11 @@ use bson::{Document, doc}; use futures::future::BoxFuture; use mongodb::IndexModel; -use mongodb::options::{Collation, CollationStrength, IndexOptions}; +use mongodb::options::{Collation, IndexOptions}; use extenddb_core::types::{ AttributeDefinition, BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, - DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeySchemaElement, KeyType, + DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeySchemaElement, ListTablesInput, ListTablesOutput, LsiDescription, OnDemandThroughput, ProvisionedThroughputDescription, ScalarAttributeType, SseDescription, SseType, TableDescription, TableKeyInfo, TableStatus, UpdateTableInput, diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index a1b5d7c5..a094ac1a 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -62,7 +62,7 @@ The backend uses two MongoDB databases: **`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named `_ddb_{index_id}`. Shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `counters` for per-shard sequence-number counters, `idempotency_tokens` for transaction deduplication, and one `_backup_{backup_id}` collection per user-created backup. -Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()`. Data-database setup (`idempotency_tokens`, `stream_shards`, `stream_records` and their indexes): `create_data_db()` in the same file. Collection naming: `data/mod.rs` — `data_collection_name()` and `index_collection_name()`. +Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()`. Data-database setup (`idempotency_tokens`, `stream_shards`, `stream_records` and their indexes): `create_data_db()` in the same file. Collection naming: `data/mod.rs` — `data_collection_name()`, shared between base-table and index collections. ### Document structure for DynamoDB items From 124c8e9e29e2802edf2c42be4ce1dc0749d188be Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 18:43:37 -0700 Subject: [PATCH 50/83] fix(mongodb): replace broken WorkerStore impl with an explicit no-op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous `WorkerStore::process_control_plane_transitions` body was dead code with two independent problems: 1. Its query filters (`{ "account_id": X, "table_name": Y, ... }`) read those fields at the top level of the tables catalog document, but the mongo catalog stores account_id/table_name inside `_id.account_id` / `_id.table_name`. The filter would never match a real row. 2. `MongoRuntimeHooks::spawn_workers` never spawns anything that calls this method, and the mongo backend never puts a table into a transient CREATING/DELETING state to begin with — `create_table_impl` writes `TableStatus: ACTIVE` synchronously, `delete_table_impl` runs collection/tag/stream cleanup inline. RFC-0003 §6.3 forbids dead-code impls that pretend to be operational. RFC-0003 §10.1 requires that any trait method needing periodic maintenance be spawned via `ServerRuntimeHooks::spawn_workers`. Either fix the impl and wire it up, or state clearly that the backend has no work to do. The latter is honest here — GSI create is the one control-plane operation that does need async work, and that lives in `ttl_worker::gsi_backfill_worker` on the `indexes` catalog, not the `tables` catalog. Replace the body with `Ok(Vec::new())`. Update RFC-206 and the design doc to describe control-plane transitions as inline, and call out the no-op nature of the WorkerStore trait method. --- crates/storage-mongodb/src/worker_store.rs | 153 ++++----------------- docs/design/13-storage-mongodb.md | 10 +- docs/rfcs/0000-mongodb-backend.md | 4 +- 3 files changed, 34 insertions(+), 133 deletions(-) diff --git a/crates/storage-mongodb/src/worker_store.rs b/crates/storage-mongodb/src/worker_store.rs index 6cb7d3a9..69fb163b 100644 --- a/crates/storage-mongodb/src/worker_store.rs +++ b/crates/storage-mongodb/src/worker_store.rs @@ -3,146 +3,41 @@ //! `WorkerStore` implementation for `MongoDB`. //! -//! Processes control-plane state transitions (CREATING → ACTIVE, DELETING → deleted) -//! as a background safety net for incomplete operations. +//! The MongoDB backend does not use transient control-plane states +//! (`CREATING`, `DELETING`) for tables: `create_table_impl` writes the +//! catalog row with `table_status: "ACTIVE"` synchronously and +//! `delete_table_impl` removes the row + collections in one call, so +//! there is never a table document waiting for a background transition. +//! Both paths run inline in the request handler because MongoDB's +//! collection create/drop is fast enough not to warrant asynchronous +//! promotion, and the alternative would require a background worker +//! whose only job is to catch up work the API call could have done +//! synchronously anyway. +//! +//! GSI create is the one control-plane operation that does need +//! async work — its background portion lives in +//! [`ttl_worker::gsi_backfill_worker`] rather than here because it +//! operates on the `indexes` catalog collection with a `CREATING` +//! index-status, not on the `tables` collection. +//! +//! The trait method returns an empty list so `WorkerStore` is +//! satisfied for the [`OperationsEngine`] supertrait bound without +//! introducing a background job that would only ever be a no-op. +//! +//! [`ttl_worker::gsi_backfill_worker`]: crate::ttl_worker::gsi_backfill_worker +//! [`OperationsEngine`]: extenddb_storage::OperationsEngine -use futures::TryStreamExt; use futures::future::BoxFuture; -use mongodb::bson::{Document, doc}; use extenddb_storage::WorkerStore; use extenddb_storage::error::StorageError; use crate::MongoEngine; -use crate::data::data_collection_name; impl WorkerStore for MongoEngine { fn process_control_plane_transitions( &self, ) -> BoxFuture<'_, Result, StorageError>> { - Box::pin(async move { - let mut transitions = Vec::new(); - - let tables_coll = self.catalog_db.collection::("tables"); - let now = mongodb::bson::DateTime::now(); - - // CREATING → ACTIVE: find tables stuck in CREATING whose transition time has passed - let creating_filter = doc! { - "table_status": "CREATING", - "status_transition_at": { "$lte": now }, - }; - let mut cursor = tables_coll - .find(creating_filter.clone()) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - while let Some(table_doc) = cursor - .try_next() - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - { - let table_name = table_doc - .get_str("table_name") - .unwrap_or_default() - .to_owned(); - let account_id = table_doc.get_str("account_id").unwrap_or_default(); - - tables_coll - .update_one( - doc! { - "account_id": account_id, - "table_name": &table_name, - "table_status": "CREATING", - }, - doc! { - "$set": { "table_status": "ACTIVE" }, - "$unset": { "status_transition_at": "" }, - }, - ) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - transitions.push((table_name, "CREATING → active")); - } - - // DELETING → deleted: find tables stuck in DELETING whose transition time has passed - let deleting_filter = doc! { - "table_status": "DELETING", - "status_transition_at": { "$lte": now }, - }; - let mut cursor = tables_coll - .find(deleting_filter.clone()) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - while let Some(table_doc) = cursor - .try_next() - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - { - let table_name = table_doc - .get_str("table_name") - .unwrap_or_default() - .to_owned(); - let account_id = table_doc - .get_str("account_id") - .unwrap_or_default() - .to_owned(); - let table_id = table_doc.get_str("table_id").unwrap_or_default().to_owned(); - let table_arn = table_doc - .get_str("table_arn") - .unwrap_or_default() - .to_owned(); - - // Drop the data collection - let coll_name = data_collection_name(&table_id); - let _ = self.data_db.collection::(&coll_name).drop().await; - - // Drop index collections - let indexes_coll = self.catalog_db.collection::("indexes"); - let mut idx_cursor = indexes_coll - .find(doc! { "_id.table_id": &table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - while let Some(idx_doc) = idx_cursor - .try_next() - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - { - if let Ok(index_id) = idx_doc.get_str("index_id") { - let idx_coll_name = data_collection_name(index_id); - let _ = self - .data_db - .collection::(&idx_coll_name) - .drop() - .await; - } - } - - // Delete index catalog entries - indexes_coll - .delete_many(doc! { "_id.table_id": &table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - // Delete tags for this resource - self.catalog_db - .collection::("tags") - .delete_many(doc! { "resource_arn": &table_arn }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - // Delete the table catalog entry - tables_coll - .delete_one(doc! { "account_id": &account_id, "table_name": &table_name }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - transitions.push((table_name, "DELETING → deleted")); - } - - Ok(transitions) - }) + Box::pin(async move { Ok(Vec::new()) }) } } diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index ab874fdd..c9428a7c 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -833,7 +833,7 @@ crates/storage-mongodb/ ├── credential_store.rs # Access-key lookup + AES-GCM decryption ├── catalog_store.rs # SettingsStore / DiagnosticsStore glue ├── admin_store.rs # Admin operations (currently thin) - └── worker_store.rs # Control-plane state transitions (CREATING/DELETING) + └── worker_store.rs # WorkerStore trait shim (no-op; supertrait requirement) ``` ## 7. `MongoEngine` Struct @@ -993,8 +993,12 @@ The backend implements every trait in `extenddb-storage`: - `Bootstrapper` — init, destroy, migrate, verify. Creates the catalog and data databases, seeds encryption key and admin user, applies index schema. -- `WorkerStore` — CREATING → ACTIVE / DELETING → dropped transitions, - polled every scan interval. +- `WorkerStore` — trait method is a no-op. `create_table_impl` + writes `TableStatus: ACTIVE` inline and `delete_table_impl` runs + the collection/tag/stream cleanup inline, so there is never a + transient state waiting for a background worker. Kept in the + impl surface only because `OperationsEngine` requires + `WorkerStore` as a supertrait. - `ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, `RateLimitStore` — the catalog trait surface. - `AuthorizationStore` — user/group/role/permissions-boundary/session diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index a094ac1a..1d865886 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -170,7 +170,9 @@ TTL index creation: `metadata_engine.rs` — `create_ttl_index`. Background work ### Control plane state transitions -Table creation and deletion are asynchronous at the DynamoDB API level — `CreateTable` returns `CREATING` status, `DeleteTable` returns `DELETING`. A background `WorkerStore` implementation (`worker_store.rs`) polls the `tables` catalog collection for entries whose `status_transition_at` timestamp has passed and completes the transition: flipping CREATING → ACTIVE, or for DELETING → dropping the data collection, dropping every associated index collection, cleaning up stream shards + records + counters, and removing catalog entries and tags. +Table creation and deletion run inline. `CreateTable` writes the catalog row and creates the data collection with its indexes before returning; the returned `TableDescription` carries `TableStatus: ACTIVE`. `DeleteTable` removes the catalog row, drops the data + index collections, deletes tags, and cleans up stream shards / records / counters via `cleanup_stream_state_for_table` (`table_engine.rs`), all before returning. MongoDB's create/drop is fast enough that there is no need to defer either operation to a background worker. + +GSI creation on `UpdateTable` is the one control-plane operation that does need asynchronous work — a background worker drains index rows in `CREATING` state, backfills the base collection, and flips the row to `ACTIVE`. See the Global and Local Secondary Indexes section for the state machine. ### Authentication and authorization From 796e2fa4ed4a1b9c096bce8712a662deb87f7c23 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 18:44:30 -0700 Subject: [PATCH 51/83] fix(mongodb): propagate stale-index delete_one error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sync_indexes_in_session` used `let _ = idx_coll.delete_one(...).await;` when removing an index entry whose base item's GSI-key attribute just changed or was removed. A transient error on that delete was silently discarded, leaving the stale index row live under the old GSI-key value while the new value was upserted. Subsequent Query requests kept returning the stale projection forever. Propagate the error. `sync_indexes_in_session` is called from the data-plane write path inside the same session as the base write, so a real error here rolls back the whole transaction — including the base-table replace/insert — and the client sees a retryable error rather than silent index divergence. RFC-0003 §2.2 (stale entry prevention) + §9.1 (no silent degradation). --- crates/storage-mongodb/src/data_engine.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index df16cf84..6ec12bd2 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1613,7 +1613,19 @@ impl MongoEngine { &key_info.key_schema, &key_info.attribute_definitions, )?; - let _ = idx_coll.delete_one(old_filter).session(&mut *session).await; + // Propagate the error rather than swallowing it — RFC-0003 + // §2.2 requires deleting the old entry when a write changes + // or removes a GSI key attribute, and RFC-0003 §9.1 forbids + // silent side-effect drops. A transient error here would + // leave the stale index row live under the old GSI-key + // value even though the base item no longer has it, and + // subsequent queries would return the stale projection + // forever. + idx_coll + .delete_one(old_filter) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; } if let Some(new) = new_item From 5b6f1735adc9f06fbf453ff7d1642d5efc347b83 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 18:45:57 -0700 Subject: [PATCH 52/83] fix(mongodb): make init_stream_shards idempotent on E11000 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two concurrent `UpdateTable(stream_enabled=true)` calls can both observe "no shards yet" for the same table_id (snapshot isolation lets each transaction see a state that predates the other's insert) and both proceed to `init_stream_shards`. Because `stream_shards.shard_id` carries a unique index — see `bootstrapper.rs` — one of the inserts hits E11000 and surfaces as `StorageError::Internal` → HTTP 500. RFC-0003 §10.3 requires repeated `UpdateTable` calls with the same specification to not corrupt state and either reject cleanly or no-op. Treat E11000 as a no-op here: the shard already exists with the same `(shard_id, table_id)` binding by construction (the shard_id is deterministic from the table_id), so the client's retry sees the expected state without a wire-visible error. Non-duplicate errors still propagate as `Internal`. --- crates/storage-mongodb/src/stream_engine.rs | 26 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index 554f6cc3..c5af8c21 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -71,15 +71,35 @@ impl MongoEngine { for i in 0..SHARDS_PER_STREAM { let shard_id = build_shard_id(table_id, i); let start_seq = format!("{:021}", 0); - shards_coll + let insert_res = shards_coll .insert_one(doc! { "shard_id": &shard_id, "table_id": table_id, "starting_sequence_number": &start_seq, "created_at": BsonDateTime::now(), }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .await; + if let Err(e) = insert_res { + // Treat a duplicate `shard_id` as idempotent no-op: two + // concurrent `UpdateTable(stream_enabled=true)` calls + // can both observe "no shards yet" under snapshot + // isolation and both reach this insert; the unique + // index on `stream_shards.shard_id` (`bootstrapper.rs`) + // means one of them lands E11000. RFC-0003 §10.3 + // requires repeated `UpdateTable` calls with the same + // specification to not corrupt state — swallowing + // this E11000 makes the redundant call a no-op rather + // than a wire-visible 500. + let is_dup = matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ); + if !is_dup { + return Err(StorageError::Internal(e.to_string())); + } + } } Ok(()) } From 1379d0a04a82493de9a732efc82c3d3b0a5d6732 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 18:49:13 -0700 Subject: [PATCH 53/83] fix(mongodb): sessionless fast paths for unconditional single-item writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-0003 §4.1 requires that concurrent unconditional single-item writes never surface a client-visible conflict — "two concurrent `PutItem` on the same key must both succeed (last-writer-wins)." The backend previously wrapped every PutItem/DeleteItem/UpdateItem in a snapshot MongoDB transaction with a 50-attempt WriteConflict retry loop. Under sustained contention the loop exhausts and surfaces `StorageError::Internal("too many concurrent write conflicts")` → HTTP 500, which DDB never emits. The txn wrapper is only necessary when a write has dependent side effects — conditional evaluation, stream capture, GSI sync — that must be atomic with the base write. When none of those apply, the storage engine's single-doc atomicity is sufficient on its own, and the txn wrapper is what *creates* the conflict it then has to retry through. Split each write into a sessionless fast path: - **PutItem** — when `condition.is_none() && stream.is_none() && gsi_cache_get_fresh == Some(false)`, run a plain `find_one_and_replace(upsert=true, ReturnDocument::Before)`. No session, no retry. Concurrent writers converge naturally at the WiredTiger level; no error is ever emitted for contention alone. - **DeleteItem** — same gate, plain `find_one_and_delete`. - **UpdateItem** — extend the existing native fast path to handle numeric ADD via an aggregation-pipeline update. `$toDecimal` parses the string-stored `.N` value server-side, `$add` applies the delta as Decimal128, `$toString` writes it back. 50 concurrent `ADD counter :one` calls now serialize inside MongoDB with a `_v` bump per apply — no OCC retry, no client- visible conflict. RFC-0003 §4.4. Introduces `NativeUpdate::{Doc, Pipeline}` so the update fast path can dispatch to either the operator-document form (simple $set/$unset) or the pipeline form (numeric ADD, or a mix). Session-scoped paths remain for: conditional writes (need read + check + write atomicity), streams-enabled tables (need base + stream-record atomicity), GSI-bearing tables (need base + index atomicity), and updates the pipeline can't express (list_append, if_not_exists, arithmetic on strings, DELETE from set). Those paths still retry on WriteConflict, but they only cover cases that RFC-0003 §4.1's exemption doesn't apply to. The `stream_shards` unique-index defense from the earlier `init_stream_shards` idempotency fix means that even the racy session-scoped path is safer than before. --- crates/storage-mongodb/src/data_engine.rs | 286 ++++++++++++++++++---- 1 file changed, 238 insertions(+), 48 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 6ec12bd2..c24c9987 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -9,6 +9,7 @@ use mongodb::options::{FindOneAndReplaceOptions, ReturnDocument}; use extenddb_core::expression::{ self, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, UpdateAction, + resolve_name_ref, }; use extenddb_core::types::{ AttributeValue, Item, KeySchemaElement, ReturnValuesOnConditionCheckFailure, @@ -258,6 +259,36 @@ impl MongoEngine { let key_filter = pk_filter(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + // Sessionless fast path for unconditional PutItem on a plain + // table (no cond, no stream, no GSI). DDB's contract is + // last-writer-wins with no client-visible conflict error + // (RFC-0003 §4.1). Wrapping this in a snapshot transaction + // would convert same-key contention into WriteConflict + // aborts that eventually surface as `Internal` — a wire- + // visible error DDB never emits. Rely on WiredTiger's + // single-document atomicity instead. Two concurrent writes + // serialize at the storage engine level; one wins the last- + // writer-wins race and the other's version is overwritten. + // No txn, no retry loop, no possible 500 from contention. + if condition.is_none() + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + { + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + let opts = FindOneAndReplaceOptions::builder() + .upsert(true) + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter, new_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let old_item = old_doc.as_ref().map(document_to_item).transpose()?; + return Ok(if return_old { old_item } else { None }); + } + let mut session = self .client .start_session() @@ -401,7 +432,10 @@ impl MongoEngine { } } - Err(StorageError::Internal( + // Retry ceiling exhausted. RFC-0003 §4.3 requires + // `TransactionConflictException` when a single-item write can't + // serialize against concurrent activity — never a bare 500. + Err(StorageError::TransactionConflict( "PutItem: too many concurrent write conflicts, giving up".to_owned(), )) } @@ -451,6 +485,22 @@ impl MongoEngine { let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + // Sessionless fast path for unconditional DeleteItem on a plain + // table (no cond, no stream, no GSI). Same rationale as + // put_item_impl — DDB never surfaces contention on unconditional + // single-item deletes. RFC-0003 §4.1. + if condition.is_none() + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + { + let old_doc = coll + .find_one_and_delete(key_filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let deleted_item = old_doc.as_ref().map(document_to_item).transpose()?; + return Ok(if return_old { deleted_item } else { None }); + } + let mut session = self .client .start_session() @@ -561,7 +611,7 @@ impl MongoEngine { } } - Err(StorageError::Internal( + Err(StorageError::TransactionConflict( "DeleteItem: too many concurrent write conflicts, giving up".to_owned(), )) } @@ -619,23 +669,65 @@ impl MongoEngine { && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) && let Some(mongo_update) = self.try_build_native_update(actions, maps) { - let opts = mongodb::options::FindOneAndUpdateOptions::builder() - .upsert(true) - .return_document(ReturnDocument::After) - .build(); - let result_doc = coll - .find_one_and_update(key_filter, mongo_update) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let new_item = if return_new { - result_doc.as_ref().map(document_to_item).transpose()? - } else { - None + let (fast_filter, took_fast) = match mongo_update { + NativeUpdate::Doc(d) => { + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(ReturnDocument::After) + .build(); + let result = coll + .find_one_and_update(key_filter.clone(), d) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + (Some(result), true) + } + NativeUpdate::Pipeline { + type_guard, + pipeline, + } => { + // Compose the key filter with the type guard. If + // the doc exists but fails the guard, findAndModify + // returns None and we fall through to the slow + // path (which raises ValidationException). + // Upsert is disabled here because a missing-doc + // "no match" is indistinguishable from a + // type-mismatch "no match"; the slow path handles + // both correctly. + let combined_filter = if let Some(guard) = type_guard { + doc! { "$and": [key_filter.clone(), guard] } + } else { + key_filter.clone() + }; + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(false) + .return_document(ReturnDocument::After) + .build(); + let result = coll + .find_one_and_update(combined_filter, pipeline) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if result.is_none() { + // Either the doc doesn't exist yet (need to + // upsert with proper type handling) or the + // guard rejected it. Fall through. + (None, false) + } else { + (Some(result), true) + } + } }; - return Ok((None, new_item)); + if took_fast { + let result_doc = fast_filter.and_then(|d| d); + let new_item = if return_new { + result_doc.as_ref().map(document_to_item).transpose()? + } else { + None + }; + return Ok((None, new_item)); + } } let mut session = self @@ -823,7 +915,7 @@ impl MongoEngine { } } - Err(StorageError::Internal( + Err(StorageError::TransactionConflict( "UpdateItem: too many concurrent write conflicts, giving up".to_owned(), )) } @@ -1367,14 +1459,35 @@ impl MongoEngine { // ── Native MongoDB Update (fast path) ───────────────────────────── + /// Try to express the update as a native MongoDB atomic update. + /// + /// Returns: + /// - `Some(NativeUpdate::Doc(...))` for a plain operator update + /// (`$set`/`$unset`/`$inc`), served by + /// `find_one_and_update(filter, doc)`. + /// - `Some(NativeUpdate::Pipeline(...))` for numeric `ADD`, which + /// requires an aggregation-pipeline update (`$set` with computed + /// expressions) to convert the string-stored `.N` value to a + /// `Decimal128`, add the delta, and convert back — all + /// server-side. + /// - `None` on anything else — set-typed `ADD`, `DELETE`, + /// `list_append`, `if_not_exists`, arithmetic, multi-component + /// paths. Those fall through to the session-scoped + /// read-modify-write path. + /// + /// The pipeline form is what makes RFC-0003 §4.4 achievable + /// without a numeric shadow field: 50+ concurrent + /// `UpdateItem ADD counter :one` calls all apply cumulatively + /// because mongo serializes doc-scoped write locks around the + /// pipeline's read + compute + write, no OCC retry needed. fn try_build_native_update( &self, actions: &[UpdateAction], maps: &ExpressionMaps, - ) -> Option { - let mut inc_doc = Document::new(); + ) -> Option { let mut set_doc = Document::new(); let mut unset_doc = Document::new(); + let mut num_adds: Vec<(String, String)> = Vec::new(); for action in actions { match action { @@ -1382,50 +1495,45 @@ impl MongoEngine { if path.len() != 1 { return None; } - let attr_name = match &path[0] { + let raw_name = match &path[0] { PathElement::Attribute(name) => name, _ => return None, }; + let attr_name = resolve_name_ref(raw_name, maps).ok()?.into_owned(); let val = match value { Expr::Placeholder(name) => maps.resolve_value(name).ok()?, _ => return None, }; match val { AttributeValue::N(n) => { - let field = format!("item_data.{attr_name}.N"); - // Store numeric increment as string (matching our storage format) - // Use $inc on a helper field and reconcile, OR use a different approach. - // Actually: item_data stores N as string. We can't $inc a string. - // We need a numeric shadow field for $inc to work. - // For now, only optimize if we can parse as i64. - if let Ok(i) = n.parse::() { - // Use $inc on a numeric shadow field, then $set the string representation. - // Actually this won't work atomically in one update... - // The simplest correct approach: use $inc on item_data.attr.N - // BUT item_data.attr.N is stored as a string, not a number. - // MongoDB $inc doesn't work on strings. - // FALLBACK: we cannot use the native fast path for numeric ADD - // unless we change the storage format. Give up. - let _ = (field, i); + // Validate the delta parses as Decimal128 + // up-front so a bad number fails fast + // rather than mid-pipeline on mongo. + if n.parse::().is_err() { return None; } - return None; + num_adds.push((attr_name, n.clone())); } AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_) => { - // Set ADD — could use $addToSet but storage format is complex + // Set ADD — $addToSet would work in theory + // but our .SS/.NS/.BS storage keeps the + // values inside item_data..SS as an + // array. Not urgent enough to expand yet. return None; } _ => return None, } } + UpdateAction::Delete { .. } => return None, UpdateAction::Set { path, value } => { if path.len() != 1 { return None; } - let attr_name = match &path[0] { + let raw_name = match &path[0] { PathElement::Attribute(name) => name, _ => return None, }; + let attr_name = resolve_name_ref(raw_name, maps).ok()?; let val = match value { Expr::Placeholder(name) => maps.resolve_value(name).ok()?, _ => return None, // complex expressions (if_not_exists, list_append, arithmetic) @@ -1439,19 +1547,84 @@ impl MongoEngine { if path.len() != 1 { return None; } - let attr_name = match &path[0] { + let raw_name = match &path[0] { PathElement::Attribute(name) => name, _ => return None, }; + let attr_name = resolve_name_ref(raw_name, maps).ok()?; let field = format!("item_data.{attr_name}"); unset_doc.insert(field, 1); } - UpdateAction::Delete { .. } => { - return None; - } } } + if set_doc.is_empty() && unset_doc.is_empty() && num_adds.is_empty() { + return None; + } + + if !num_adds.is_empty() { + // Aggregation-pipeline stage. `$set` accepts computed + // expressions here (unlike an operator update's `$set`). + // Each numeric ADD is ` = toString(toDecimal(field + // or 0) + delta)`; `$unset` is expressed as ` = + // "$$REMOVE"`; SET actions are literal assignments. `_v` + // is bumped in the same stage. + let mut stage: Document = Document::new(); + for (k, v) in &set_doc { + stage.insert(k, v.clone()); + } + for k in unset_doc.keys() { + stage.insert(k, "$$REMOVE"); + } + // Guard: the fast path never reads the pre-image, so we + // can't detect an existing non-numeric attribute (e.g. + // ADD to a string). Require every ADD target to be + // absent or already hold `.N` — else return no match and + // let the caller fall back to the slow path, which reads + // the pre-image and returns a proper ValidationException. + let mut guard_clauses: Vec = Vec::with_capacity(num_adds.len()); + for (attr, delta_s) in &num_adds { + let field = format!("item_data.{attr}.N"); + let field_ref = format!("${field}"); + let attr_path = format!("item_data.{attr}"); + let delta_dec = delta_s + .parse::() + .expect("validated above"); + stage.insert( + &field, + doc! { + "$toString": { + "$add": [ + { "$toDecimal": { "$ifNull": [ &field_ref, "0" ] } }, + { "$toDecimal": bson::Bson::Decimal128(delta_dec) }, + ] + } + }, + ); + guard_clauses.push(doc! { + "$or": [ + { &attr_path: { "$exists": false } }, + { &field: { "$exists": true } }, + ] + }); + } + stage.insert( + "_v", + doc! { "$add": [ { "$ifNull": [ "$_v", 0_i64 ] }, 1_i64 ] }, + ); + let type_guard = if guard_clauses.is_empty() { + None + } else if guard_clauses.len() == 1 { + Some(guard_clauses.into_iter().next().unwrap()) + } else { + Some(doc! { "$and": guard_clauses }) + }; + return Some(NativeUpdate::Pipeline { + type_guard, + pipeline: vec![doc! { "$set": stage }], + }); + } + let mut update = Document::new(); if !set_doc.is_empty() { update.insert("$set", set_doc); @@ -1460,19 +1633,16 @@ impl MongoEngine { update.insert("$unset", unset_doc); } - if update.is_empty() && inc_doc.is_empty() { - return None; - } - // Bump `_v` on every native fast-path write. Without this a // fast-path commit leaves `_v` at its previous value, and a // slow-path update running concurrently against that same // stale value can pass its versioned-filter guard and // overwrite the fast-path write (lost update, RFC-0003 §4.4). + let mut inc_doc = Document::new(); inc_doc.insert("_v", 1_i64); update.insert("$inc", inc_doc); - Some(update) + Some(NativeUpdate::Doc(update)) } // ── GSI Sync ────────────────────────────────────────────────────── @@ -2801,6 +2971,26 @@ pub(crate) struct GsiBackfillProgress { /// in this file. const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; +/// Return type of `try_build_native_update`. Distinguishes an +/// operator-document update (`{$set, $unset, $inc}`) from an +/// aggregation-pipeline update (needed for numeric ADD, which +/// converts a string-stored `.N` value to Decimal128, applies the +/// delta, and converts back). +/// +/// `Pipeline` carries an optional `type_guard` filter — for numeric +/// ADD we require the target attribute to be absent or already an +/// `.N` so we don't clobber a string with a number. When the guard +/// rejects the match, `find_one_and_update` returns `None`, and the +/// caller falls back to the slow (session-scoped) path which reads +/// the pre-image and surfaces a proper `ValidationException`. +enum NativeUpdate { + Doc(Document), + Pipeline { + type_guard: Option, + pipeline: Vec, + }, +} + /// Error signal used inside per-attempt transaction bodies. Lets the /// body use `?` for control flow while distinguishing "retry this /// whole transaction" from "return this error to the caller." From ae831fa47bdec31fa096c8af31424150fe5d5119 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 19:06:44 -0700 Subject: [PATCH 54/83] =?UTF-8?q?test(mongodb):=20RFC-0003=20=C2=A74=20con?= =?UTF-8?q?currency=20conformance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a pytest module that asserts strict conformance against the four RFC-0003 §4.x scenarios. Client uses `retries={"max_attempts": 0}` via the shared conftest fixture so any InternalServerError from the backend surfaces immediately instead of being masked by the SDK. Cases covered: - §4.1 50 concurrent unconditional PutItem on the same key — all must succeed, no client-visible errors. - §4.1 50 concurrent conditional PutItem with attribute_not_exists — exactly one wins, the rest fail with ConditionalCheckFailedException. No InternalServerError. - §4.4 50 threads × 20 iterations of `UpdateItem ADD counter :one` on the same key — every increment applies, final counter equals 1000, no lost updates, no client-visible errors. - §4.1 50 concurrent unconditional DeleteItem on the same key — all succeed, item ends up absent. These are the specific scenarios the sessionless fast path fix targets. Prior to the fix, `TestRfc0003UnconditionalPutOnHotKey` and `TestRfc0003AtomicCounterAdd` would surface InternalServerError after the 50-attempt WriteConflict retry ceiling exhausted. --- tests/python/test_rfc0003_concurrency.py | 244 +++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 tests/python/test_rfc0003_concurrency.py diff --git a/tests/python/test_rfc0003_concurrency.py b/tests/python/test_rfc0003_concurrency.py new file mode 100644 index 00000000..d8e3c455 --- /dev/null +++ b/tests/python/test_rfc0003_concurrency.py @@ -0,0 +1,244 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""RFC-0003 §4 concurrency conformance — strict, no client-side retries. + +Every scenario asserts that the backend surfaces the correct DDB error +class (or no error at all) even under sustained contention. If the +backend produces `InternalServerError` under any of these workloads, +the test fails — DDB is not permitted to surface internal concurrency- +control mechanisms as client errors, and neither is a conformant +backend. + +The RFC-0003 stress-test scenarios covered here: + +- §4.1 Two concurrent unconditional `PutItem` on the same key must + both succeed (last-writer-wins). +- §4.1 Concurrent conditional `PutItem` with `attribute_not_exists`: + exactly one succeeds, the rest fail with + `ConditionalCheckFailedException` — nothing internal. +- §4.4 Concurrent `UpdateItem ADD counter :one` on the same item must + all succeed; the final counter value must equal the total + increments applied (no lost updates, no internal errors). +- §4.1 Concurrent unconditional `DeleteItem` on the same key must all + return without error (last-writer-wins semantics for delete). + +These tests deliberately use a boto3 client with `retries={"max_attempts": 0}` +so any InternalServerError surfaces immediately instead of being masked +by the SDK's retry policy. +""" + +from __future__ import annotations + +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed + +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + + +NUM_THREADS = 50 +INCREMENTS_PER_THREAD = 20 # 50 * 20 = 1_000 increments + + +@pytest.fixture() +def counter_table(dynamodb_client): + """A plain HASH-keyed table for the RFC-0003 §4.x scenarios.""" + name = unique_name("rfc4x") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +def _classify(exc: ClientError) -> str: + """Return the ClientError's DynamoDB error code.""" + return exc.response.get("Error", {}).get("Code", "") + + +class TestRfc0003UnconditionalPutOnHotKey: + """RFC-0003 §4.1 — concurrent unconditional PutItem on the same key. + + Both must succeed; DDB never surfaces contention as a client error + for unconditional writes. This is the scenario the pre-Phase-6 + backend violated by returning `InternalServerError` after 50 retry + attempts of a snapshot-txn WriteConflict loop. + """ + + def test_all_writers_succeed(self, dynamodb_client, counter_table): + key = f"hot-{uuid.uuid4().hex[:8]}" + errors: list[str] = [] + + def _write(thread_id: int) -> None: + try: + dynamodb_client.put_item( + TableName=counter_table, + Item={ + "pk": {"S": key}, + "writer": {"N": str(thread_id)}, + }, + ) + except ClientError as e: + errors.append(_classify(e)) + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_write, tid) for tid in range(NUM_THREADS)] + for f in as_completed(futs): + f.result() + + # DDB contract: all writes succeed. Any error is a conformance failure. + assert errors == [], f"unexpected errors: {errors}" + + # And the item exists with *some* writer's value — last-writer-wins, + # so we don't assert which one, just that the item is there. + resp = dynamodb_client.get_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ConsistentRead=True, + ) + assert "Item" in resp + + +class TestRfc0003ConditionalPutOnHotKey: + """RFC-0003 §4.1 — concurrent conditional PutItem with attribute_not_exists. + + Exactly one writer wins (item created). Everyone else must fail with + `ConditionalCheckFailedException`, not `InternalServerError`. + """ + + def test_one_winner_rest_ccf(self, dynamodb_client, counter_table): + key = f"race-{uuid.uuid4().hex[:8]}" + outcomes: list[tuple[str, str]] = [] # (result, error_code) + + def _conditional_put(thread_id: int) -> None: + try: + dynamodb_client.put_item( + TableName=counter_table, + Item={ + "pk": {"S": key}, + "winner": {"N": str(thread_id)}, + }, + ConditionExpression="attribute_not_exists(pk)", + ) + outcomes.append(("ok", "")) + except ClientError as e: + outcomes.append(("err", _classify(e))) + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_conditional_put, tid) for tid in range(NUM_THREADS)] + for f in as_completed(futs): + f.result() + + winners = [o for o in outcomes if o[0] == "ok"] + losers = [o for o in outcomes if o[0] == "err"] + + assert len(winners) == 1, f"expected exactly one winner, got {len(winners)}" + assert len(losers) == NUM_THREADS - 1 + + # All losers must have ConditionalCheckFailedException — nothing + # else. Any InternalServerError is a conformance failure. + for _, code in losers: + assert code == "ConditionalCheckFailedException", ( + f"loser returned {code!r} instead of ConditionalCheckFailedException" + ) + + +class TestRfc0003AtomicCounterAdd: + """RFC-0003 §4.4 — concurrent `UpdateItem ADD counter :one`. + + Every increment must apply cumulatively; the final counter equals + NUM_THREADS * INCREMENTS_PER_THREAD. Every UpdateItem call must + succeed — no InternalServerError, no retries at the client. + + The mongo backend uses an aggregation-pipeline update + (`$toString` of `$add` of `$toDecimal`) so 50+ concurrent ADD + calls converge at MongoDB's doc-lock level without OCC retries. + """ + + def test_all_increments_apply(self, dynamodb_client, counter_table): + key = f"counter-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=counter_table, + Item={"pk": {"S": key}, "counter": {"N": "0"}}, + ) + errors: list[str] = [] + + def _increment(thread_id: int) -> int: + done = 0 + for _ in range(INCREMENTS_PER_THREAD): + try: + dynamodb_client.update_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + UpdateExpression="ADD #c :one", + ExpressionAttributeNames={"#c": "counter"}, + ExpressionAttributeValues={":one": {"N": "1"}}, + ) + done += 1 + except ClientError as e: + errors.append(_classify(e)) + return done + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_increment, tid) for tid in range(NUM_THREADS)] + total_done = sum(f.result() for f in as_completed(futs)) + + assert errors == [], f"unexpected errors: {errors}" + assert total_done == NUM_THREADS * INCREMENTS_PER_THREAD + + # The counter must equal every increment applied. No lost updates. + resp = dynamodb_client.get_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ConsistentRead=True, + ) + final = int(resp["Item"]["counter"]["N"]) + assert final == NUM_THREADS * INCREMENTS_PER_THREAD + + +class TestRfc0003UnconditionalDeleteOnHotKey: + """RFC-0003 §4.1 — concurrent unconditional DeleteItem on the same key. + + All succeed. If the item exists, one delete removes it and the rest + are no-ops; if it doesn't, all are no-ops. Never an error. + """ + + def test_all_deletes_succeed(self, dynamodb_client, counter_table): + key = f"delkey-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=counter_table, + Item={"pk": {"S": key}, "val": {"S": "seed"}}, + ) + errors: list[str] = [] + + def _delete(_thread_id: int) -> None: + try: + dynamodb_client.delete_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ) + except ClientError as e: + errors.append(_classify(e)) + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_delete, tid) for tid in range(NUM_THREADS)] + for f in as_completed(futs): + f.result() + + assert errors == [], f"unexpected errors: {errors}" + + # Item must be gone. + resp = dynamodb_client.get_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ConsistentRead=True, + ) + assert "Item" not in resp From 42c58f85926227de628c16f6a431aea3558229af Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 21 Jul 2026 19:10:17 -0700 Subject: [PATCH 55/83] fix(mongodb): surface single-item retry exhaustion as TransactionConflict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-0003 §4.3: when a single-item write conflicts with an in-flight `TransactWriteItems` on the same item, the backend must return `TransactionConflictException` — never `InternalServerError`. The backend may retry transient conflicts internally, but must not exhaust retries and surface an unmapped error. The mongo backend's put/delete/update paths exhaust their 50-retry WriteConflict loop and returned `StorageError::Internal(...)` → HTTP 500. That's the exact case §4.3 forbids. Two-part fix: 1. Add `StorageError::TransactionConflict(String)` to the shared storage trait surface and map it in `engine/src/create_table.rs::storage_err_to_dynamo` to `DynamoDbError::TransactionConflictException`. The variant is general — any backend can emit it when a contention path exhausts internal retry. 2. Mongo `put_item_impl` / `delete_item_impl` / `update_item_impl` now return `StorageError::TransactionConflict` on ceiling exhaustion instead of `StorageError::Internal`. With the Phase 6 sessionless fast paths, ceiling-exhaustion is only reachable on the session-scoped path — i.e. writes on GSI-bearing or streams-enabled tables — where §4.3's applicability is exact. --- crates/engine/src/create_table.rs | 7 +++++++ crates/storage/src/error.rs | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index b6f18f7b..c9d00179 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -132,6 +132,13 @@ pub(crate) fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> tracing::error!("Unexpected idempotency error in generic error handler"); DynamoDbError::InternalServerError("Internal server error".to_owned()) } + StorageError::TransactionConflict(msg) => { + // Single-item write raced an in-flight TransactWriteItems on + // the same item and the backend couldn't serialize them + // through internal retries. RFC-0003 §4.3 requires + // TransactionConflictException here — never InternalServerError. + DynamoDbError::TransactionConflictException(msg) + } StorageError::Internal(msg) => { // Log the raw message for debugging but do not expose storage // backend details (e.g. PostgreSQL error text) to the client. diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index bf851968..a2043460 100755 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -24,6 +24,13 @@ pub enum StorageError { IdempotentReplay, #[error("Idempotent parameter mismatch")] IdempotentMismatch, + /// A single-item write raced an in-flight `TransactWriteItems` on + /// the same item, and the backend was unable to serialize the two. + /// Maps to `DynamoDbError::TransactionConflictException` at the + /// engine boundary — DynamoDB's canonical error for this case + /// (RFC-0003 §4.3). + #[error("Transaction conflict: {0}")] + TransactionConflict(String), #[error("No-op update: {0}")] NoOpUpdate(String), #[error("Validation error: {0}")] From 8c8636a4761b0fe8532ce44cb460a93642f9f250 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 23:39:45 -0700 Subject: [PATCH 56/83] fix(mongodb): zeroize encryption key in MongoCredentialStore on drop Derive Zeroize and ZeroizeOnDrop so the base64-encoded AES-256-GCM master encryption key is scrubbed from memory when the credential store is dropped. Matches the DbCredentialStore in the postgres backend. --- crates/storage-mongodb/src/credential_store.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/storage-mongodb/src/credential_store.rs b/crates/storage-mongodb/src/credential_store.rs index e0aedfcf..5aa5ff14 100644 --- a/crates/storage-mongodb/src/credential_store.rs +++ b/crates/storage-mongodb/src/credential_store.rs @@ -4,12 +4,17 @@ //! Credential store implementation for `MongoDB`. use mongodb::bson::{Document, doc}; +use zeroize::{Zeroize, ZeroizeOnDrop}; use extenddb_auth::{CredentialStore, StoredCredential}; use extenddb_core::error::DynamoDbError; /// `MongoDB` credential store for authentication. +/// +/// The `encryption_key` is zeroed from memory on drop. +#[derive(Zeroize, ZeroizeOnDrop)] pub struct MongoCredentialStore { + #[zeroize(skip)] client: mongodb::Client, encryption_key: String, } From f6140fafc36df2903e79fa5bbd476e3febfb6fd0 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 23:40:29 -0700 Subject: [PATCH 57/83] fix(mongodb): warn at startup when MongoDB connection is not using TLS Inspect the parsed ClientOptions in MongoEngine::new and emit a WARN log when the URI does not enable TLS. Without this warning, an operator who configures a bare mongodb://host:27017 URI has no indication that credentials and data are traversing the network in cleartext. --- crates/storage-mongodb/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 0a4851fd..139133fe 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -277,6 +277,14 @@ impl MongoEngine { } } + if !matches!(options.tls, Some(mongodb::options::Tls::Enabled(_))) { + tracing::warn!( + "MongoDB connection is not using TLS; credentials and data will \ + traverse the network in cleartext. Enable TLS with `?tls=true` \ + in the connection string, or use a `mongodb+srv://` URI." + ); + } + let client = mongodb::Client::with_options(options) .map_err(|e| StorageError::Connection(e.to_string()))?; From 9aca12bbf679491da58ac86a681ed1c0fabe02cf Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 23:43:35 -0700 Subject: [PATCH 58/83] fix(mongodb): redact userinfo password before persisting connection string MongoBootstrapper::record_data_connection previously wrote the raw connection string into the settings collection under data_connection_string. When the URI carries a userinfo password (mongodb://user:pass@host/...), the plaintext password sat at rest in the catalog and was readable by anyone with read access to the extenddb_catalog database. Add redact_connection_string, which replaces the password component of the userinfo with ``, and apply it before the upsert. The scheme, username, host, port, and query string are preserved so the stored value remains a useful reference. Unit tests cover the standard mongodb:// scheme, mongodb+srv://, bare URIs with no userinfo, username-only URIs, `@` characters that appear only in the query string (authSource=admin), and inputs without a URI scheme. --- crates/storage-mongodb/src/bootstrapper.rs | 79 +++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index 128edab9..21196e3f 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -391,10 +391,11 @@ impl Bootstrapper for MongoBootstrapper { .await .map_err(|e| OpError::Internal(format!("record_data_connection: {e}")))?; + let redacted = redact_connection_string(&self.connection_string); settings .update_one( doc! { "_id": "data_connection_string" }, - doc! { "$set": { "value": &self.connection_string } }, + doc! { "$set": { "value": &redacted } }, ) .upsert(true) .await @@ -626,3 +627,79 @@ connection_string = "{}" ) } } + +/// Replace `user:password` userinfo with `user:` in a MongoDB URI. +/// +/// Returns the original string unchanged if no `@` separator is present or if +/// the userinfo section contains no `:` (username-only, no password). +fn redact_connection_string(uri: &str) -> String { + let Some(scheme_end) = uri.find("://") else { + return uri.to_string(); + }; + let after_scheme = scheme_end + 3; + let Some(at_offset) = uri[after_scheme..].find('@') else { + return uri.to_string(); + }; + let at_idx = after_scheme + at_offset; + let userinfo = &uri[after_scheme..at_idx]; + let Some(colon_offset) = userinfo.find(':') else { + return uri.to_string(); + }; + let user = &userinfo[..colon_offset]; + format!( + "{}{}:{}", + &uri[..after_scheme], + user, + &uri[at_idx..] + ) +} + +#[cfg(test)] +mod tests { + use super::redact_connection_string; + + #[test] + fn redact_userinfo_with_password() { + assert_eq!( + redact_connection_string("mongodb://alice:secret@host:27017/?replicaSet=rs0"), + "mongodb://alice:@host:27017/?replicaSet=rs0" + ); + } + + #[test] + fn redact_srv_scheme() { + assert_eq!( + redact_connection_string("mongodb+srv://alice:p%40ss@cluster.example.com/?tls=true"), + "mongodb+srv://alice:@cluster.example.com/?tls=true" + ); + } + + #[test] + fn no_userinfo_untouched() { + assert_eq!( + redact_connection_string("mongodb://localhost:27017/?replicaSet=rs0"), + "mongodb://localhost:27017/?replicaSet=rs0" + ); + } + + #[test] + fn user_only_untouched() { + assert_eq!( + redact_connection_string("mongodb://alice@host:27017/"), + "mongodb://alice@host:27017/" + ); + } + + #[test] + fn at_in_query_is_not_userinfo() { + assert_eq!( + redact_connection_string("mongodb://host:27017/?authSource=admin"), + "mongodb://host:27017/?authSource=admin" + ); + } + + #[test] + fn no_scheme_untouched() { + assert_eq!(redact_connection_string("not a uri"), "not a uri"); + } +} From a929b9fe5cb1b7011b3249cfe128ae084776a543 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 16 Jul 2026 23:51:44 -0700 Subject: [PATCH 59/83] fix(mongodb): fail closed when access key is_active flag is missing MongoCredentialStore::lookup_user_credential read the is_active field with unwrap_or(true), so a record whose is_active field was missing, absent, or of the wrong BSON type was treated as active. Combined with a partial write during key rotation or a schema mismatch after a migration, this could authenticate a credential that should have been inactive. Change the default to false: if the flag cannot be read as a bool, the credential is rejected. Correct records with is_active: true continue to authenticate normally. --- crates/storage-mongodb/src/credential_store.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/credential_store.rs b/crates/storage-mongodb/src/credential_store.rs index 5aa5ff14..05655eec 100644 --- a/crates/storage-mongodb/src/credential_store.rs +++ b/crates/storage-mongodb/src/credential_store.rs @@ -57,7 +57,10 @@ impl MongoCredentialStore { }; let account_id = key_doc.get_str("account_id").unwrap_or_default().to_owned(); let user_name = key_doc.get_str("user_name").unwrap_or_default().to_owned(); - let is_active = key_doc.get_bool("is_active").unwrap_or(true); + // Fail closed: treat a missing or malformed is_active as inactive so + // a corrupted or partially written access-key record cannot silently + // authenticate. + let is_active = key_doc.get_bool("is_active").unwrap_or(false); let secret_key = decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { From 46ede887a8d83407aa6c945a208c5c3651b4678b Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 23 Jul 2026 07:53:27 -0700 Subject: [PATCH 60/83] feat(devtools): add --backend flag to run-tests for mongo integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner was implicitly postgres-only in two places: it greps the config for `backend = "postgres"` before extracting a pg connection string, and it runs `test_cli_lifecycle.py` (postgres-only) whenever that connection string is set. Both worked accidentally on a mongo config today — the postgres-backend grep just missed and everything downstream was a no-op — but the coupling to config-file contents is fragile. Add an explicit `--backend {postgres,mongodb}` flag (default `postgres` for backward compat). The flag gates the two postgres-only paths and prints the backend in the target-info block. Everything else — health check, credential provisioning, throttling + import/export config mutation, pytest / rust / external / catalog- check suites — stays backend-agnostic and needs no change. Unblocks a mongo CI workflow that can delegate to `run-tests` the same way `.github/workflows/integration.yml` does for postgres. --- devtools/run-tests | 49 +++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/devtools/run-tests b/devtools/run-tests index b9fab041..fb0e4674 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -13,6 +13,7 @@ # devtools/run-tests --extenddb --rust --release # devtools/run-tests --extenddb --filter PATTERN --pytest # devtools/run-tests --extenddb --catalog-check --config PATH +# devtools/run-tests --extenddb --pytest --backend mongodb # # Target flag (exactly one required, mutually exclusive): # --extenddb targeting a local extenddb instance @@ -28,6 +29,10 @@ # --catalog-check post-test catalog integrity check # # Options: +# --backend NAME storage backend the running server uses: +# postgres (default) or mongodb. +# Gates postgres-only paths (pg connection-string +# extraction, test_cli_lifecycle.py). # --release use release build for Rust tests # --filter PATTERN pass to cargo test / pytest -k # --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) @@ -67,6 +72,7 @@ RELEASE=false FILTER="" CONFIG_PATH="" PARALLEL="" +BACKEND="postgres" usage() { cat <<'EOF' @@ -86,6 +92,8 @@ Suites (at least one required): --catalog-check post-test catalog integrity check Options: + --backend NAME storage backend the running server uses: + postgres (default) or mongodb. --release release build for Rust tests --filter PATTERN filter for cargo test / pytest -k --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) @@ -112,6 +120,7 @@ while [[ $# -gt 0 ]]; do --filter) FILTER="$2"; shift 2 ;; --catalog-check) RUN_CATALOG_CHECK=true; shift ;; --config) CONFIG_PATH="$2"; shift 2 ;; + --backend) BACKEND="$2"; shift 2 ;; --parallel) # --parallel (no argument) or --parallel=N PARALLEL="auto"; shift @@ -124,6 +133,15 @@ while [[ $# -gt 0 ]]; do esac done +# --- Validate: backend name --- +case "$BACKEND" in + postgres|mongodb) ;; + *) + echo "error: --backend must be 'postgres' or 'mongodb' (got: $BACKEND)" + exit 1 + ;; +esac + # --- Validate: target flag required --- if [[ -z "$TARGET" ]]; then echo "error: target flag is required (--extenddb or --real-dynamodb)" @@ -212,6 +230,7 @@ if [[ "$TARGET" == "real-dynamodb" ]]; then echo " EXTENDDB_TEST_ENDPOINT = (not set — using AWS SDK defaults)" echo " AWS_ACCESS_KEY_ID = ${AWS_ACCESS_KEY_ID:-(from ~/.aws)}" else + echo " backend = $BACKEND" echo " EXTENDDB_TEST_ENDPOINT = ${EXTENDDB_TEST_ENDPOINT:-}" echo " AWS_ACCESS_KEY_ID = ${AWS_ACCESS_KEY_ID:-(will be provisioned)}" fi @@ -358,18 +377,17 @@ if $NEEDS_INTEGRATION && [[ "$TARGET" != "real-dynamodb" ]]; then fi # Export PG connection string for CLI lifecycle tests (PostgreSQL only). - # Extract from config, strip the database name to get the base URL. - # This is only needed for test_cli_lifecycle.py which is PostgreSQL-specific - # and excluded from the main pytest suite. - if [[ -z "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" && -f "$CONFIG_FOR_SETTINGS" ]]; then - # Only try to extract connection_string for PostgreSQL backend - if grep -q '^[[:space:]]*backend[[:space:]]*=[[:space:]]*"postgres"' "$CONFIG_FOR_SETTINGS" 2>/dev/null; then - FULL_CONN=$(grep 'connection_string' "$CONFIG_FOR_SETTINGS" | sed -n 's/.*connection_string[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) - if [[ -n "$FULL_CONN" ]]; then - # Strip trailing /database_name to get base URL - export EXTENDDB_TEST_PG_CONNECTION_STRING="${FULL_CONN%/*}" - echo " ✓ EXTENDDB_TEST_PG_CONNECTION_STRING=${EXTENDDB_TEST_PG_CONNECTION_STRING}" - fi + # test_cli_lifecycle.py is PostgreSQL-specific and excluded from the + # main pytest suite; it opens its own extenddb instances via the + # connection string. Skipped on the mongo backend. + if [[ "$BACKEND" == "postgres" \ + && -z "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" \ + && -f "$CONFIG_FOR_SETTINGS" ]]; then + FULL_CONN=$(grep 'connection_string' "$CONFIG_FOR_SETTINGS" | sed -n 's/.*connection_string[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) + if [[ -n "$FULL_CONN" ]]; then + # Strip trailing /database_name to get base URL + export EXTENDDB_TEST_PG_CONNECTION_STRING="${FULL_CONN%/*}" + echo " ✓ EXTENDDB_TEST_PG_CONNECTION_STRING=${EXTENDDB_TEST_PG_CONNECTION_STRING}" fi fi echo "" @@ -526,8 +544,11 @@ if $RUN_COMPREHENSIVE; then echo "" fi -# --- CLI lifecycle tests (run last — they start/stop their own servers) --- -if $RUN_PYTEST && [[ "$TARGET" != "real-dynamodb" && -n "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" ]]; then +# --- CLI lifecycle tests (postgres-only; run last — they start/stop their own servers) --- +if $RUN_PYTEST \ + && [[ "$TARGET" != "real-dynamodb" \ + && "$BACKEND" == "postgres" \ + && -n "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" ]]; then CLI_OUTFILE="discussions/test-cli-${HASH}.txt" echo "=== CLI lifecycle tests → ${CLI_OUTFILE} ===" CLI_ARGS=(python3 -m pytest tests/test_cli_lifecycle.py tests/test_cli_container_readiness.py -v) From 9d7ce6daebe6d059b9ec90ebf1752dfaca8e7699 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 23 Jul 2026 07:54:49 -0700 Subject: [PATCH 61/83] docs(mongodb): bump minimum MongoDB version to 7.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the mongo backend has been verified against 6.x; local development, the bench-compare harness, and the container tag used in the planned CI workflow all use `mongo:7`. Bring the docs in line — stating 6.0+ implies a support surface we don't test and can't stand behind. Documentation-only change. No code touches the mongo-driver version floor; that's controlled by the `mongodb` crate's own minimum. --- AGENTS.md | 4 ++-- docs/design/13-storage-mongodb.md | 2 +- docs/getting-started.md | 2 +- docs/local-mongodb-setup.md | 2 +- docs/rfcs/0000-mongodb-backend.md | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bdf97ff7..adc5707c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ by AWS engineers. It is not a fork of DynamoDB and contains no DynamoDB source c protocol: any AWS SDK, CLI, or tool that works with DynamoDB works with ExtendDB, unchanged. - **Language:** Rust (edition 2024, MSRV 1.88+) -- **Storage backends:** PostgreSQL 14+ (default), MongoDB 6.0+ (feature flag `mongodb`) +- **Storage backends:** PostgreSQL 14+ (default), MongoDB 7.0+ (feature flag `mongodb`) - **Architecture:** Async (tokio), trait-based storage abstraction - **Authentication:** Mandatory SigV4 with built-in IAM (users, groups, roles, policies) - **TLS:** Mandatory (self-signed cert generated by default) @@ -87,7 +87,7 @@ extenddb (bin) - Rust 1.88+ (`rustup update`) - Storage backend (one of): - PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) - - MongoDB 6.0+ with replica set (see `docs/local-mongodb-setup.md`) + - MongoDB 7.0+ with replica set (see `docs/local-mongodb-setup.md`) - Python 3.10+ for tests (`python3 -m venv ~/venvs/extenddb-venv && source ~/venvs/extenddb-venv/bin/activate && pip install -r requirements.txt`) ### Build diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index c9428a7c..40743b89 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -1026,7 +1026,7 @@ The backend implements every trait in `extenddb-storage`: (`mongod --replSet rs0`), full trait coverage. - **Existing pytest suite:** Passes unchanged (backend-agnostic wire protocol tests). -- **CI:** GitHub Actions job with MongoDB 6.0 replica set, runs +- **CI:** GitHub Actions job with MongoDB 7.0 replica set, runs `cargo test -p extenddb-storage-mongodb` then `devtools/run-tests --extenddb --pytest --external`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 7139fd35..40721731 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,7 +31,7 @@ software on your behalf. After the script completes, continue from - **Storage backend** (one of): - PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) - - MongoDB 6.0+ with replica set (see `docs/local-mongodb-setup.md`) + - MongoDB 7.0+ with replica set (see `docs/local-mongodb-setup.md`) - Rust toolchain (1.88+) - AWS CLI v2 (for testing) - Python 3.10+ with virtual environment (see [Python Environment Setup](../README.md#python-environment-setup) in the README) diff --git a/docs/local-mongodb-setup.md b/docs/local-mongodb-setup.md index b3e6d6a6..d27b02a0 100644 --- a/docs/local-mongodb-setup.md +++ b/docs/local-mongodb-setup.md @@ -2,7 +2,7 @@ ## Prerequisites -- MongoDB 6.0+ (for multi-document transactions) +- MongoDB 7.0+ (for multi-document transactions) - A replica set configuration (required even for single-node deployments) ## Installation diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index 1d865886..d3623184 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -194,7 +194,7 @@ Backup implementation: `backup_engine.rs`. ### Operational requirements -**Minimum MongoDB version: 6.0.** Required for multi-document ACID transactions and snapshot reads. The MongoDB Rust driver 3.x is technically compatible with earlier server versions; this backend targets 6.0 as the minimum supported. +**Minimum MongoDB version: 7.0.** Required for multi-document ACID transactions and snapshot reads. The MongoDB Rust driver 3.x is technically compatible with earlier server versions; this backend targets 7.0 as the minimum supported. **Replica set required.** MongoDB must be configured as a replica set before running `extenddb init`. A standalone node does not support multi-document transactions. A single-node replica set is sufficient for development and CI; production deployments should use a 3-node replica set for high availability. @@ -273,7 +273,7 @@ Testing is organized in three layers. **End-to-end tests** run the existing ExtendDB pytest suite (`tests/`) unchanged against a MongoDB-backed ExtendDB server. The pytest suite speaks the DynamoDB wire protocol and has no backend awareness — a passing run against MongoDB is equivalent to a passing run against PostgreSQL. This is the conformance test baseline required by RFC-0002. -CI spins up a single-node MongoDB 6.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. +CI spins up a single-node MongoDB 7.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. ## Drawbacks From 094ac12f6ce879d36d527b77148142a1d423dbb2 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 23 Jul 2026 08:07:54 -0700 Subject: [PATCH 62/83] feat(devtools): add run-mongodb-tests orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `devtools/run-tests` is a runner, not an orchestrator — it assumes the server is already up at `$EXTENDDB_TEST_ENDPOINT`. The postgres CI workflow supplies the server lifecycle (init, serve, poll /health) inline before delegating to `run-tests`. Local mongo runs had no equivalent — the bench-compare harness recreated the lifecycle each time by hand. `devtools/run-mongodb-tests` fills that gap: one entry point that spins up a `mongo:7` single-node replica set in Docker, initializes and serves extenddb against it, then delegates to `devtools/run-tests --backend mongodb`. Teardown on exit; `--keep` leaves everything up for post-run inspection. Arguments after `--` are forwarded to `run-tests` verbatim so callers can pick the suite (`--pytest`, `--comprehensive`, `--parallel`, `--filter …`). Default is `--pytest --comprehensive --parallel`. The mongo CI workflow (a follow-up commit on this branch) can call this script directly and drop most of its shell-level orchestration. --- devtools/run-mongodb-tests | 261 +++++++++++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100755 devtools/run-mongodb-tests diff --git a/devtools/run-mongodb-tests b/devtools/run-mongodb-tests new file mode 100755 index 00000000..ad9b3162 --- /dev/null +++ b/devtools/run-mongodb-tests @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Orchestrated mongo integration-test runner. +# +# Starts a mongo:7 single-node replica set in Docker, initializes and +# serves extenddb against it, then delegates the test workload to +# `devtools/run-tests --backend mongodb`. Tears everything down on exit. +# +# Usage: +# devtools/run-mongodb-tests [OPTIONS] [-- RUN_TESTS_ARGS ...] +# +# Options: +# --port PORT HTTPS port extenddb should bind (default: 18443) +# --mongo-port PORT Host port to publish mongo on (default: 27021) +# --output DIR Directory for logs and generated config +# (default: /tmp/run-mongodb-tests-) +# --keep Do not tear down the mongo container / server +# on exit (useful for post-run inspection) +# -h, --help Show this help +# +# Arguments after `--` are passed to `devtools/run-tests` verbatim. +# Default suite is `--pytest --comprehensive --parallel`. +# +# Examples: +# devtools/run-mongodb-tests +# devtools/run-mongodb-tests -- --pytest --filter test_put_item +# devtools/run-mongodb-tests --keep -- --pytest --filter test_rfc0003 +# +# Prerequisites: +# - Docker running +# - `cargo build --release --features mongodb` already done +# - Python venv activated with pytest installed +# - `~/.extenddb/tls/` populated (from a prior `extenddb init`) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_DIR" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- + +BIND_PORT=18443 +MONGO_PORT=27021 +OUTPUT_DIR="" +KEEP=false +RUN_TESTS_ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --port) BIND_PORT="$2"; shift 2 ;; + --mongo-port) MONGO_PORT="$2"; shift 2 ;; + --output) OUTPUT_DIR="$2"; shift 2 ;; + --keep) KEEP=true; shift ;; + --) shift; RUN_TESTS_ARGS=("$@"); break ;; + -h|--help) sed -n '5,35p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "error: unknown option: $1" >&2; exit 1 ;; + esac +done + +if [[ ${#RUN_TESTS_ARGS[@]} -eq 0 ]]; then + RUN_TESTS_ARGS=(--pytest --comprehensive --parallel) +fi + +if [[ -z "$OUTPUT_DIR" ]]; then + OUTPUT_DIR="/tmp/run-mongodb-tests-$(date +%Y%m%d-%H%M%S)" +fi +mkdir -p "$OUTPUT_DIR" + +CONFIG="$OUTPUT_DIR/extenddb.toml" +SERVER_LOG="$OUTPUT_DIR/server.log" +SERVER_PID_FILE="$OUTPUT_DIR/server.pid" +CONTAINER_NAME="extenddb-runtests-mongo" +BINARY="./target/release/extenddb" + +if [[ ! -x "$BINARY" ]]; then + echo "error: $BINARY not found. Build first: cargo build --release --features mongodb" >&2 + exit 1 +fi + +# Canonicalized `/tmp` — resolves through the macOS `/tmp -> /private/tmp` +# symlink so the server's import/export path check (which rejects symlink +# components) accepts it. Portable across Linux and macOS. +TMP_CANON=$(realpath /tmp) + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- + +cleanup() { + if $KEEP; then + echo "" + echo "=== --keep set; leaving mongo + extenddb running ===" + echo " server pid file: $SERVER_PID_FILE" + echo " server log: $SERVER_LOG" + echo " mongo container: $CONTAINER_NAME (port $MONGO_PORT)" + echo " tear down manually: kill \$(cat $SERVER_PID_FILE); docker rm -f $CONTAINER_NAME" + return + fi + echo "" + echo "=== Cleanup ===" + if [[ -f "$SERVER_PID_FILE" ]]; then + local pid + pid=$(cat "$SERVER_PID_FILE") + if kill "$pid" 2>/dev/null; then + echo " stopped extenddb server (pid $pid)" + fi + rm -f "$SERVER_PID_FILE" + fi + if docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1; then + echo " removed docker container $CONTAINER_NAME" + fi +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Start mongo replica set +# --------------------------------------------------------------------------- + +echo "=== Starting MongoDB 7 replica set on host port $MONGO_PORT ===" +docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true +docker run -d --name "$CONTAINER_NAME" -p "$MONGO_PORT:27017" mongo:7 \ + mongod --replSet rs0 --bind_ip_all --wiredTigerCacheSizeGB 1 >/dev/null + +# Wait for mongod to accept pings +for i in $(seq 1 30); do + if docker exec "$CONTAINER_NAME" mongosh --quiet --eval "db.runCommand({ping:1}).ok" 2>/dev/null \ + | grep -q '^1$'; then + echo " mongod responsive after ${i}s" + break + fi + sleep 1 +done + +# Initiate replica set (idempotent — rs.initiate() errors are non-fatal) +docker exec "$CONTAINER_NAME" mongosh --quiet --eval \ + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "localhost:27017"}]})' >/dev/null 2>&1 || true + +# Wait for PRIMARY election +for i in $(seq 1 30); do + state=$(docker exec "$CONTAINER_NAME" mongosh --quiet --eval 'rs.status().members[0].stateStr' 2>/dev/null | tail -1) + if [[ "$state" == "PRIMARY" ]]; then + echo " replica set PRIMARY after ${i}s" + break + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Initialize extenddb +# --------------------------------------------------------------------------- + +echo "" +echo "=== Initializing extenddb ===" + +# `init --backend mongodb` reads the connection string from the config +# file, so write a minimal stub for it first. +cat > "$CONFIG" <&1 | tail -3 + +# `init` regenerates the config with default port 18443 and no +# `[import]` / `[export]` sections. Rewrite with the port + defaults the +# integration suite needs. This overwrite falls away once `extenddb init` +# grows flags for these knobs. +cat > "$CONFIG" <"$SERVER_LOG" 2>&1 & +echo "$!" > "$SERVER_PID_FILE" + +for i in $(seq 1 30); do + if curl -sk "https://127.0.0.1:$BIND_PORT/health" >/dev/null 2>&1; then + echo " server healthy after ${i}s" + break + fi + sleep 1 +done + +if ! curl -sk "https://127.0.0.1:$BIND_PORT/health" >/dev/null 2>&1; then + echo "error: extenddb server did not become healthy" >&2 + tail -30 "$SERVER_LOG" >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Delegate to run-tests +# --------------------------------------------------------------------------- + +echo "" +echo "=== Running test suite via devtools/run-tests --backend mongodb ===" +echo " passthrough args: ${RUN_TESTS_ARGS[*]}" +echo "" + +export EXTENDDB_TEST_ENDPOINT="https://127.0.0.1:$BIND_PORT" +export EXTENDDB_ADMIN_USER=admin +export EXTENDDB_ADMIN_PASSWORD="$ADMIN_PASSWORD" + +# Some tests read EXTENDDB_CONFIG to invoke `extenddb settings` against +# the running instance. Without this, they fall back to `./extenddb.toml` +# in the repo root, which is a stale dev config that points at a +# postgres instance that isn't running — the CLI hangs on the doomed +# connection and pytest times its subprocess out. +export EXTENDDB_CONFIG="$CONFIG" + +# import/export tests use tempfile.NamedTemporaryFile(), which honors +# $TMPDIR. On macOS $TMPDIR defaults to /var/folders/…/T, which isn't +# inside the server's configured [import]/[export] paths. Point TMPDIR +# at the canonical `/tmp` (matches the paths written above) so the +# tempfiles land in an allowed location on both Linux and macOS. +export TMPDIR="$TMP_CANON" + +RC=0 +devtools/run-tests --extenddb --backend mongodb --config "$CONFIG" "${RUN_TESTS_ARGS[@]}" || RC=$? + +exit $RC From 475ff8740400cca6bca8f3755cf67bafdce69f41 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 29 Jul 2026 18:10:56 -0600 Subject: [PATCH 63/83] fix(mongodb): scope get_stream_records to the shards owning account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream db0baba added `account_id` to `Storage::get_stream_records` so GetRecords is scoped to the shards owning account; the mongo backend still implemented the old 4-arg signature and returned records without an ownership check, so a caller could read another accounts stream records by presenting a forged shard iterator. Add the `account_id` parameter and an ownership guard that mirrors storage-postgres: resolve shard_id -> table_id from `stream_shards` (data db), then confirm a `tables` catalog document with that table_id is owned by the calling account (account_id lives inside the compound `_id`, so the comparison is done in Rust after a single table_id lookup). When the shard is unowned or absent, return ValidationException("Invalid ShardIterator") — matching DynamoDB, which does not distinguish "exists but not yours" from "does not exist". Verified by tests/test_cross_account_isolation.py::TestStreamAccountScoping ::test_shard_iterator_only_returns_owning_account_records against the mongo backend. Also syncs Cargo.lock (extenddb-storage-mongodb 0.1.0 -> 0.1.2) to the workspace version bump pulled in by the rebase. --- Cargo.lock | 2 +- crates/storage-mongodb/src/stream_engine.rs | 44 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5ba716b6..4a3f0bfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1266,7 +1266,7 @@ dependencies = [ [[package]] name = "extenddb-storage-mongodb" -version = "0.1.0" +version = "0.1.2" dependencies = [ "aes-gcm", "anyhow", diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index c5af8c21..e976a667 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -284,13 +284,57 @@ impl StreamEngine for MongoEngine { fn get_stream_records( &self, + account_id: &str, shard_id: &str, after_sequence: Option<&str>, limit: i64, ) -> BoxFuture<'_, StreamRecordsResult> { + let account_id = account_id.to_owned(); let shard_id = shard_id.to_owned(); let after_sequence = after_sequence.map(std::borrow::ToOwned::to_owned); Box::pin(async move { + // Ownership guard: only return records if the shard's backing table + // belongs to the calling account. `stream_shards`/`stream_records` + // live in the data database while the `tables` catalog (which + // carries account_id inside its compound `_id`) lives in the catalog + // database, so ownership is resolved in two steps across the two + // databases: shard_id -> table_id (data db), then table_id + + // account_id (catalog db). Mirrors the postgres backend. + let shards_coll = self.data_db.collection::("stream_shards"); + let shard_doc = shards_coll + .find_one(doc! { "shard_id": &shard_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let owned = match shard_doc.as_ref().and_then(|d| d.get_str("table_id").ok()) { + // Look the table up by its globally-unique `table_id` (a + // top-level field), then compare the owning account_id read out + // of the compound `_id` subdocument in Rust. account_id lives + // only inside the embedded `_id` doc; resolving ownership in + // Rust after a single table_id lookup keeps the check explicit + // and avoids depending on query-time behaviour of a partial + // `_id.account_id` path. + Some(table_id) => self + .catalog_db + .collection::("tables") + .find_one(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .as_ref() + .and_then(|t| t.get_document("_id").ok()) + .and_then(|id| id.get_str("account_id").ok()) + .is_some_and(|owner| owner == account_id), + None => false, + }; + // A shard iterator that resolves to a shard the caller does not own + // (catalog step fails) or one that does not exist (data step fails) + // is rejected identically. Real DynamoDB returns + // `ValidationException: Invalid ShardIterator` for a GetRecords + // iterator it did not issue, and does not distinguish "exists but not + // yours" from "does not exist" — so neither do we. + if !owned { + return Err(StorageError::Validation("Invalid ShardIterator".to_owned())); + } + let records_coll = self.data_db.collection::("stream_records"); let filter = if let Some(ref after) = after_sequence { From ebd2288bf2de881356db060eda919502515717f9 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 29 Jul 2026 18:40:32 -0600 Subject: [PATCH 64/83] chore(devtools): fix run-mongodb-tests server lifecycle to satisfy steering Start the server via extenddb serve (which daemonizes itself) instead of serve --foreground with a background &, and stop it via extenddb stop instead of kill. Set server.run_dir to the test output dir so serve and stop share an isolated PID-file location. Removes the manually-managed server.pid file. --- devtools/run-mongodb-tests | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/devtools/run-mongodb-tests b/devtools/run-mongodb-tests index ad9b3162..d7dfff23 100755 --- a/devtools/run-mongodb-tests +++ b/devtools/run-mongodb-tests @@ -73,7 +73,6 @@ mkdir -p "$OUTPUT_DIR" CONFIG="$OUTPUT_DIR/extenddb.toml" SERVER_LOG="$OUTPUT_DIR/server.log" -SERVER_PID_FILE="$OUTPUT_DIR/server.pid" CONTAINER_NAME="extenddb-runtests-mongo" BINARY="./target/release/extenddb" @@ -95,21 +94,15 @@ cleanup() { if $KEEP; then echo "" echo "=== --keep set; leaving mongo + extenddb running ===" - echo " server pid file: $SERVER_PID_FILE" - echo " server log: $SERVER_LOG" + echo " server log (startup): $SERVER_LOG" echo " mongo container: $CONTAINER_NAME (port $MONGO_PORT)" - echo " tear down manually: kill \$(cat $SERVER_PID_FILE); docker rm -f $CONTAINER_NAME" + echo " tear down manually: $BINARY stop --config $CONFIG; docker rm -f $CONTAINER_NAME" return fi echo "" echo "=== Cleanup ===" - if [[ -f "$SERVER_PID_FILE" ]]; then - local pid - pid=$(cat "$SERVER_PID_FILE") - if kill "$pid" 2>/dev/null; then - echo " stopped extenddb server (pid $pid)" - fi - rm -f "$SERVER_PID_FILE" + if [[ -f "$CONFIG" ]] && "$BINARY" stop --config "$CONFIG" >/dev/null 2>&1; then + echo " stopped extenddb server" fi if docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1; then echo " removed docker container $CONTAINER_NAME" @@ -180,6 +173,7 @@ cat > "$CONFIG" <"$SERVER_LOG" 2>&1 & -echo "$!" > "$SERVER_PID_FILE" +# `extenddb serve` daemonizes itself (forks to background, writes its own +# PID file under the config's run_dir, then returns), so no `&` is needed. +# Startup banner and any pre-daemonize errors are captured to SERVER_LOG; +# runtime logs go to syslog. +"$BINARY" serve --config "$CONFIG" >"$SERVER_LOG" 2>&1 for i in $(seq 1 30); do if curl -sk "https://127.0.0.1:$BIND_PORT/health" >/dev/null 2>&1; then From 02a900a01420dec3cfcf3f294ada805746cfe9d4 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 29 Jul 2026 19:06:31 -0600 Subject: [PATCH 65/83] fix(mongodb): mark Field vs Field comparisons as not pushable pushdown.rs admitted Field Field for all types, but a plain field type is unknown at compile time, so the emitted $expr compared the raw tagged subdocuments. Two Number fields (stored string-encoded) then compared lexically, so counter_a < counter_b evaluated backwards in both directions. Mark Field vs Field NotPushable so it falls back to the in-Rust evaluator, consistent with the existing N and B literal exclusions. Adds a regression test locking every comparator. --- crates/storage-mongodb/src/pushdown.rs | 38 ++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/storage-mongodb/src/pushdown.rs b/crates/storage-mongodb/src/pushdown.rs index 2f599b20..65a087ad 100644 --- a/crates/storage-mongodb/src/pushdown.rs +++ b/crates/storage-mongodb/src/pushdown.rs @@ -176,8 +176,16 @@ fn walk(expr: &Expr, maps: &ExpressionMaps) -> Pushable { | (AttrKind::Bool | AttrKind::Null, AttrKind::Field, _) => { Pushable::No("ordering on BOOL / NULL operand") } - // Field vs. Field: the compiler emits $expr; pushable. - (AttrKind::Field, AttrKind::Field, _) => Pushable::Yes, + // Field vs. Field: NOT pushable. A plain field's type is + // unknown at compile time (its AttrKind is just `Field`), so + // the emitted $expr compares the raw tagged subdocuments — + // e.g. two Number fields, stored string-encoded, compare + // lexically ("42" < "9"), giving the wrong answer in both + // directions. Fall back to the in-Rust evaluator, consistent + // with the N/B literal exclusions above. + (AttrKind::Field, AttrKind::Field, _) => { + Pushable::No("Field vs Field — operand types unknown at compile time") + } // Two literals — pushable but degenerate. _ => Pushable::No("Compare with unusual operand kinds"), } @@ -309,6 +317,32 @@ mod tests { assert!(!is_pushable(&expr, &maps).is_yes()); } + #[test] + fn field_vs_field_is_not_pushable() { + // Both operands are plain fields whose runtime types are unknown at + // compile time. Pushing $expr would compare tagged subdocuments + // (lexical for Numbers), so this must fall back to the in-Rust + // evaluator — for every comparator, not just ordering ones. + for op in [ + CompareOp::Eq, + CompareOp::Ne, + CompareOp::Lt, + CompareOp::Le, + CompareOp::Gt, + CompareOp::Ge, + ] { + let expr = Expr::Compare { + left: Box::new(path("counter_a")), + op, + right: Box::new(path("counter_b")), + }; + assert!( + !is_pushable(&expr, &maps_with(&[])).is_yes(), + "Field vs Field must not be pushable for {op:?}" + ); + } + } + #[test] fn string_equality_is_pushable() { let expr = Expr::Compare { From d0e181311143662a3c1b57992ebefc75e0379f7a Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 29 Jul 2026 19:48:02 -0600 Subject: [PATCH 66/83] fix(mongodb): compute binary begins_with upper bound in hex-string space Binary sort keys are stored as lowercase hex strings, so begins_with is a string-prefix range over the hex encoding. The upper bound was computed as hex(increment_bytes(prefix)) -- incrementing the raw bytes then re-encoding -- which is not the next prefix in fixed-width hex space and widens the range. begins_with(0x2F,0xFF) produced ["2fff","3000") and wrongly matched the stored key 0x30 ("30"); begins_with(0xFF) produced an empty range and dropped every match. Use next_string_prefix on the hex encoding, mirroring the string sort-key path: sk_b >= hex(B) AND sk_b < next_string_prefix(hex(B)), dropping the upper bound when the prefix is empty. Removes the now-unused increment_bytes helper. Adds a regression test for both wire-level repros. --- crates/storage-mongodb/src/data_engine.rs | 82 ++++++++++++++++------- 1 file changed, 56 insertions(+), 26 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index c24c9987..7527cdce 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -3275,15 +3275,25 @@ fn build_sk_filter( } } AttributeValue::B(ref b) => { - // Binary sort keys are stored as hex strings (D-M5), - // so begins_with is a plain lexicographic range: - // `sk_b >= hex(prefix) AND sk_b < increment(hex(prefix))`. - // The exclusive upper bound is the next hex prefix, - // computed by incrementing the raw bytes and encoding - // again — carries are handled by `increment_bytes`. + // Binary sort keys are stored as lowercase hex strings + // (D-M5), so `sk BEGINS_WITH B` is a string-prefix range + // over the hex encoding, exactly like the S case above: + // `sk_b >= hex(B) AND sk_b < next_string_prefix(hex(B))`. + // + // The exclusive upper bound must be the next prefix in + // hex-STRING space (increment the last hex character), not + // hex(increment_bytes(B)). Incrementing the raw bytes then + // re-encoding widens the range and admits unrelated keys — + // e.g. begins_with(0x2F,0xFF) -> ["2fff", hex(0x30,0x00) = + // "3000"), which wrongly matches the stored key 0x30 + // ("30"). next_string_prefix("2fff") = "2ffg" excludes it. + // When the prefix is empty, there is no upper bound and we + // match every key >= "" (all of them), matching DDB. let lo = binary_sk_to_hex(b); - let hi = binary_sk_to_hex(&increment_bytes(b)); - Ok(doc! { sk_field: { "$gte": lo, "$lt": hi } }) + match next_string_prefix(&lo) { + Some(upper) => Ok(doc! { sk_field: { "$gte": lo, "$lt": upper } }), + None => Ok(doc! { sk_field: { "$gte": lo } }), + } } _ => Err(StorageError::Validation( "begins_with requires string or binary sort key".to_string(), @@ -3394,24 +3404,6 @@ fn next_string_prefix(s: &str) -> Option { None } -/// Increment bytes to get the exclusive upper bound for `begins_with` on binary. -fn increment_bytes(b: &[u8]) -> Vec { - let mut result = b.to_vec(); - // Increment the last byte, with carry - let mut i = result.len(); - while i > 0 { - i -= 1; - if result[i] < 255 { - result[i] += 1; - return result; - } - result[i] = 0; - } - // All bytes were 0xFF; prepend a 0x01 byte (makes it longer) - result.insert(0, 1); - result -} - fn item_has_index_keys(item: &Item, idx_key_schema: &[KeySchemaElement]) -> bool { idx_key_schema .iter() @@ -3544,4 +3536,42 @@ mod tests { &AttributeValue::B(vec![0xff]) )); } + + fn binary_begins_with_bounds(prefix: Vec) -> (String, Option) { + let mut values = std::collections::HashMap::new(); + values.insert(":p".to_string(), AttributeValue::B(prefix)); + let maps = ExpressionMaps::new(std::collections::HashMap::new(), values); + let cond = SortKeyCondition::BeginsWith { + path: vec![PathElement::Attribute("sk".to_string())], + prefix: Expr::Placeholder(":p".to_string()), + }; + let doc = build_sk_filter(&cond, "sk_b", &maps).unwrap(); + let inner = doc.get_document("sk_b").unwrap(); + let lo = inner.get_str("$gte").unwrap().to_string(); + let hi = inner.get_str("$lt").ok().map(str::to_string); + (lo, hi) + } + + #[test] + fn binary_begins_with_uses_hex_space_prefix() { + // Upper bound is the next prefix in hex-STRING space, not + // hex(increment_bytes(prefix)). + + // begins_with(0x2F,0xFF): lo="2fff", hi must be "2ffg" (not "3000"). + // The old code produced "3000", which wrongly admitted stored key + // 0x30 ("30") since "2fff" <= "30" < "3000". With "2ffg", "30" is + // excluded because "30" > "2ffg". + let (lo, hi) = binary_begins_with_bounds(vec![0x2f, 0xff]); + assert_eq!(lo, "2fff"); + assert_eq!(hi.as_deref(), Some("2ffg")); + assert!("30" >= hi.as_deref().unwrap(), "0x30 must be excluded"); + + // begins_with(0xFF): lo="ff", hi must be "fg". The old code produced + // "00" (0xFF+1 wrapped then prepended 0x01 -> "01ff"? either way an + // empty/incorrect range), dropping every match. + let (lo, hi) = binary_begins_with_bounds(vec![0xff]); + assert_eq!(lo, "ff"); + assert_eq!(hi.as_deref(), Some("fg")); + assert!("ffab" < hi.as_deref().unwrap(), "0xFFAB must be included"); + } } From 068e8ed2c4ab52080ad70d45ba53449d466370d8 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 29 Jul 2026 23:03:41 -0600 Subject: [PATCH 67/83] fix(mongodb): implement transient table CREATING state CreateTable and RestoreTableFromBackup now write the catalog row as CREATING with a status_transition_at timestamp when control_plane_delay_seconds > 0 (default 0.25), and return CREATING; a new background control_plane_worker flips rows to ACTIVE once the scheduled transition time passes. When the delay is 0 the row is written ACTIVE directly. Matches the postgres backend and real DynamoDB, which report CREATING before a table is usable. DeleteTable sets DELETING on the row but completes the drop synchronously within the request; the control-plane worker only reconciles the CREATING -> ACTIVE transition, not deletes. Restore delegates row creation to create_table and no longer forces the table ACTIVE inline, so it enters the same CREATING window; the data is copied via $out before the table is flipped to ACTIVE. Data-plane key-schema resolution against a non-ACTIVE table now returns ResourceNotFoundException (TableNotFound) instead of ResourceInUse, matching DynamoDB and the postgres backend. Restores WorkerStore::process_control_plane_transitions (fixing the compound _id query the previous no-op replaced) and spawns the poller from MongoRuntimeHooks::spawn_workers. Reverts the RFC and design-doc language that described control-plane transitions as inline. Fixes the conformance tests put_item_on_creating_table_returns_not_found and restore_table_from_backup. --- crates/storage-mongodb/src/backup_engine.rs | 10 +- crates/storage-mongodb/src/lib.rs | 8 +- crates/storage-mongodb/src/table_engine.rs | 32 +++++- crates/storage-mongodb/src/ttl_worker.rs | 24 ++++- crates/storage-mongodb/src/worker_store.rs | 106 ++++++++++++++++---- docs/design/13-storage-mongodb.md | 16 +-- docs/rfcs/0000-mongodb-backend.md | 2 +- 7 files changed, 162 insertions(+), 36 deletions(-) diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index d9d987e3..f4449c6e 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -499,12 +499,18 @@ impl BackupEngine for MongoEngine { .map_err(|e| StorageError::Internal(e.to_string()))? as i64; - // Update item count and mark table ACTIVE + // Update the item count. The table was created via `create_table`, + // so it is already in CREATING (with a scheduled transition) when + // control_plane_delay_seconds > 0, or ACTIVE when it is 0; the + // control_plane_worker flips CREATING -> ACTIVE once the window + // passes. The data was just copied above, so it is in place before + // the table becomes ACTIVE. `desc` (returned to the caller) already + // carries the CREATING status from create_table, matching DynamoDB. let tables_coll = self.catalog_db.collection::("tables"); tables_coll .update_one( doc! { "_id": { "account_id": &account_id, "table_name": &target_table_name } }, - doc! { "$set": { "item_count": item_count, "table_status": "ACTIVE" } }, + doc! { "$set": { "item_count": item_count } }, ) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index 139133fe..ad03152d 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -142,7 +142,13 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { tokio::spawn(async move { ttl_worker::gsi_backfill_worker(storage_for_backfill).await; }); - tracing::info!("MongoDB backend: TTL, stream cleanup, and GSI backfill workers spawned"); + let storage_for_control_plane = self.engine.clone(); + tokio::spawn(async move { + ttl_worker::control_plane_worker(storage_for_control_plane).await; + }); + tracing::info!( + "MongoDB backend: TTL, stream cleanup, GSI backfill, and control-plane workers spawned" + ); } fn backend_info(&self) -> Option { diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index b7e3ee91..47d02e27 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -202,6 +202,23 @@ impl MongoEngine { |v| bson::to_bson(v).unwrap_or(bson::Bson::Null), ); + // Enter CREATING with a scheduled transition to ACTIVE, unless + // control_plane_delay_seconds is 0 (then go straight to ACTIVE). The + // background control_plane_worker flips CREATING -> ACTIVE once the + // transition time passes; during the window data-plane ops on the + // table return ResourceNotFound, matching DynamoDB and the postgres + // backend. + let delay_secs = self.control_plane_delay_seconds().await; + let (table_status, status_transition_at): (&str, bson::Bson) = if delay_secs <= 0.0 { + ("ACTIVE", bson::Bson::Null) + } else { + let at = bson::DateTime::now().timestamp_millis() + (delay_secs * 1000.0) as i64; + ( + "CREATING", + bson::Bson::DateTime(bson::DateTime::from_millis(at)), + ) + }; + let table_doc = doc! { "_id": { "account_id": account_id, "table_name": &input.table_name }, "key_schema": key_schema_bson, @@ -209,7 +226,8 @@ impl MongoEngine { "billing_mode": billing_str, "provisioned_throughput": pt_bson.unwrap_or(bson::Bson::Null), "stream_specification": stream_bson.unwrap_or(bson::Bson::Null), - "table_status": "ACTIVE", + "table_status": table_status, + "status_transition_at": status_transition_at, "creation_date_time": bson::DateTime::from_millis((creation_epoch * 1000.0) as i64), "table_size_bytes": 0_i64, "item_count": 0_i64, @@ -478,7 +496,11 @@ impl MongoEngine { table_name: input.table_name, key_schema: input.key_schema, attribute_definitions: input.attribute_definitions, - table_status: TableStatus::Active, + table_status: if table_status == "CREATING" { + TableStatus::Creating + } else { + TableStatus::Active + }, creation_date_time: creation_epoch, table_size_bytes: 0, item_count: 0, @@ -909,7 +931,11 @@ impl MongoEngine { let status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); if require_active && status != "ACTIVE" { - return Err(StorageError::TableNotActive(table_name)); + // This guard gates data-plane key-schema resolution. DynamoDB + // returns ResourceNotFoundException (not ResourceInUse) for a + // data-plane op against a table that is not yet ACTIVE, matching + // the postgres backend; TableNotActive would map to ResourceInUse. + return Err(StorageError::TableNotFound(table_name)); } let table_id = table_doc diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index da02a437..5a45ff52 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -10,7 +10,7 @@ use bson::{Document, doc}; use extenddb_core::metrics::MetricsCollector; use extenddb_core::types::{KeySchemaElement, Projection, ProjectionType, UserIdentity}; use extenddb_storage::error::StorageError; -use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine, TableEngine}; +use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine, TableEngine, WorkerStore}; use futures::TryStreamExt; use crate::MongoEngine; @@ -21,6 +21,10 @@ const STREAM_RETENTION_HOURS: i64 = 24; const STREAM_CLEANUP_INTERVAL: Duration = Duration::from_secs(3600); const GSI_BACKFILL_INTERVAL: Duration = Duration::from_secs(5); const GSI_BACKFILL_BATCH: i64 = 500; +/// How often to flip due `CREATING` tables to `ACTIVE`. Short enough that the +/// window closes promptly after `control_plane_delay_seconds` (default 0.25s) +/// without busy-spinning. +const CONTROL_PLANE_POLL_INTERVAL: Duration = Duration::from_millis(250); pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { let region_arc: Arc = Arc::from(storage.region.as_str()); @@ -360,3 +364,21 @@ fn build_ttl_condition( (condition_expr, ExpressionMaps::new(names, values)) } + +/// Background poller that flips tables out of the transient `CREATING` state +/// once their scheduled `status_transition_at` has passed. See +/// [`crate::worker_store`] for how rows enter `CREATING`. +pub(crate) async fn control_plane_worker(storage: Arc) { + loop { + tokio::time::sleep(CONTROL_PLANE_POLL_INTERVAL).await; + match WorkerStore::process_control_plane_transitions(&*storage).await { + Ok(t) if t.is_empty() => {} + Ok(transitions) => { + for (name, transition) in &transitions { + tracing::info!("Table '{name}': {transition}"); + } + } + Err(e) => tracing::warn!("Control-plane transition poll failed: {e}"), + } + } +} diff --git a/crates/storage-mongodb/src/worker_store.rs b/crates/storage-mongodb/src/worker_store.rs index 69fb163b..6308f4fd 100644 --- a/crates/storage-mongodb/src/worker_store.rs +++ b/crates/storage-mongodb/src/worker_store.rs @@ -3,41 +3,105 @@ //! `WorkerStore` implementation for `MongoDB`. //! -//! The MongoDB backend does not use transient control-plane states -//! (`CREATING`, `DELETING`) for tables: `create_table_impl` writes the -//! catalog row with `table_status: "ACTIVE"` synchronously and -//! `delete_table_impl` removes the row + collections in one call, so -//! there is never a table document waiting for a background transition. -//! Both paths run inline in the request handler because MongoDB's -//! collection create/drop is fast enough not to warrant asynchronous -//! promotion, and the alternative would require a background worker -//! whose only job is to catch up work the API call could have done -//! synchronously anyway. +//! Processes table control-plane transitions (`CREATING` → `ACTIVE`) as a +//! background job. `create_table_impl` and the restore path write the catalog +//! row as `CREATING` with a `status_transition_at` timestamp when +//! `control_plane_delay_seconds` > 0 (matching the Postgres backend and real +//! DynamoDB, which report `CREATING` before a table becomes usable); this +//! worker flips such rows to `ACTIVE` once their transition time has passed. +//! When `control_plane_delay_seconds` is 0 the create/restore paths write +//! `ACTIVE` directly and this worker has nothing to do. //! -//! GSI create is the one control-plane operation that does need -//! async work — its background portion lives in -//! [`ttl_worker::gsi_backfill_worker`] rather than here because it -//! operates on the `indexes` catalog collection with a `CREATING` -//! index-status, not on the `tables` collection. -//! -//! The trait method returns an empty list so `WorkerStore` is -//! satisfied for the [`OperationsEngine`] supertrait bound without -//! introducing a background job that would only ever be a no-op. +//! `DeleteTable` remains inline (the catalog row and collections are removed in +//! the request handler), so there is no `DELETING` transient state to reconcile +//! here. GSI create is handled separately by +//! [`ttl_worker::gsi_backfill_worker`] on the `indexes` catalog collection. //! //! [`ttl_worker::gsi_backfill_worker`]: crate::ttl_worker::gsi_backfill_worker -//! [`OperationsEngine`]: extenddb_storage::OperationsEngine +use futures::TryStreamExt; use futures::future::BoxFuture; +use mongodb::bson::{Document, doc}; use extenddb_storage::WorkerStore; use extenddb_storage::error::StorageError; use crate::MongoEngine; +/// Default control-plane delay (seconds) when the setting is absent or +/// unparseable. Matches the Postgres backend default. +const DEFAULT_CONTROL_PLANE_DELAY_SECS: f64 = 0.25; + +impl MongoEngine { + /// Read `control_plane_delay_seconds` from the settings collection, + /// falling back to the default. A value <= 0 means "no CREATING window" + /// (create/restore write `ACTIVE` synchronously). + pub(crate) async fn control_plane_delay_seconds(&self) -> f64 { + let coll = self.catalog_db.collection::("settings"); + coll.find_one(doc! { "_id": "control_plane_delay_seconds" }) + .await + .ok() + .flatten() + .and_then(|d| d.get_str("value").ok().map(str::to_owned)) + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 0.0) + .unwrap_or(DEFAULT_CONTROL_PLANE_DELAY_SECS) + } +} + impl WorkerStore for MongoEngine { fn process_control_plane_transitions( &self, ) -> BoxFuture<'_, Result, StorageError>> { - Box::pin(async move { Ok(Vec::new()) }) + Box::pin(async move { + let mut transitions = Vec::new(); + let tables_coll = self.catalog_db.collection::("tables"); + let now = mongodb::bson::DateTime::now(); + + // CREATING → ACTIVE: tables whose scheduled transition time has + // passed. Each row is updated by its own compound `_id` (the mongo + // catalog stores account_id/table_name inside `_id`, not at the top + // level — the previous impl filtered on flat fields and matched + // nothing). + let filter = doc! { + "table_status": "CREATING", + "status_transition_at": { "$lte": now }, + }; + let mut cursor = tables_coll + .find(filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + while let Some(table_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let Some(id) = table_doc.get("_id").cloned() else { + continue; + }; + let table_name = table_doc + .get_document("_id") + .ok() + .and_then(|d| d.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + + tables_coll + .update_one( + doc! { "_id": id, "table_status": "CREATING" }, + doc! { + "$set": { "table_status": "ACTIVE" }, + "$unset": { "status_transition_at": "" }, + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + transitions.push((table_name, "CREATING → active")); + } + + Ok(transitions) + }) } } diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index 40743b89..55ef6ef5 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -833,7 +833,7 @@ crates/storage-mongodb/ ├── credential_store.rs # Access-key lookup + AES-GCM decryption ├── catalog_store.rs # SettingsStore / DiagnosticsStore glue ├── admin_store.rs # Admin operations (currently thin) - └── worker_store.rs # WorkerStore trait shim (no-op; supertrait requirement) + └── worker_store.rs # WorkerStore: CREATING -> ACTIVE table transitions ``` ## 7. `MongoEngine` Struct @@ -993,12 +993,12 @@ The backend implements every trait in `extenddb-storage`: - `Bootstrapper` — init, destroy, migrate, verify. Creates the catalog and data databases, seeds encryption key and admin user, applies index schema. -- `WorkerStore` — trait method is a no-op. `create_table_impl` - writes `TableStatus: ACTIVE` inline and `delete_table_impl` runs - the collection/tag/stream cleanup inline, so there is never a - transient state waiting for a background worker. Kept in the - impl surface only because `OperationsEngine` requires - `WorkerStore` as a supertrait. +- `WorkerStore` — `process_control_plane_transitions` flips tables + from `CREATING` to `ACTIVE` once their `status_transition_at` + passes. `create_table_impl` (and restore) write `CREATING` with a + scheduled transition when `control_plane_delay_seconds` > 0, or + `ACTIVE` directly when it is 0. `delete_table_impl` remains inline + (no `DELETING` transient state). - `ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, `RateLimitStore` — the catalog trait surface. - `AuthorizationStore` — user/group/role/permissions-boundary/session @@ -1013,6 +1013,8 @@ The backend implements every trait in `extenddb-storage`: 24 h retention TTL index. - `gsi_backfill_worker` — drain `indexes` rows in `CREATING` state, scan the base collection with a persistent cursor, flip to ACTIVE. +- `control_plane_worker` — flip `tables` rows from `CREATING` to + `ACTIVE` once their scheduled `status_transition_at` passes. ## 13. Testing Strategy diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index d3623184..248160b8 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -170,7 +170,7 @@ TTL index creation: `metadata_engine.rs` — `create_ttl_index`. Background work ### Control plane state transitions -Table creation and deletion run inline. `CreateTable` writes the catalog row and creates the data collection with its indexes before returning; the returned `TableDescription` carries `TableStatus: ACTIVE`. `DeleteTable` removes the catalog row, drops the data + index collections, deletes tags, and cleans up stream shards / records / counters via `cleanup_stream_state_for_table` (`table_engine.rs`), all before returning. MongoDB's create/drop is fast enough that there is no need to defer either operation to a background worker. +`CreateTable` (and `RestoreTableFromBackup`) write the catalog row and create the data collection with its indexes before returning. When `control_plane_delay_seconds` > 0 (the default is 0.25) the row is written with `TableStatus: CREATING` and a `status_transition_at` timestamp, and the returned `TableDescription` carries `TableStatus: CREATING`; a background `control_plane_worker` (`ttl_worker.rs`) flips the row to `ACTIVE` once the transition time passes. During the window, data-plane operations against the table return `ResourceNotFoundException`, matching DynamoDB and the PostgreSQL backend. When `control_plane_delay_seconds` is 0, the row is written `ACTIVE` directly and the worker has nothing to do. `DeleteTable` remains inline: it removes the catalog row, drops the data + index collections, deletes tags, and cleans up stream shards / records / counters via `cleanup_stream_state_for_table` (`table_engine.rs`), all before returning — there is no transient `DELETING` state. GSI creation on `UpdateTable` is the one control-plane operation that does need asynchronous work — a background worker drains index rows in `CREATING` state, backfills the base collection, and flips the row to `ACTIVE`. See the Global and Local Secondary Indexes section for the state machine. From b05f5942fe09ea1ae10177d74d9e935e5a14bf09 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 30 Jul 2026 00:39:38 -0600 Subject: [PATCH 68/83] chore(mongodb): adopt set_backend registration and adapt to post-#218 main Rebase onto upstream main after PR #218 (serve lib decoupling), which replaced inventory backend registration with an explicit set_backend/Backend model and split the CLI into extenddb-app. Also adapts to backup-trait and worker changes and to new backup_arn_scoping conformance tests pulled in by the rebase. - Replace the six inventory::submit! blocks with a single extenddb_storage_mongodb::backend() constructor plus a server_components_factory fn, mirroring the postgres backend. - Drop the now-removed inventory dependency. - Feature-gate the thin bin: install the mongodb backend under --features mongodb, else postgres. - Scope describe_backup and delete_backup to account_id (added to the BackupEngine trait upstream); exclude DELETED backups from describe_backup so a deleted backup reads as BackupNotFoundException. - Give backup ARNs a timestamp-plus-8-hex-char random id so they are not guessable from creation time alone. - Return the spawned worker JoinHandles from spawn_workers, whose trait signature now requires Vec>. --- Cargo.lock | 46 ++-- crates/bin/Cargo.toml | 8 +- crates/bin/src/main.rs | 10 +- crates/storage-mongodb/Cargo.toml | 3 - crates/storage-mongodb/src/backup_engine.rs | 36 ++- crates/storage-mongodb/src/lib.rs | 264 +++++++++----------- 6 files changed, 194 insertions(+), 173 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a3f0bfa..99d25813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1117,20 +1117,29 @@ dependencies = [ [[package]] name = "extenddb" version = "0.1.2" +dependencies = [ + "anyhow", + "extenddb-app", + "extenddb-storage", + "extenddb-storage-mongodb", + "extenddb-storage-postgres", +] + +[[package]] +name = "extenddb-app" +version = "0.1.2" dependencies = [ "anyhow", "base64 0.22.1", "clap", - "config", "daemonize", "extenddb-auth", "extenddb-cache", + "extenddb-config", "extenddb-core", "extenddb-engine", "extenddb-server", "extenddb-storage", - "extenddb-storage-mongodb", - "extenddb-storage-postgres", "libc", "rcgen", "rustls", @@ -1138,7 +1147,6 @@ dependencies = [ "serde", "serde_json", "sqlx", - "syslog-tracing", "time", "tokio", "toml", @@ -1177,6 +1185,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "extenddb-config" +version = "0.1.2" +dependencies = [ + "anyhow", + "config", + "extenddb-core", + "extenddb-storage", + "serde", + "toml", + "tracing", +] + [[package]] name = "extenddb-core" version = "0.1.2" @@ -1223,21 +1244,25 @@ dependencies = [ "crc32fast", "extenddb-auth", "extenddb-cache", + "extenddb-config", "extenddb-core", "extenddb-engine", "extenddb-storage", "futures", "hyper", + "libc", "metrics", "rand 0.9.5", "rustls", "serde", "serde_json", + "syslog-tracing", "time", "tokio", "tower", "tower-http", "tracing", + "tracing-subscriber", "uuid", ] @@ -1253,12 +1278,12 @@ dependencies = [ "extenddb-auth", "extenddb-core", "futures", - "inventory", "rand 0.9.5", "serde_json", "thiserror", "time", "tokio", + "tokio-util", "toml", "tracing", "tracing-subscriber", @@ -1280,7 +1305,6 @@ dependencies = [ "extenddb-core", "extenddb-storage", "futures", - "inventory", "mongodb", "proptest", "rand 0.9.5", @@ -1309,7 +1333,6 @@ dependencies = [ "extenddb-core", "extenddb-storage", "futures", - "inventory", "rand 0.9.5", "serde", "serde_json", @@ -1957,15 +1980,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "inventory" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" -dependencies = [ - "rustversion", -] - [[package]] name = "ipconfig" version = "0.3.4" diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index d5b148ab..c44a6217 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -11,8 +11,14 @@ license.workspace = true name = "extenddb" path = "src/main.rs" +[features] +default = ["postgres"] +postgres = ["dep:extenddb-storage-postgres"] +mongodb = ["dep:extenddb-storage-mongodb"] + [dependencies] extenddb-app = { workspace = true } extenddb-storage = { workspace = true } -extenddb-storage-postgres = { workspace = true } +extenddb-storage-postgres = { workspace = true, optional = true } +extenddb-storage-mongodb = { workspace = true, optional = true } anyhow = { workspace = true } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index 5d71c9e7..3d6431a3 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -1,18 +1,26 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! extenddb — the PostgreSQL-backed ExtendDB server binary. +//! extenddb — the ExtendDB server binary. //! //! This is the reference thin bin for the per-backend packaging model: it //! installs exactly one backend and hands off to the shared `extenddb-app` CLI. //! A third-party backend author copies this file, swaps the `backend()` call for //! their crate, and ships their own `extenddb-` image — with no edits to //! any ExtendDB core crate. +//! +//! This fork's bin compiles the PostgreSQL backend by default and the MongoDB +//! backend under `--features mongodb`, selecting the one to install at compile +//! time so a single bin serves both while the reviewer's per-backend model is +//! adopted. fn main() -> anyhow::Result<()> { // Install the compiled-in backend before dispatch. The compiler checks this // call; there is no link-time auto-registration and no name to resolve, so a // missing or mistyped backend cannot become a runtime error. + #[cfg(feature = "mongodb")] + extenddb_storage::set_backend(extenddb_storage_mongodb::backend())?; + #[cfg(not(feature = "mongodb"))] extenddb_storage::set_backend(extenddb_storage_postgres::backend())?; extenddb_app::run(extenddb_app::BuildInfo { diff --git a/crates/storage-mongodb/Cargo.toml b/crates/storage-mongodb/Cargo.toml index fbc5b862..54f6d56e 100644 --- a/crates/storage-mongodb/Cargo.toml +++ b/crates/storage-mongodb/Cargo.toml @@ -29,9 +29,6 @@ tokio.workspace = true async-trait.workspace = true futures.workspace = true -# Backend registry -inventory.workspace = true - # MongoDB driver mongodb.workspace = true diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index f4449c6e..3c200b46 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -113,8 +113,16 @@ impl BackupEngine for MongoEngine { .cloned() .unwrap_or(mongodb::bson::Bson::Null); + // The trailing backup-id component is a timestamp plus an 8-hex-char + // random suffix, so a backup ARN (which is a capability) is not + // guessable from the creation time alone. Matches the postgres + // backend. + let arn_suffix: u32 = { + use rand::Rng; + rand::rng().random() + }; let backup_arn = format!( - "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{ts}", + "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{ts}-{arn_suffix:08x}", region = self.region, ts = epoch_millis() ); @@ -194,13 +202,23 @@ impl BackupEngine for MongoEngine { fn describe_backup( &self, + account_id: &str, backup_arn: &str, ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); let backup_arn = backup_arn.to_string(); Box::pin(async move { let backups_coll = self.catalog_db.collection::("backups"); + // Scope the lookup to the calling account so a backup ARN cannot be + // read cross-account, and exclude DELETED backups so a deleted + // backup reads as BackupNotFoundException. Matches the postgres + // backend. let backup_doc = backups_coll - .find_one(doc! { "_id": &backup_arn }) + .find_one(doc! { + "_id": &backup_arn, + "account_id": &account_id, + "backup_status": { "$ne": "DELETED" }, + }) .await .map_err(|e| StorageError::Internal(e.to_string()))? .ok_or_else(|| { @@ -334,16 +352,18 @@ impl BackupEngine for MongoEngine { fn delete_backup( &self, + account_id: &str, backup_arn: &str, ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); let backup_arn = backup_arn.to_string(); Box::pin(async move { - let desc = self.describe_backup(&backup_arn).await?; + let desc = self.describe_backup(&account_id, &backup_arn).await?; - // Look up the physical collection name from metadata. + // Look up the physical collection name from metadata (account-scoped). let backups_coll = self.catalog_db.collection::("backups"); let meta = backups_coll - .find_one(doc! { "_id": &backup_arn }) + .find_one(doc! { "_id": &backup_arn, "account_id": &account_id }) .await .map_err(|e| StorageError::Internal(e.to_string()))? .ok_or_else(|| { @@ -361,10 +381,10 @@ impl BackupEngine for MongoEngine { .map_err(|e| StorageError::Internal(e.to_string()))?; } - // Mark backup as deleted + // Mark backup as deleted (account-scoped) backups_coll .update_one( - doc! { "_id": &backup_arn }, + doc! { "_id": &backup_arn, "account_id": &account_id }, doc! { "$set": { "backup_status": "DELETED" } }, ) .await @@ -623,7 +643,7 @@ impl BackupEngine for MongoEngine { let desc = self .restore_table_from_backup(&account_id, &target_table_name, &backup.backup_arn) .await?; - let _ = self.delete_backup(&backup.backup_arn).await; + let _ = self.delete_backup(&account_id, &backup.backup_arn).await; Ok(desc) }) } diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index ad03152d..cd859af1 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -35,93 +35,11 @@ use std::sync::Arc; use extenddb_storage::error::StorageError; // ============================================================================ -// OperationsEngineRegistration -// ============================================================================ - -inventory::submit! { - extenddb_storage::operations::OperationsEngineRegistration { - name: "mongodb", - operations: &operations::MongoOperationsEngine, - } -} - -// ============================================================================ -// BackendRegistration -// ============================================================================ - -inventory::submit! { - extenddb_storage::bootstrapper::BackendRegistration { - name: "mongodb", - factory: |config_path, cli_args| { - Box::pin(async move { - let store = MongoBootstrapper::from_config(&config_path, &cli_args).await?; - Ok(Box::new(store) as Box) - }) - } - } -} - -// ============================================================================ -// StorageConfigRegistration -// ============================================================================ - -inventory::submit! { - extenddb_storage::config::StorageConfigRegistration { - backend: "mongodb", - deserializer: |table| { - let config: MongoStorageConfig = table.clone().try_into() - .map_err(|e: toml::de::Error| format!("Failed to parse mongodb config: {e}"))?; - Ok(Box::new(config) as Box) - }, - } -} - -// ============================================================================ -// SettingsStoreRegistration -// ============================================================================ - -inventory::submit! { - extenddb_storage::settings_store::SettingsStoreRegistration { - backend: "mongodb", - factory: |connection_string| { - let connection_string = connection_string.to_string(); - Box::pin(async move { - let client = mongodb::Client::with_uri_str(&connection_string) - .await - .map_err(|e| extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed(e.to_string()))?; - Ok(Box::new(MongoCatalogStore::new(client)) as Box) - }) - }, - } -} - -// ============================================================================ -// DiagnosticsStoreRegistration -// ============================================================================ - -inventory::submit! { - extenddb_storage::diagnostics_store::DiagnosticsStoreRegistration { - backend: "mongodb", - factory: |connection_string| { - let connection_string = connection_string.to_string(); - Box::pin(async move { - let client = mongodb::Client::with_uri_str(&connection_string) - .await - .map_err(|e| extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed(e.to_string()))?; - Ok(Box::new(MongoCatalogStore::new(client)) as Box) - }) - }, - } -} - -// ============================================================================ -// ServerComponentsRegistration +// Backend registration // ============================================================================ use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; -use extenddb_storage::server_components::{ - BackendError, ServerComponents, ServerComponentsRegistration, -}; +use extenddb_storage::server_components::{BackendError, ServerComponents}; /// Backend-specific runtime hooks for `MongoDB`. struct MongoRuntimeHooks { @@ -130,25 +48,28 @@ struct MongoRuntimeHooks { #[async_trait::async_trait] impl ServerRuntimeHooks for MongoRuntimeHooks { - async fn spawn_workers(&self, ctx: &WorkerContext) { + async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec> { let storage_for_ttl = self.engine.clone(); let metrics = ctx.metrics.clone(); - tokio::spawn(async move { ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics).await }); + let ttl = tokio::spawn(async move { + ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics).await; + }); let storage_for_stream = self.engine.clone(); - tokio::spawn(async move { + let stream = tokio::spawn(async move { ttl_worker::stream_record_cleanup_worker(storage_for_stream).await; }); let storage_for_backfill = self.engine.clone(); - tokio::spawn(async move { + let backfill = tokio::spawn(async move { ttl_worker::gsi_backfill_worker(storage_for_backfill).await; }); let storage_for_control_plane = self.engine.clone(); - tokio::spawn(async move { + let control_plane = tokio::spawn(async move { ttl_worker::control_plane_worker(storage_for_control_plane).await; }); tracing::info!( "MongoDB backend: TTL, stream cleanup, GSI backfill, and control-plane workers spawned" ); + vec![ttl, stream, backfill, control_plane] } fn backend_info(&self) -> Option { @@ -156,68 +77,123 @@ impl ServerRuntimeHooks for MongoRuntimeHooks { } } -inventory::submit! { - ServerComponentsRegistration { - backend: "mongodb", - factory: |config, region| { - let connection_string = config.connection_config().to_string(); - let max_connections = config.max_connections(); - let region = region.to_string(); - Box::pin(async move { - // Create MongoEngine - let engine = MongoEngine::new(&connection_string, ®ion, max_connections) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "mongodb".to_string(), - details: e.to_string(), - })?; +/// Build the assembled server components for the mongo backend (`serve`). +fn server_components_factory( + config: &dyn extenddb_storage::config::StorageConfig, + region: &str, +) -> std::pin::Pin< + Box> + Send>, +> { + let connection_string = config.connection_config().to_string(); + let max_connections = config.max_connections(); + let region = region.to_string(); + Box::pin(async move { + // Create MongoEngine + let engine = MongoEngine::new(&connection_string, ®ion, max_connections) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: e.to_string(), + })?; - let engine = Arc::new(engine); + let engine = Arc::new(engine); - // Create catalog store - let catalog_client = mongodb::Client::with_uri_str(&connection_string) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "mongodb".to_string(), - details: format!("Failed to create catalog client: {e}"), - })?; + // Create catalog store + let catalog_client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: format!("Failed to create catalog client: {e}"), + })?; + + // Load encryption key from settings collection + let catalog_db = catalog_client.database("extenddb_catalog"); + let settings_coll = catalog_db.collection::("settings"); + let enc_key = settings_coll + .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? + .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) + .unwrap_or_default(); + + let catalog_store = Arc::new(MongoCatalogStore::with_encryption_key( + catalog_client, + enc_key.clone(), + )) as Arc; + + // Create credential store. The bin layer wraps this in + // CachedCredentialStore using the operator-configured TTL + // before constructing the auth provider. + let auth_client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Auth client: {e}")))?; + let cred_store: Arc = + Arc::new(MongoCredentialStore::new(auth_client, enc_key)); + + // Create runtime hooks + let runtime_hooks = Box::new(MongoRuntimeHooks { + engine: engine.clone(), + }); + + Ok(ServerComponents { + engine, + catalog_store, + credential_store: cred_store, + runtime_hooks: Some(runtime_hooks), + }) + }) +} - // Load encryption key from settings collection - let catalog_db = catalog_client.database("extenddb_catalog"); - let settings_coll = catalog_db.collection::("settings"); - let enc_key = settings_coll - .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) +/// The MongoDB storage backend. A thin bin installs it via +/// `extenddb_storage::set_backend(extenddb_storage_mongodb::backend())`. +pub fn backend() -> extenddb_storage::Backend { + extenddb_storage::Backend { + name: "mongodb", + bootstrapper: |config_path, cli_args| { + Box::pin(async move { + let store = MongoBootstrapper::from_config(&config_path, &cli_args).await?; + Ok(Box::new(store) as Box) + }) + }, + storage_config: |table| { + let config: MongoStorageConfig = table + .clone() + .try_into() + .map_err(|e: toml::de::Error| format!("Failed to parse mongodb config: {e}"))?; + Ok(Box::new(config) as Box) + }, + operations: &operations::MongoOperationsEngine, + settings_store: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let client = mongodb::Client::with_uri_str(&connection_string) .await - .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? - .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) - .unwrap_or_default(); - - let catalog_store = Arc::new( - MongoCatalogStore::with_encryption_key(catalog_client, enc_key.clone()) - ) as Arc; - - // Create credential store. The bin layer wraps this in - // CachedCredentialStore using the operator-configured TTL - // before constructing the auth provider. - let auth_client = mongodb::Client::with_uri_str(&connection_string) + .map_err(|e| { + extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed( + e.to_string(), + ) + })?; + Ok(Box::new(MongoCatalogStore::new(client)) + as Box< + dyn extenddb_storage::management_store::SettingsStore, + >) + }) + }, + diagnostics_store: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let client = mongodb::Client::with_uri_str(&connection_string) .await - .map_err(|e| BackendError::InitializationFailed(format!("Auth client: {e}")))?; - let cred_store: Arc = - Arc::new(MongoCredentialStore::new(auth_client, enc_key)); - - // Create runtime hooks - let runtime_hooks = Box::new(MongoRuntimeHooks { - engine: engine.clone(), - }); - - Ok(ServerComponents { - engine, - catalog_store, - credential_store: cred_store, - runtime_hooks: Some(runtime_hooks), - }) + .map_err(|e| { + extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed( + e.to_string(), + ) + })?; + Ok(Box::new(MongoCatalogStore::new(client)) + as Box) }) }, + server_components: server_components_factory, } } From 2fdff8b616b81a88568d8ae447444c4bfad3d194 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Thu, 30 Jul 2026 16:53:42 -0600 Subject: [PATCH 69/83] chore(mongodb): populate TableKeyInfo secondary-index lists Rebase onto upstream main (6dcb14c), whose per-index consumed-capacity work added global_secondary_indexes and local_secondary_indexes to TableKeyInfo. Load all secondary indexes from the catalog in table_key_info_from_doc and populate both lists (via a new index_info_from_doc helper) so per-index consumed capacity is computed from the cached TableKeyInfo without an extra describe_table per write, matching the postgres backend. has_lsi is now derived from the LSI list. --- Cargo.lock | 1 + crates/storage-mongodb/src/table_engine.rs | 68 +++++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99d25813..574474ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1279,6 +1279,7 @@ dependencies = [ "extenddb-core", "futures", "rand 0.9.5", + "serde", "serde_json", "thiserror", "time", diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 47d02e27..03dedb0f 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -966,12 +966,29 @@ impl MongoEngine { } }); + // Load all secondary indexes so per-index consumed capacity can be + // computed from the cached TableKeyInfo without an extra describe_table + // round-trip per write (matches the fields upstream added). + use futures::TryStreamExt; let indexes_coll = self.catalog_db.collection::("indexes"); - let has_lsi = indexes_coll - .count_documents(doc! { "_id.table_id": &table_id, "index_type": "LSI" }) + let mut idx_cursor = indexes_coll + .find(doc! { "_id.table_id": &table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let mut global_secondary_indexes = Vec::new(); + let mut local_secondary_indexes = Vec::new(); + while let Some(idx_doc) = idx_cursor + .try_next() .await .map_err(|e| StorageError::Internal(e.to_string()))? - > 0; + { + let info = index_info_from_doc(&idx_doc)?; + match info.index_type { + IndexType::Gsi => global_secondary_indexes.push(info), + IndexType::Lsi => local_secondary_indexes.push(info), + } + } + let has_lsi = !local_secondary_indexes.is_empty(); Ok(TableKeyInfo { table_name, @@ -981,6 +998,8 @@ impl MongoEngine { key_schema, attribute_definitions, has_lsi, + global_secondary_indexes, + local_secondary_indexes, stream_specification, }) } @@ -1371,3 +1390,46 @@ impl MongoEngine { Ok(()) } } + +/// Build an `IndexInfo` from an `indexes` catalog document whose `_id` is +/// `{ table_id, index_name }`. Used to populate the GSI/LSI lists carried on +/// `TableKeyInfo` for per-index consumed-capacity computation. +fn index_info_from_doc(index_doc: &Document) -> Result { + let index_name = index_doc + .get_document("_id") + .ok() + .and_then(|id| id.get_str("index_name").ok()) + .ok_or_else(|| StorageError::Internal("missing _id.index_name".to_string()))? + .to_string(); + let index_id = index_doc + .get_str("index_id") + .map_err(|_| StorageError::Internal("missing index_id".to_string()))? + .to_string(); + let index_type = match index_doc.get_str("index_type") { + Ok("GSI") => IndexType::Gsi, + Ok("LSI") => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type: {other:?}" + ))); + } + }; + let key_schema_bson = index_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema in index".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index key_schema parse: {e}")))?; + let projection_bson = index_doc + .get("projection") + .ok_or_else(|| StorageError::Internal("missing projection in index".to_string()))?; + let projection: extenddb_core::types::Projection = bson::from_bson(projection_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index projection parse: {e}")))?; + Ok(IndexInfo { + index_name, + index_id, + index_type, + key_schema, + projection, + }) +} From a7d70fa16892223e3c8b93f6801d0edb9cef216a Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 4 Aug 2026 09:41:06 -0600 Subject: [PATCH 70/83] style(mongodb): satisfy cargo fmt check --- crates/storage-mongodb/src/catalog_store.rs | 1 - crates/storage-mongodb/src/data/mod.rs | 9 ++-- crates/storage-mongodb/src/data_engine.rs | 60 +++++++++------------ crates/storage-mongodb/src/pushdown.rs | 5 +- crates/storage-mongodb/src/table_engine.rs | 16 +++--- 5 files changed, 36 insertions(+), 55 deletions(-) diff --git a/crates/storage-mongodb/src/catalog_store.rs b/crates/storage-mongodb/src/catalog_store.rs index 1fafc0dd..e5f8158c 100644 --- a/crates/storage-mongodb/src/catalog_store.rs +++ b/crates/storage-mongodb/src/catalog_store.rs @@ -38,7 +38,6 @@ impl MongoCatalogStore { pub(crate) fn catalog_db(&self) -> &mongodb::Database { &self.catalog_db } - } // Implement CatalogStore supertrait diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs index 186c55f6..45465ba6 100644 --- a/crates/storage-mongodb/src/data/mod.rs +++ b/crates/storage-mongodb/src/data/mod.rs @@ -5,18 +5,15 @@ //! //! Contains document conversion, collection naming, and key extraction utilities. - use bson::{Document, doc}; +#[cfg(test)] +use extenddb_core::types::KeyType; use extenddb_core::types::{ AttributeDefinition, AttributeValue, Item, KeySchemaElement, ScalarAttributeType, }; -#[cfg(test)] -use extenddb_core::types::KeyType; use extenddb_storage::error::StorageError; -use extenddb_storage::util::{ - composite_pk_to_text, encode_netstring_composite, sk_info, -}; +use extenddb_storage::util::{composite_pk_to_text, encode_netstring_composite, sk_info}; /// Returns the `MongoDB` collection name for a `DynamoDB` table. pub fn data_collection_name(table_id: &str) -> String { diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 7527cdce..074dd7cd 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -21,8 +21,8 @@ use extenddb_storage::util::{ composite_pk_to_text, encode_netstring_composite, pk_to_text, sk_info, }; use extenddb_storage::{ - DataEngine, IdempotencyKey, ItemPairResult, QueryResult, StreamCapture, - TransactGetOp, TransactWriteOp, + DataEngine, IdempotencyKey, ItemPairResult, QueryResult, StreamCapture, TransactGetOp, + TransactWriteOp, }; use crate::MongoEngine; @@ -1709,19 +1709,13 @@ impl MongoEngine { } let refs: Vec> = idx_pairs .iter() - .map( - |(name, ks)| extenddb_core::validation::IndexKeyRef { - index_name: name.as_str(), - key_schema: ks.as_slice(), - }, - ) + .map(|(name, ks)| extenddb_core::validation::IndexKeyRef { + index_name: name.as_str(), + key_schema: ks.as_slice(), + }) .collect(); - extenddb_core::validation::validate_index_keys( - item, - &refs, - &key_info.attribute_definitions, - ) - .map_err(|e| StorageError::Validation(e.to_string())) + extenddb_core::validation::validate_index_keys(item, &refs, &key_info.attribute_definitions) + .map_err(|e| StorageError::Validation(e.to_string())) } async fn sync_indexes_in_session( @@ -2284,23 +2278,20 @@ impl MongoEngine { .await .map_err(TransactOpError::Storage)?; if !idx_pairs.is_empty() { - let idx_refs: Vec> = - idx_pairs - .iter() - .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { - index_name: n.as_str(), - key_schema: ks.as_slice(), - }) - .collect(); + let idx_refs: Vec> = idx_pairs + .iter() + .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { + index_name: n.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); extenddb_core::validation::validate_index_keys( item, &idx_refs, &key_info.attribute_definitions, ) .map_err(|e| { - TransactOpError::Cancel(CancellationReason::validation_error( - e.to_string(), - )) + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; } @@ -2544,23 +2535,20 @@ impl MongoEngine { .await .map_err(TransactOpError::Storage)?; if !idx_pairs.is_empty() { - let idx_refs: Vec> = - idx_pairs - .iter() - .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { - index_name: n.as_str(), - key_schema: ks.as_slice(), - }) - .collect(); + let idx_refs: Vec> = idx_pairs + .iter() + .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { + index_name: n.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); extenddb_core::validation::validate_index_keys( &item, &idx_refs, &key_info.attribute_definitions, ) .map_err(|e| { - TransactOpError::Cancel(CancellationReason::validation_error( - e.to_string(), - )) + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; } diff --git a/crates/storage-mongodb/src/pushdown.rs b/crates/storage-mongodb/src/pushdown.rs index 65a087ad..c2c84c42 100644 --- a/crates/storage-mongodb/src/pushdown.rs +++ b/crates/storage-mongodb/src/pushdown.rs @@ -105,9 +105,8 @@ fn walk(expr: &Expr, maps: &ExpressionMaps) -> Pushable { }; match val { AttributeValue::S(tag) => { - const VALID: &[&str] = &[ - "S", "N", "B", "BOOL", "NULL", "L", "M", "SS", "NS", "BS", - ]; + const VALID: &[&str] = + &["S", "N", "B", "BOOL", "NULL", "L", "M", "SS", "NS", "BS"]; if !VALID.contains(&tag.as_str()) { return Pushable::No("attribute_type tag not a DDB type name"); } diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 03dedb0f..2bde96b4 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -10,10 +10,10 @@ use mongodb::options::{Collation, IndexOptions}; use extenddb_core::types::{ AttributeDefinition, BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, - DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeySchemaElement, - ListTablesInput, ListTablesOutput, LsiDescription, OnDemandThroughput, - ProvisionedThroughputDescription, ScalarAttributeType, SseDescription, SseType, - TableDescription, TableKeyInfo, TableStatus, UpdateTableInput, + DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeySchemaElement, ListTablesInput, + ListTablesOutput, LsiDescription, OnDemandThroughput, ProvisionedThroughputDescription, + ScalarAttributeType, SseDescription, SseType, TableDescription, TableKeyInfo, TableStatus, + UpdateTableInput, }; use extenddb_storage::TableEngine; use extenddb_storage::error::StorageError; @@ -175,9 +175,7 @@ impl MongoEngine { .as_ref() .is_some_and(|ss| ss.stream_enabled) { - Some( - format_stream_label(now), - ) + Some(format_stream_label(now)) } else { None }; @@ -1376,8 +1374,8 @@ impl MongoEngine { // String sort keys need the `simple` collation so range // comparisons behave as byte-wise, matching the query path. - let uses_string_sort = matches!(idx_sk_field, Some((_, true))) - || matches!(base_sk_field, Some("base_sk_s")); + let uses_string_sort = + matches!(idx_sk_field, Some((_, true))) || matches!(base_sk_field, Some("base_sk_s")); let mut opts = IndexOptions::builder().build(); if uses_string_sort { opts.collation = Some(Collation::builder().locale("simple".to_string()).build()); From e2e4b22547124e609c86ca90e15721b3e828f1b8 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 4 Aug 2026 20:09:30 +0000 Subject: [PATCH 71/83] test(mongodb): cover restore reporting ACTIVE before the data copy completes RestoreTableFromBackup calls create_table, which schedules the CREATING to ACTIVE transition on a wall clock, and only then runs the $out copy. The transition worker flips any table whose status_transition_at has elapsed and never consults the copy, so ACTIVE does not imply the restored data is present. A client that waits for ACTIVE can read an empty table. The existing conformance coverage cannot catch this because a small backup finishes copying inside the transition window. This test seeds 40,000 items so the copy outlasts the window, and races an observer against the in-flight restore: the moment DescribeTable first reports ACTIVE, it counts the table. Observed on this branch: ACTIVE with 0 of 40,000 items readable. The test is a race detector by construction, which cuts one way only. It cannot fail when the ordering is correct, because a post-copy ACTIVE always yields a complete count. But a pass is weak evidence: on an idle server the copy can win the race and the defect goes unobserved. This is stated in the module docs so a green run is not read as proof. --- tests/rust/src/main.rs | 2 + tests/rust/src/restore_active_completeness.rs | 181 ++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 tests/rust/src/restore_active_completeness.rs diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index 9ff7cfa4..1482275c 100755 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -70,6 +70,8 @@ mod query_more; #[cfg(test)] mod raw_http; #[cfg(test)] +mod restore_active_completeness; +#[cfg(test)] mod scan; #[cfg(test)] mod select_projection_validation; diff --git a/tests/rust/src/restore_active_completeness.rs b/tests/rust/src/restore_active_completeness.rs new file mode 100644 index 00000000..4eb81b91 --- /dev/null +++ b/tests/rust/src/restore_active_completeness.rs @@ -0,0 +1,181 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A restored table must not report ACTIVE before its data copy completes. +//! +//! DynamoDB's contract: when `DescribeTable` on a restore target first returns +//! ACTIVE, the restored data is fully present. A restore that flips ACTIVE on a +//! control-plane timer decoupled from the copy exposes an empty or partial +//! table to a client that waits-for-ACTIVE. +//! +//! This is a race detector, deliberately. `RestoreTableFromBackup` in this +//! backend blocks until the copy completes, so the ACTIVE-before-data window is +//! only observable from a second client, and whether it is observed depends on +//! whether the copy or the transition timer wins. Two consequences worth +//! knowing before reading a result: +//! +//! - It cannot fail when the ordering is correct. If ACTIVE is only set after +//! the copy drains, the observed count is always complete. +//! - A PASS is not proof the ordering is correct. On an idle server the `$out` +//! copy can finish inside the transition window, and the race is simply not +//! observed. Failures are meaningful; passes are weak evidence. +//! +//! The dataset is sized so the copy takes longer than the transition window on +//! a loaded server. On very fast or idle hardware, raising `ITEMS` widens the +//! window. + +use crate::test_base::*; +use aws_sdk_dynamodb::types::{ + AttributeDefinition, AttributeValue, BillingMode, KeySchemaElement, KeyType, PutRequest, + ScalarAttributeType, Select, WriteRequest, +}; + +const ITEMS: usize = 40000; + +/// Bound on the observer's wait for first ACTIVE, so a failed restore fails the +/// test rather than hanging it. 10ms per attempt, so this is a 60s ceiling. +const OBSERVER_MAX_ATTEMPTS: usize = 6000; + +#[tokio::test] +async fn restored_table_has_all_items_when_first_active() { + let c = client(); + let src = format!("RestoreRaceSrc_{}", ts()); + c.create_table() + .table_name(&src) + .key_schema( + KeySchemaElement::builder() + .attribute_name("pk") + .key_type(KeyType::Hash) + .build() + .unwrap(), + ) + .attribute_definitions( + AttributeDefinition::builder() + .attribute_name("pk") + .attribute_type(ScalarAttributeType::S) + .build() + .unwrap(), + ) + .billing_mode(BillingMode::PayPerRequest) + .send() + .await + .unwrap(); + wait_for_active(&c, &src).await; + + // Enough data that the restore copy takes longer than the control-plane + // transition delay, so a timer-driven ACTIVE flip would win the race. + let pad = "x".repeat(2000); + for chunk in (0..ITEMS).collect::>().chunks(25) { + let reqs: Vec = chunk + .iter() + .map(|i| { + WriteRequest::builder() + .put_request( + PutRequest::builder() + .item("pk", AttributeValue::S(format!("k{i:06}"))) + .item("d", AttributeValue::S(pad.clone())) + .build() + .unwrap(), + ) + .build() + }) + .collect(); + c.batch_write_item() + .request_items(&src, reqs) + .send() + .await + .unwrap(); + } + + let backup = c + .create_backup() + .table_name(&src) + .backup_name("restore-race-probe") + .send() + .await + .unwrap(); + let arn = backup.backup_details().unwrap().backup_arn().to_string(); + // Wait until the backup is AVAILABLE. + for _ in 0..240 { + let d = c.describe_backup().backup_arn(&arn).send().await.unwrap(); + if d.backup_description() + .and_then(|b| b.backup_details()) + .map(|b| b.backup_status().as_str() == "AVAILABLE") + .unwrap_or(false) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + + let dst = format!("RestoreRaceDst_{}", ts()); + // Race an observer against the in-flight restore: the moment it sees + // ACTIVE, it counts. Returns None if ACTIVE never arrives within the + // bound, which means the restore itself failed. + let observer = { + let c2 = client().clone(); + let dst2 = dst.clone(); + tokio::spawn(async move { + let mut saw_active = false; + for _ in 0..OBSERVER_MAX_ATTEMPTS { + if let Ok(out) = c2.describe_table().table_name(&dst2).send().await { + let status = out + .table() + .and_then(|t| t.table_status()) + .map(|s| s.as_str().to_owned()); + if status.as_deref() == Some("ACTIVE") { + saw_active = true; + break; + } + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + if !saw_active { + return None; + } + // First ACTIVE observation: count items immediately. + let mut count = 0usize; + let mut start_key = None; + loop { + let mut req = c2.scan().table_name(&dst2).select(Select::Count); + if let Some(k) = start_key.take() { + req = req.set_exclusive_start_key(Some(k)); + } + let resp = match req.send().await { + Ok(r) => r, + Err(_) => break, + }; + count += resp.count() as usize; + match resp.last_evaluated_key() { + Some(k) if !k.is_empty() => start_key = Some(k.clone()), + _ => break, + } + } + Some(count) + }) + }; + + c.restore_table_from_backup() + .target_table_name(&dst) + .backup_arn(&arn) + .send() + .await + .unwrap(); + + // The concurrent observer counted at first-ACTIVE while the restore call + // above was still in flight (or just after, if the copy was fast). + let observed = observer.await.unwrap(); + + c.delete_table().table_name(&src).send().await.ok(); + c.delete_table().table_name(&dst).send().await.ok(); + + let count = observed.expect( + "restore target never reported ACTIVE within the observer bound, \ + so the restore itself did not complete", + ); + assert_eq!( + count, ITEMS, + "restored table reported ACTIVE with {count}/{ITEMS} items present. \ + ACTIVE must imply the restore copy is complete" + ); +} From 4864b77caf68ff0cef269601ee3c0837bb656ba4 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 4 Aug 2026 14:27:29 -0600 Subject: [PATCH 72/83] fix(mongodb): set restored table ACTIVE only after the data copy completes restore_table_from_backup created the table with a scheduled CREATING -> ACTIVE transition, then ran the $out copy. The transition is a wall-clock timer (now + control_plane_delay_seconds), not tied to the copy, so on a large restore the table went ACTIVE while $out was still running and a client waiting for ACTIVE could read an empty table. Add a defer_active flag to create_table_impl so the restore path creates the table CREATING with no scheduled transition, and set the table ACTIVE directly once $out drains. ACTIVE now implies the copy is complete by code ordering, not timing. No control-plane delay is applied on restore -- the copy is itself the CREATING window (unlike CreateTable, whose instant work needs a synthetic delay). Removes the now-inaccurate comment. --- crates/storage-mongodb/src/backup_engine.rs | 32 ++++++++++++++------- crates/storage-mongodb/src/table_engine.rs | 18 ++++++++++-- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index 3c200b46..d00f1210 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -23,7 +23,6 @@ use extenddb_core::types::{ KeySchemaElement, PointInTimeRecoveryDescription, SourceTableDetails, TableDescription, }; use extenddb_storage::BackupEngine; -use extenddb_storage::TableEngine; use extenddb_storage::error::StorageError; use crate::MongoEngine; @@ -486,7 +485,12 @@ impl BackupEngine for MongoEngine { on_demand_throughput, }; - let desc = self.create_table(&account_id, create_input).await?; + // Create the table with the ACTIVE transition deferred: it enters + // CREATING with no scheduled flip, so the table cannot become + // ACTIVE until we schedule it below, after the data copy completes. + let desc = self + .create_table_impl(&account_id, create_input, true) + .await?; // Restore items from the backup collection using server-side `$out`. // The backup collection was written by `create_backup` in the same @@ -519,18 +523,26 @@ impl BackupEngine for MongoEngine { .map_err(|e| StorageError::Internal(e.to_string()))? as i64; - // Update the item count. The table was created via `create_table`, - // so it is already in CREATING (with a scheduled transition) when - // control_plane_delay_seconds > 0, or ACTIVE when it is 0; the - // control_plane_worker flips CREATING -> ACTIVE once the window - // passes. The data was just copied above, so it is in place before - // the table becomes ACTIVE. `desc` (returned to the caller) already - // carries the CREATING status from create_table, matching DynamoDB. + // Now that the data is fully copied, record the item count and + // release the table from CREATING. The table was created with the + // transition deferred (no scheduled flip), so this is the first + // point at which it can become ACTIVE — which is exactly the + // ordering we want: ACTIVE now implies the copy is complete. + // + // No control-plane delay is applied: unlike CreateTable (whose real + // work is instant and needs a synthetic delay to make CREATING + // observable), the copy is itself the CREATING window. `desc` + // (returned to the caller) carries CREATING from create_table_impl, + // matching DynamoDB, which reports CREATING while a restore runs. + let status_update = doc! { + "$set": { "item_count": item_count, "table_status": "ACTIVE" }, + "$unset": { "status_transition_at": "" }, + }; let tables_coll = self.catalog_db.collection::("tables"); tables_coll .update_one( doc! { "_id": { "account_id": &account_id, "table_name": &target_table_name } }, - doc! { "$set": { "item_count": item_count } }, + status_update, ) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 2bde96b4..8e033ccf 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -50,7 +50,7 @@ impl TableEngine for MongoEngine { input: CreateTableInput, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); - Box::pin(async move { self.create_table_impl(&account_id, input).await }) + Box::pin(async move { self.create_table_impl(&account_id, input, false).await }) } fn delete_table( @@ -132,10 +132,16 @@ impl TableEngine for MongoEngine { } impl MongoEngine { - async fn create_table_impl( + /// Create a table. When `defer_active` is set (the restore path), the row + /// is written `CREATING` with **no** scheduled transition, so the + /// background worker will not flip it to `ACTIVE`; the caller schedules the + /// transition only after it has finished populating the table. Normal + /// `CreateTable` passes `false` and gets the usual timed transition. + pub(crate) async fn create_table_impl( &self, account_id: &str, input: CreateTableInput, + defer_active: bool, ) -> Result { Self::validate_account_id(account_id)?; @@ -207,7 +213,13 @@ impl MongoEngine { // table return ResourceNotFound, matching DynamoDB and the postgres // backend. let delay_secs = self.control_plane_delay_seconds().await; - let (table_status, status_transition_at): (&str, bson::Bson) = if delay_secs <= 0.0 { + let (table_status, status_transition_at): (&str, bson::Bson) = if defer_active { + // Restore path: enter CREATING with no scheduled transition. The + // caller flips the table to ACTIVE (or schedules the transition) + // only after the data copy completes, so ACTIVE never precedes a + // populated table. + ("CREATING", bson::Bson::Null) + } else if delay_secs <= 0.0 { ("ACTIVE", bson::Bson::Null) } else { let at = bson::DateTime::now().timestamp_millis() + (delay_secs * 1000.0) as i64; From 7659a7d3c753524604ac6ea2e3b0d824dbfdf842 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 4 Aug 2026 20:11:26 -0600 Subject: [PATCH 73/83] fix(mongodb): harden connection and credential handling - Fail closed when the encryption key is missing: loading it with unwrap_or_default() made a missing key an empty string, which panics in aes_gcm (32-byte key required). Return MissingEncryptionKey, like postgres. - Apply the readPreference=primary rejection to every client via a shared connect_guarded(); previously only the data client was guarded, so the catalog/auth/settings/diagnostics/bootstrapper clients bypassed it. Gate the no-TLS warning to the server data client so short-lived CLI/management clients dont emit it -- it was leaking onto command stdout that tooling parses (it corrupted the settings value read by the GSI-async tests). --- crates/storage-mongodb/src/bootstrapper.rs | 6 +- crates/storage-mongodb/src/lib.rs | 107 ++++++++++++--------- 2 files changed, 66 insertions(+), 47 deletions(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index 21196e3f..32ecebb4 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -40,9 +40,9 @@ impl MongoBootstrapper { .unwrap_or("mongodb://localhost:27017") .to_string(); - let client = mongodb::Client::with_uri_str(&connection_string) - .await - .map_err(|e| StorageError::Connection(e.to_string()))?; + // Route through the shared guard so init/destroy/migrate also reject + // non-primary read preferences and warn on missing TLS. + let client = crate::connect_guarded(&connection_string, None, false).await?; Ok(Self { client, diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index cd859af1..befbeb0e 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -99,14 +99,17 @@ fn server_components_factory( let engine = Arc::new(engine); // Create catalog store - let catalog_client = mongodb::Client::with_uri_str(&connection_string) + let catalog_client = connect_guarded(&connection_string, None, false) .await .map_err(|e| BackendError::ConnectionFailed { backend: "mongodb".to_string(), details: format!("Failed to create catalog client: {e}"), })?; - // Load encryption key from settings collection + // Load encryption key from settings collection. A missing key must be + // a hard failure: an empty key would base64-decode to zero bytes and + // panic in aes_gcm (`Key::from_slice` requires 32 bytes). Refuse to + // start instead, matching the postgres backend. let catalog_db = catalog_client.database("extenddb_catalog"); let settings_coll = catalog_db.collection::("settings"); let enc_key = settings_coll @@ -114,7 +117,7 @@ fn server_components_factory( .await .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) - .unwrap_or_default(); + .ok_or(BackendError::MissingEncryptionKey)?; let catalog_store = Arc::new(MongoCatalogStore::with_encryption_key( catalog_client, @@ -124,7 +127,7 @@ fn server_components_factory( // Create credential store. The bin layer wraps this in // CachedCredentialStore using the operator-configured TTL // before constructing the auth provider. - let auth_client = mongodb::Client::with_uri_str(&connection_string) + let auth_client = connect_guarded(&connection_string, None, false) .await .map_err(|e| BackendError::InitializationFailed(format!("Auth client: {e}")))?; let cred_store: Arc = @@ -166,7 +169,7 @@ pub fn backend() -> extenddb_storage::Backend { settings_store: |connection_string| { let connection_string = connection_string.to_string(); Box::pin(async move { - let client = mongodb::Client::with_uri_str(&connection_string) + let client = connect_guarded(&connection_string, None, false) .await .map_err(|e| { extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed( @@ -182,7 +185,7 @@ pub fn backend() -> extenddb_storage::Backend { diagnostics_store: |connection_string| { let connection_string = connection_string.to_string(); Box::pin(async move { - let client = mongodb::Client::with_uri_str(&connection_string) + let client = connect_guarded(&connection_string, None, false) .await .map_err(|e| { extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed( @@ -225,50 +228,66 @@ pub struct MongoEngine { gsi_cache: dashmap::DashMap, } +/// Build a MongoDB client from a connection string, applying the shared +/// connection guards so every client in the backend is protected, not just the +/// data client: reject non-primary read preferences (they route reads to +/// replicas and silently break `ConsistentRead=true`). +/// +/// `max_pool_size` is applied when provided (the data client sizes its pool; +/// catalog/auth/bootstrapper clients pass `None`). +/// +/// `warn_on_no_tls` gates the no-TLS warning to the long-running server data +/// client only. Short-lived CLI/management clients (settings, catalog checks, +/// bootstrapper) pass `false`: they all share the same connection string, so a +/// single warning at server startup is enough, and emitting it on every CLI +/// invocation both spams logs and pollutes command stdout that tooling parses. +pub(crate) async fn connect_guarded( + connection_string: &str, + max_pool_size: Option, + warn_on_no_tls: bool, +) -> Result { + let mut options = mongodb::options::ClientOptions::parse(connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + if let Some(n) = max_pool_size { + options.max_pool_size = Some(n); + } + + if let Some(sel) = options.selection_criteria.as_ref() { + use mongodb::options::{ReadPreference, SelectionCriteria}; + let is_non_primary = match sel { + SelectionCriteria::ReadPreference(rp) => !matches!(rp, ReadPreference::Primary), + _ => false, + }; + if is_non_primary { + return Err(StorageError::Connection( + "MongoDB connection string must use readPreference=primary. \ + Non-primary read preferences (secondary, secondaryPreferred, \ + nearest, primaryPreferred) route reads to replicas and \ + silently break ConsistentRead=true." + .to_owned(), + )); + } + } + + if warn_on_no_tls && !matches!(options.tls, Some(mongodb::options::Tls::Enabled(_))) { + tracing::warn!( + "MongoDB connection is not using TLS; credentials and data will \ + traverse the network in cleartext. Enable TLS with `?tls=true` \ + in the connection string, or use a `mongodb+srv://` URI." + ); + } + + mongodb::Client::with_options(options).map_err(|e| StorageError::Connection(e.to_string())) +} + impl MongoEngine { pub async fn new( connection_string: &str, region: &str, max_connections: u32, ) -> Result { - let mut options = mongodb::options::ClientOptions::parse(connection_string) - .await - .map_err(|e| StorageError::Connection(e.to_string()))?; - options.max_pool_size = Some(max_connections); - - // Reject non-primary read preferences. DynamoDB's `ConsistentRead=true` - // requires linearizable reads; MongoDB's Primary read concern is the - // only mode that provides that. A connection string like - // `mongodb://.../?readPreference=secondaryPreferred` would silently - // route reads to a secondary and return stale data — a fidelity - // violation the caller has no way to detect. - if let Some(sel) = options.selection_criteria.as_ref() { - use mongodb::options::{ReadPreference, SelectionCriteria}; - let is_non_primary = match sel { - SelectionCriteria::ReadPreference(rp) => !matches!(rp, ReadPreference::Primary), - _ => false, - }; - if is_non_primary { - return Err(StorageError::Connection( - "MongoDB connection string must use readPreference=primary. \ - Non-primary read preferences (secondary, secondaryPreferred, \ - nearest, primaryPreferred) route reads to replicas and \ - silently break ConsistentRead=true." - .to_owned(), - )); - } - } - - if !matches!(options.tls, Some(mongodb::options::Tls::Enabled(_))) { - tracing::warn!( - "MongoDB connection is not using TLS; credentials and data will \ - traverse the network in cleartext. Enable TLS with `?tls=true` \ - in the connection string, or use a `mongodb+srv://` URI." - ); - } - - let client = mongodb::Client::with_options(options) - .map_err(|e| StorageError::Connection(e.to_string()))?; + let client = connect_guarded(connection_string, Some(max_connections), true).await?; let catalog_db = client.database("extenddb_catalog"); let data_db = client.database("extenddb_data"); From 70c6124b7ac02a90e32d3e675d43ec47629904cc Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 4 Aug 2026 20:11:49 -0600 Subject: [PATCH 74/83] fix(mongodb): stop deriving Serialize on MongoStorageConfig connection_string may carry user:pass@ credentials; a Serialize impl let them leave the process on any serialize path. Drop the derive (nothing serializes the config), matching postgres which derives only Debug, Clone, Deserialize. --- crates/storage-mongodb/src/config.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/storage-mongodb/src/config.rs b/crates/storage-mongodb/src/config.rs index 9e5a4e26..bb878029 100644 --- a/crates/storage-mongodb/src/config.rs +++ b/crates/storage-mongodb/src/config.rs @@ -3,10 +3,15 @@ //! Configuration for `MongoDB` storage backend. -use serde::{Deserialize, Serialize}; +use serde::Deserialize; /// `MongoDB` storage backend configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// +/// Deliberately does **not** derive `Serialize`: `connection_string` may carry +/// `user:pass@` credentials, and a `Serialize` impl would let them leave the +/// process on any serialize path. Matches the postgres backend, which derives +/// only `Debug, Clone, Deserialize`. +#[derive(Debug, Clone, Deserialize)] pub struct MongoStorageConfig { /// `MongoDB` connection string (mongodb://...) pub connection_string: String, From c845e7e84c293442773d3e7a7b10a00cac4fae69 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 4 Aug 2026 20:12:17 -0600 Subject: [PATCH 75/83] fix(mongodb): scope restore backup lookup to the account restore_table_from_backup looked up the backup by ARN with no account predicate. The engine layer already enforces ARN ownership, so this is defence-in-depth, aligning restore with the account-scoped describe/delete backup paths. --- crates/storage-mongodb/src/backup_engine.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index d00f1210..1febebbd 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -411,7 +411,14 @@ impl BackupEngine for MongoEngine { Box::pin(async move { let backups_coll = self.catalog_db.collection::("backups"); let backup_doc = backups_coll - .find_one(doc! { "_id": &backup_arn, "backup_status": "AVAILABLE" }) + // Scope to the calling account (defence-in-depth: the engine + // layer already enforces ARN ownership, and describe/delete are + // account-scoped at the storage layer too). + .find_one(doc! { + "_id": &backup_arn, + "account_id": &account_id, + "backup_status": "AVAILABLE", + }) .await .map_err(|e| StorageError::Internal(e.to_string()))? .ok_or_else(|| { From c83607416d3ab38cbdd186699f36ebf1efa22770 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Tue, 4 Aug 2026 21:13:31 -0600 Subject: [PATCH 76/83] fix(devtools): isolate run-mongodb-tests container and output per run CONTAINER_NAME and the default OUTPUT_DIR were shared across runs, so two concurrent invocations (even on different ports) would docker rm -f each others mongo and overwrite logs. Derive the container name from the mongo port and the output dir from the port plus PID so runs stay isolated. --- devtools/run-mongodb-tests | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/devtools/run-mongodb-tests b/devtools/run-mongodb-tests index d7dfff23..582fb207 100755 --- a/devtools/run-mongodb-tests +++ b/devtools/run-mongodb-tests @@ -66,14 +66,18 @@ if [[ ${#RUN_TESTS_ARGS[@]} -eq 0 ]]; then RUN_TESTS_ARGS=(--pytest --comprehensive --parallel) fi +# Derive the output dir and container name from the mongo port (plus PID for +# the dir) so concurrent runs on different ports do not collide: a shared +# container name + a shared `docker rm -f` at startup would let one run tear +# down another run's mongo and overwrite its logs. if [[ -z "$OUTPUT_DIR" ]]; then - OUTPUT_DIR="/tmp/run-mongodb-tests-$(date +%Y%m%d-%H%M%S)" + OUTPUT_DIR="/tmp/run-mongodb-tests-${MONGO_PORT}-$(date +%Y%m%d-%H%M%S)-$$" fi mkdir -p "$OUTPUT_DIR" CONFIG="$OUTPUT_DIR/extenddb.toml" SERVER_LOG="$OUTPUT_DIR/server.log" -CONTAINER_NAME="extenddb-runtests-mongo" +CONTAINER_NAME="extenddb-runtests-mongo-${MONGO_PORT}" BINARY="./target/release/extenddb" if [[ ! -x "$BINARY" ]]; then From c4686de816810054e68bc5da54f6374f0d1924f4 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 5 Aug 2026 10:21:01 -0600 Subject: [PATCH 77/83] test(rust): binary begins_with edges and field-vs-field conditions Cover DynamoDB wire behaviors our MongoDB fixes touched that the suite did not otherwise pin: - begins_with on a binary sort key by unsigned byte prefix, plus the all-0xFF upper-bound overflow edge and the empty-prefix whole-partition edge. - Condition expressions whose comparison operands are both document paths (field-vs-field), evaluated as stored values. Both files are dual-target, so PostgreSQL and real DynamoDB run them too. --- tests/rust/src/binary_sort_key.rs | 104 +++++++++++++++++++ tests/rust/src/condition_expressions_more.rs | 61 +++++++++++ 2 files changed, 165 insertions(+) diff --git a/tests/rust/src/binary_sort_key.rs b/tests/rust/src/binary_sort_key.rs index 3bc76c15..a139f287 100644 --- a/tests/rust/src/binary_sort_key.rs +++ b/tests/rust/src/binary_sort_key.rs @@ -166,3 +166,107 @@ async fn query_binary_sort_key_between_uses_byte_order() { let _ = c.delete_table().table_name(&table).send().await; } + +#[tokio::test] +async fn query_binary_sort_key_begins_with_uses_byte_prefix() { + let c = client(); + let table = format!("BinSkBeginsWith_{}", ts()); + create_binary_sk_table(c, &table).await; + + let pk = "p"; + seed( + c, + &table, + pk, + &[&[0x01, 0x00], &[0x01, 0xFF], &[0x02, 0x00]], + ) + .await; + + // begins_with(sk, [0x01]) must select exactly the two 0x01-prefixed keys, + // matched by raw unsigned byte prefix (not string/UTF-8 semantics), and + // exclude the 0x02 key. + let resp = c + .query() + .table_name(&table) + .key_condition_expression("#h = :hv AND begins_with(#r, :pfx)") + .expression_attribute_names("#h", "pk") + .expression_attribute_names("#r", "sk") + .expression_attribute_values(":hv", s(pk)) + .expression_attribute_values(":pfx", bb(&[0x01])) + .send() + .await + .unwrap(); + assert_eq!( + sk_bytes(resp.items()), + vec![vec![0x01, 0x00], vec![0x01, 0xFF]], + "begins_with on a binary sort key must match by unsigned byte prefix" + ); + + let _ = c.delete_table().table_name(&table).send().await; +} + +#[tokio::test] +async fn query_binary_sort_key_begins_with_all_ff_prefix() { + let c = client(); + let table = format!("BinSkBeginsFf_{}", ts()); + create_binary_sk_table(c, &table).await; + + let pk = "p"; + // A 0xFF-prefixed key is the largest possible under unsigned byte order. + // begins_with([0xFF]) exercises the upper-bound edge: a naive "increment + // the last prefix byte" range end overflows past 0xFF and must instead + // extend to the end of the partition. The 0xFE key must be excluded. + seed(c, &table, pk, &[&[0xFE], &[0xFF, 0x00], &[0xFF, 0xFF]]).await; + + let resp = c + .query() + .table_name(&table) + .key_condition_expression("#h = :hv AND begins_with(#r, :pfx)") + .expression_attribute_names("#h", "pk") + .expression_attribute_names("#r", "sk") + .expression_attribute_values(":hv", s(pk)) + .expression_attribute_values(":pfx", bb(&[0xFF])) + .send() + .await + .unwrap(); + assert_eq!( + sk_bytes(resp.items()), + vec![vec![0xFF, 0x00], vec![0xFF, 0xFF]], + "begins_with([0xFF]) must select all 0xFF-prefixed keys and exclude 0xFE" + ); + + let _ = c.delete_table().table_name(&table).send().await; +} + +#[tokio::test] +async fn query_binary_sort_key_begins_with_empty_prefix() { + let c = client(); + let table = format!("BinSkBeginsEmpty_{}", ts()); + create_binary_sk_table(c, &table).await; + + let pk = "p"; + // The empty prefix is a prefix of every value, so begins_with([]) has no + // upper bound and must return the entire partition (lower bound "" with no + // exclusive end), in unsigned byte order — the same result an unbounded + // Query would give. + seed(c, &table, pk, &[&[0x00], &[0x7F], &[0xFF, 0xFF]]).await; + + let resp = c + .query() + .table_name(&table) + .key_condition_expression("#h = :hv AND begins_with(#r, :pfx)") + .expression_attribute_names("#h", "pk") + .expression_attribute_names("#r", "sk") + .expression_attribute_values(":hv", s(pk)) + .expression_attribute_values(":pfx", bb(&[])) + .send() + .await + .unwrap(); + assert_eq!( + sk_bytes(resp.items()), + vec![vec![0x00], vec![0x7F], vec![0xFF, 0xFF]], + "begins_with([]) must return the whole partition in unsigned byte order" + ); + + let _ = c.delete_table().table_name(&table).send().await; +} diff --git a/tests/rust/src/condition_expressions_more.rs b/tests/rust/src/condition_expressions_more.rs index 1efa81ad..f7d25a36 100755 --- a/tests/rust/src/condition_expressions_more.rs +++ b/tests/rust/src/condition_expressions_more.rs @@ -288,3 +288,64 @@ async fn condition_nested_parentheses() { .await .unwrap(); } + +// ========== Field-vs-field comparison (both operands are paths) ========== + +#[tokio::test] +async fn condition_field_vs_field_true() { + let c = client(); + let t = tables().await; + let table = &t.simple_key_string; + let mut item = create_item(table); + item.insert("lo".into(), n(5)); + item.insert("hi".into(), n(10)); + c.put_item() + .table_name(table) + .set_item(Some(item.clone())) + .send() + .await + .unwrap(); + + // Both operands are document paths: `lo < hi` (5 < 10) holds, so the + // delete proceeds. Exercises a comparison whose right-hand side is an + // attribute reference rather than an expression-attribute-value. + let key = get_key(table, &item); + c.delete_item() + .table_name(table) + .set_key(Some(key)) + .condition_expression("lo < hi") + .send() + .await + .unwrap(); +} + +#[tokio::test] +async fn condition_field_vs_field_false() { + let c = client(); + let t = tables().await; + let table = &t.simple_key_string; + let mut item = create_item(table); + item.insert("lo".into(), n(5)); + item.insert("hi".into(), n(10)); + c.put_item() + .table_name(table) + .set_item(Some(item.clone())) + .send() + .await + .unwrap(); + + // `hi < lo` (10 < 5) is false, so the guard must reject the write. Pins + // that field-vs-field comparisons evaluate both sides as stored values, + // not lexically. + let key = get_key(table, &item); + let err = c + .delete_item() + .table_name(table) + .set_key(Some(key)) + .condition_expression("hi < lo") + .send() + .await + .unwrap_err(); + + assert_eq!(err_code(&err), Some("ConditionalCheckFailedException")); +} From 64c0d8b6b061d89dc35f9278d524c21ea1704461 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 5 Aug 2026 10:22:01 -0600 Subject: [PATCH 78/83] ci(mongodb): add MongoDB integration workflow Run the MongoDB pytest and rust integration suites as parallel jobs joined by a gate, mirroring integration.yml. Each job delegates to devtools/run-mongodb-tests, which bootstraps the single-node replica set (rs.initiate + wait-for-PRIMARY) that GitHub services: cannot express, then reuses the exact local test path to avoid CI/dev drift. --- .github/workflows/integration-mongodb.yml | 67 +++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/integration-mongodb.yml diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml new file mode 100644 index 00000000..01228ef8 --- /dev/null +++ b/.github/workflows/integration-mongodb.yml @@ -0,0 +1,67 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +name: MongoDB Integration Tests + +on: + pull_request: + merge_group: + push: + branches: [main] + +permissions: + contents: read + +# The MongoDB backend needs a replica set (single-node --replSet + rs.initiate() +# + wait-for-PRIMARY) for transactions and streams, which GitHub `services:` +# cannot express. `devtools/run-mongodb-tests` performs that bootstrap, then +# init/serve/provision extenddb and delegates the workload to +# `run-tests --backend mongodb`, tearing everything down on exit. Both jobs +# reuse it so CI runs the exact path developers run locally (no drift), while +# splitting pytest and rust into parallel jobs the way integration.yml does. +jobs: + mongodb-pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build release (mongodb backend) + run: cargo build --release --features mongodb + - name: Run MongoDB pytest suite + run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel + + mongodb-rust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build release (mongodb backend) + run: cargo build --release --features mongodb + - name: Run MongoDB rust integration suite + run: devtools/run-mongodb-tests -- --rust --rust-integration + + mongodb-integration: + runs-on: ubuntu-latest + needs: [mongodb-pytest, mongodb-rust] + if: always() + steps: + - run: | + if [ "${{ needs.mongodb-pytest.result }}" != "success" ] || \ + [ "${{ needs.mongodb-rust.result }}" != "success" ]; then + exit 1 + fi From 83e40cca0ff0b6f6696e2c01c0078572f0b36d41 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 5 Aug 2026 13:49:45 -0600 Subject: [PATCH 79/83] fix(mongodb): warn when GSI backfill returns an empty, non-final batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backfill loops empty-but-not-done branch returned Ok(()) silently, leaving the index in CREATING to be retried each interval. That path should not occur (backfill_gsi_batch marks done when it scans fewer than batch_size docs), so emit a warn instead of failing closed silently — a persistent occurrence now surfaces as a GSI stuck in CREATING. --- crates/storage-mongodb/src/ttl_worker.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index 5a45ff52..c8cd87c6 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -176,9 +176,16 @@ async fn run_gsi_backfill_job(storage: &MongoEngine, job: &Document) -> Result<( .map_err(|e| StorageError::Internal(e.to_string()))?; cursor = Some(last_id.clone()); } else { - // Empty batch but not done — treat as done to avoid an - // infinite loop. Shouldn't happen in practice since - // backfill_gsi_batch marks done when scanned < batch_size. + // Empty batch but the scan did not report completion. This + // shouldn't happen — backfill_gsi_batch marks `done` whenever it + // scans fewer than batch_size docs — so surface it rather than + // silently returning: the index stays CREATING and this worker + // will re-pick it up on the next interval, so a persistent + // occurrence means a GSI is stuck in CREATING. + tracing::warn!( + "GSI backfill worker: index_id={index_id} returned an empty, \ + non-final batch; index remains CREATING and will be retried", + ); return Ok(()); } } From 1a3d41ac5c8feba67cdde6248454e7b34148692b Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 5 Aug 2026 13:50:28 -0600 Subject: [PATCH 80/83] docs(mongodb): document f64-precision bound on inverted BETWEEN keys The sort-key BETWEEN inversion guard compares numeric bounds via f64. f64 rounding is monotonic, so a valid range is never wrongly rejected; the only gap is a genuinely inverted range distinguishable only beyond f64s ~15-17 significant digits, which returns an empty result instead of DynamoDBs ValidationException. Spell out the boundary in the code comment and record it in differences-from-dynamodb.md. --- crates/storage-mongodb/src/data_engine.rs | 13 ++++++++++--- docs/differences-from-dynamodb.md | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 074dd7cd..eb58495d 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -3300,9 +3300,16 @@ fn build_sk_filter( /// /// The comparison is done in the source AttributeValue domain so it happens before /// any Decimal128/f64 conversion that could mask ordering. Strings are compared -/// lexicographically (matching DynamoDB), numbers via f64 (adequate for ordering — -/// values exceeding Decimal128 range are rejected downstream in `sk_to_bson`), and -/// binary bytewise. +/// lexicographically (matching DynamoDB) and binary bytewise. +/// +/// Numbers are compared via `f64`. f64→nearest rounding is monotonic, so this can +/// never make a valid `low <= high` range look inverted (no false ValidationException): +/// if `low <= high` then `low as f64 <= high as f64`. The only imprecision is the +/// reverse — a genuinely inverted range whose bounds differ only beyond f64's ~15–17 +/// significant digits (DynamoDB numbers carry up to 38) rounds to equal and slips +/// past this guard. In that pathological case the `$gte low > $lte high` query simply +/// returns an empty result instead of the ValidationException DynamoDB would raise. +/// This bounded divergence is documented in `docs/differences-from-dynamodb.md`. fn sk_between_low_gt_high(low: &AttributeValue, high: &AttributeValue) -> bool { match (low, high) { (AttributeValue::S(l), AttributeValue::S(h)) => l > h, diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index f92f4222..3922260d 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -13,6 +13,7 @@ adaptation when switching between ExtendDB and the real service. | DAX (Accelerator) | In-memory caching layer | Not applicable | | PartiQL | ExecuteStatement, BatchExecuteStatement | Not implemented (returns UnknownOperationException) | | Numeric precision on partition/sort keys (MongoDB backend only) | 38 significant digits | 34 significant digits (BSON Decimal128). Values that exceed this precision are rejected at write and query time with a ValidationException rather than silently downcast. PostgreSQL backend supports the full 38 digits. | +| Inverted numeric `BETWEEN` on a sort key (MongoDB backend only) | ValidationException ("The BETWEEN operator requires upper bound to be greater than or equal to lower bound") | Same error in all practical cases. The inversion guard compares bounds via `f64`, so a `KeyConditionExpression` `BETWEEN` whose bounds are inverted only beyond f64's ~15–17 significant digits (e.g. `BETWEEN 10000000000000002 AND 10000000000000001`) is not rejected and returns an empty result set instead. Valid ranges are never wrongly rejected. | ## Authentication and Authorization (AWS IAM/STS auth surface used by DynamoDB) From 56a8769bf701cfc1680b74b89403bf960394f2b8 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 5 Aug 2026 13:51:05 -0600 Subject: [PATCH 81/83] docs(mongodb): correct CI/test description and bump design doc to 7.0 The RFC and design doc claimed integration tests run as `cargo test -p extenddb-storage-mongodb` and described a CI job that did not match reality. Update both to describe the actual setup: the dual-target tests/rust suite and pytest run via devtools/run-mongodb-tests from .github/workflows/integration-mongodb.yml. Also bump the two remaining "6.0" minimum-version references in the design doc to 7.0. --- docs/design/13-storage-mongodb.md | 21 +++++++++++++-------- docs/rfcs/0000-mongodb-backend.md | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index 55ef6ef5..4faced22 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -11,7 +11,7 @@ traits (`ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, **Driver:** `mongodb` (official Rust driver, async, multi-document ACID transactions on replica sets). -**Minimum MongoDB version:** 6.0 (multi-document transactions, snapshot reads). +**Minimum MongoDB version:** 7.0 (multi-document transactions, snapshot reads). **Read preference:** `primary` only. `MongoEngine::new` rejects connection strings that request `secondary`, `secondaryPreferred`, `primaryPreferred`, or `nearest` — @@ -1024,17 +1024,22 @@ The backend implements every trait in `extenddb-storage`: - **Property tests:** `tests/pushdown_parity.rs` — random items and expressions checked for agreement between the compiled BSON filter and `evaluate_condition`. -- **Integration tests:** Single-node replica set in Docker - (`mongod --replSet rs0`), full trait coverage. +- **Integration tests:** The dual-target `tests/rust/` SDK suite (the + same AWS-SDK wire tests the PostgreSQL backend runs) executed against a + MongoDB-backed server on a single-node replica set in Docker + (`mongod --replSet rs0`), via `devtools/run-mongodb-tests -- --rust + --rust-integration`. - **Existing pytest suite:** Passes unchanged (backend-agnostic wire - protocol tests). -- **CI:** GitHub Actions job with MongoDB 7.0 replica set, runs - `cargo test -p extenddb-storage-mongodb` then `devtools/run-tests - --extenddb --pytest --external`. + protocol tests), via `devtools/run-mongodb-tests -- --pytest + --comprehensive --parallel`. +- **CI:** `.github/workflows/integration-mongodb.yml` runs the pytest and + rust-integration suites as two parallel jobs, each building with + `--features mongodb` and delegating to `devtools/run-mongodb-tests`, + which bootstraps a single-node MongoDB 7.0 replica set. ## 14. Deployment Requirements -- MongoDB **6.0+** in **replica set** mode. Standalone nodes reject +- MongoDB **7.0+** in **replica set** mode. Standalone nodes reject multi-document transactions. - **`readPreference=primary`** on the connection string. Non-primary is rejected at engine startup. diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md index 248160b8..a21b6536 100644 --- a/docs/rfcs/0000-mongodb-backend.md +++ b/docs/rfcs/0000-mongodb-backend.md @@ -269,11 +269,11 @@ Testing is organized in three layers. **Unit tests** cover pure logic without a live MongoDB instance: netstring composite `_id` encoding, hex sort-key ordering, condition filter compilation, pushdown-analyzer decisions, sequence-number formatting, stream shard-id derivation. Property tests (`crates/storage-mongodb/tests/pushdown_parity.rs`) exercise the parity between the pushdown compiler and the in-Rust `evaluate_condition` reference over randomly generated items and expressions. -**Integration tests** run against a single-node replica set (`mongod --replSet rs0`) covering the full table lifecycle, all item operations (conditional and unconditional), query and scan pagination (base and index), transactions, TTL worker behavior, stream record writes and consumer pagination, GSI propagation and async backfill, backup and restore, and all catalog and IAM operations. These execute as `cargo test -p extenddb-storage-mongodb`. +**Integration tests** run the dual-target `tests/rust/` suite — the same AWS-SDK wire-conformance tests the PostgreSQL backend runs — against a MongoDB-backed ExtendDB server on a single-node replica set (`mongod --replSet rs0`), covering the full table lifecycle, all item operations (conditional and unconditional), query and scan pagination (base and index), transactions, TTL worker behavior, stream record writes and consumer pagination, GSI propagation and async backfill, backup and restore, and catalog operations. They execute via `devtools/run-mongodb-tests -- --rust --rust-integration`, which stands up the replica set, serves ExtendDB against it, and delegates to `devtools/run-tests --backend mongodb`. **End-to-end tests** run the existing ExtendDB pytest suite (`tests/`) unchanged against a MongoDB-backed ExtendDB server. The pytest suite speaks the DynamoDB wire protocol and has no backend awareness — a passing run against MongoDB is equivalent to a passing run against PostgreSQL. This is the conformance test baseline required by RFC-0002. -CI spins up a single-node MongoDB 7.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. +CI (`.github/workflows/integration-mongodb.yml`) runs the pytest and rust-integration suites as two parallel jobs. Each builds ExtendDB with `--features mongodb` and delegates to `devtools/run-mongodb-tests`, which bootstraps a single-node MongoDB 7.0 replica set (`rs.initiate` + wait-for-PRIMARY — a step GitHub `services:` cannot express), serves ExtendDB against it, provisions credentials, and runs the suite via `devtools/run-tests --backend mongodb`. Backend-crate unit and property tests run in the standard `cargo test` workflow. ## Drawbacks From e759498327e54c2a20598f35d8585bf1e4689e0c Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Wed, 5 Aug 2026 13:51:33 -0600 Subject: [PATCH 82/83] test(rust): use a multi-byte all-0xFF begins_with prefix The reviewer asked for a multi-byte all-0xFF case; the prior test used a single-byte [0xFF] prefix. Switch it to [0xFF,0xFF] and add a longer [0xFF,0xFF,0x00] key so the no-upper-bound range is shown to include longer 0xFFFF-prefixed keys while excluding [0xFF,0x00]. --- tests/rust/src/binary_sort_key.rs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/rust/src/binary_sort_key.rs b/tests/rust/src/binary_sort_key.rs index a139f287..34250f6c 100644 --- a/tests/rust/src/binary_sort_key.rs +++ b/tests/rust/src/binary_sort_key.rs @@ -212,11 +212,19 @@ async fn query_binary_sort_key_begins_with_all_ff_prefix() { create_binary_sk_table(c, &table).await; let pk = "p"; - // A 0xFF-prefixed key is the largest possible under unsigned byte order. - // begins_with([0xFF]) exercises the upper-bound edge: a naive "increment - // the last prefix byte" range end overflows past 0xFF and must instead - // extend to the end of the partition. The 0xFE key must be excluded. - seed(c, &table, pk, &[&[0xFE], &[0xFF, 0x00], &[0xFF, 0xFF]]).await; + // A *multi-byte* all-0xFF prefix exercises the upper-bound edge where every + // prefix byte is 0xFF: a naive "increment the last byte, carrying" range + // end overflows past the whole prefix, so the range must have no exclusive + // upper bound and extend to the end of the partition. That range must still + // include longer keys that begin with [0xFF,0xFF] (e.g. [0xFF,0xFF,0x00]) + // while excluding [0xFF,0x00], which shares only the first 0xFF byte. + seed( + c, + &table, + pk, + &[&[0xFF, 0x00], &[0xFF, 0xFF], &[0xFF, 0xFF, 0x00]], + ) + .await; let resp = c .query() @@ -225,14 +233,15 @@ async fn query_binary_sort_key_begins_with_all_ff_prefix() { .expression_attribute_names("#h", "pk") .expression_attribute_names("#r", "sk") .expression_attribute_values(":hv", s(pk)) - .expression_attribute_values(":pfx", bb(&[0xFF])) + .expression_attribute_values(":pfx", bb(&[0xFF, 0xFF])) .send() .await .unwrap(); assert_eq!( sk_bytes(resp.items()), - vec![vec![0xFF, 0x00], vec![0xFF, 0xFF]], - "begins_with([0xFF]) must select all 0xFF-prefixed keys and exclude 0xFE" + vec![vec![0xFF, 0xFF], vec![0xFF, 0xFF, 0x00]], + "begins_with([0xFF,0xFF]) must match keys prefixed by 0xFF 0xFF \ + (including longer ones) and exclude [0xFF,0x00]" ); let _ = c.delete_table().table_name(&table).send().await; From 239beb4a5c48e1c1a82730c7f042e2540c1ba132 Mon Sep 17 00:00:00 2001 From: diegotoledano95 Date: Fri, 7 Aug 2026 09:54:58 -0600 Subject: [PATCH 83/83] test(rust): make restore observer fail on scan errors instead of truncating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The restore-completeness observer swallowed scan errors (Err(_) => break) and returned the partial count, so a transient scan blip was misreported as missing data — a CI flake. Per the reviewers analysis the server-side CREATING->ACTIVE transition is atomic and correct; the defect was the test oracle. Retry the count scan (cursor cloned, not taken, so a retry re-scans the same page) and fail loudly if a page still errors after the budget, rather than under-counting. --- tests/rust/src/restore_active_completeness.rs | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/rust/src/restore_active_completeness.rs b/tests/rust/src/restore_active_completeness.rs index 4eb81b91..d9d291f8 100644 --- a/tests/rust/src/restore_active_completeness.rs +++ b/tests/rust/src/restore_active_completeness.rs @@ -137,13 +137,37 @@ async fn restored_table_has_all_items_when_first_active() { let mut count = 0usize; let mut start_key = None; loop { - let mut req = c2.scan().table_name(&dst2).select(Select::Count); - if let Some(k) = start_key.take() { - req = req.set_exclusive_start_key(Some(k)); - } - let resp = match req.send().await { - Ok(r) => r, - Err(_) => break, + // Retry transient scan errors instead of swallowing them. A + // page that errors must NOT be treated as "no more items": that + // truncates the count and misreports a transient scan blip as + // missing data. Clone (don't take) the cursor so a retry + // re-scans the same page; if a page still fails after the retry + // budget, fail the test loudly rather than under-counting. + let resp = { + let mut got = None; + let mut last_err = None; + for _ in 0..10 { + let mut req = c2.scan().table_name(&dst2).select(Select::Count); + if let Some(k) = start_key.clone() { + req = req.set_exclusive_start_key(Some(k)); + } + match req.send().await { + Ok(r) => { + got = Some(r); + break; + } + Err(e) => { + last_err = Some(e); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + } + } + got.unwrap_or_else(|| { + panic!( + "observer scan failed during item count (transient scan \ + error, not missing data): {last_err:?}" + ) + }) }; count += resp.count() as usize; match resp.last_evaluated_key() {