diff --git a/Cargo.lock b/Cargo.lock index c84030c5..1a7b5931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -26,7 +26,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -235,6 +235,49 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-config" +version = "1.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.0", + "sha1", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.17.0" @@ -257,6 +300,403 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "aws-runtime" +version = "1.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 1.4.0", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-dynamodb" +version = "1.116.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c937c55ae3030bec4431c0d9146e33d2b3e5f54bb47ed32597068200c56affc" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json", + "aws-smithy-observability 0.2.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.102.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json", + "aws-smithy-observability 0.2.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.104.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json", + "aws-smithy-observability 0.2.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.107.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json", + "aws-smithy-observability 0.2.6", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +dependencies = [ + "aws-credential-types", + "aws-smithy-http 0.63.6", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2 0.11.0", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.14", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.9.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.9", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.40", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http 0.64.0", + "aws-smithy-http-client", + "aws-smithy-observability 0.3.0", + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + +[[package]] +name = "aws-smithy-types" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.8.9" @@ -268,10 +708,10 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.9.0", "hyper-util", "itoa", "matchit", @@ -299,8 +739,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -331,15 +771,15 @@ dependencies = [ "bytes", "either", "fs-err", - "http", - "http-body", - "hyper", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.9.0", "hyper-util", "pin-project-lite", - "rustls", + "rustls 0.23.40", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", ] @@ -355,6 +795,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -415,6 +865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blowfish" version = "0.9.1" @@ -443,6 +902,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.2.62" @@ -467,7 +936,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -520,6 +989,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -577,6 +1052,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -606,6 +1087,22 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -615,6 +1112,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -689,6 +1195,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -698,6 +1213,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -719,7 +1243,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -754,12 +1278,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -873,10 +1409,11 @@ dependencies = [ "extenddb-engine", "extenddb-server", "extenddb-storage", + "extenddb-storage-dynamodb", "extenddb-storage-postgres", "libc", "rcgen", - "rustls", + "rustls 0.23.40", "rustls-pemfile", "serde", "serde_json", @@ -899,9 +1436,9 @@ dependencies = [ "extenddb-core", "futures", "hex", - "hmac", + "hmac 0.12.1", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror", "time", "tokio", @@ -943,10 +1480,10 @@ dependencies = [ "extenddb-core", "extenddb-storage", "hex", - "hmac", + "hmac 0.12.1", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tokio", "tracing", "uuid", @@ -970,10 +1507,10 @@ dependencies = [ "extenddb-engine", "extenddb-storage", "futures", - "hyper", + "hyper 1.9.0", "metrics", "rand 0.9.4", - "rustls", + "rustls 0.23.40", "serde", "serde_json", "time", @@ -1007,6 +1544,29 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "extenddb-storage-dynamodb" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-dynamodb", + "aws-smithy-runtime-api", + "bigdecimal", + "extenddb-auth", + "extenddb-core", + "extenddb-storage", + "extenddb-storage-postgres", + "futures", + "inventory", + "serde", + "serde_json", + "sqlx", + "tokio", + "toml", + "tracing", +] + [[package]] name = "extenddb-storage-postgres" version = "0.1.0" @@ -1035,6 +1595,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1254,6 +1820,25 @@ dependencies = [ "polyval", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.14" @@ -1265,7 +1850,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.0", "indexmap", "slab", "tokio", @@ -1336,7 +1921,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -1345,7 +1930,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -1357,6 +1951,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.0" @@ -1367,6 +1972,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1374,7 +1990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1385,8 +2001,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1402,6 +2018,39 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.9.0" @@ -1412,15 +2061,47 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.14", + "http 1.4.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http 1.4.0", + "hyper 1.9.0", + "hyper-util", + "rustls 0.23.40", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.26.4", + "tower-service", ] [[package]] @@ -1429,13 +2110,21 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64 0.22.1", "bytes", - "http", - "http-body", - "hyper", + "futures-channel", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.9.0", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2 0.6.3", "tokio", "tower-service", + "tracing", ] [[package]] @@ -1577,6 +2266,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1714,7 +2409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -1894,6 +2589,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "ordered-multimap" version = "0.7.3" @@ -1904,6 +2605,12 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "parking" version = "2.2.1" @@ -2004,7 +2711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -2013,6 +2720,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs1" version = "0.7.5" @@ -2053,7 +2766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2239,6 +2952,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -2277,8 +2996,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -2301,6 +3020,15 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2310,6 +3038,18 @@ dependencies = [ "nom", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.40" @@ -2321,11 +3061,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -2344,6 +3096,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -2368,12 +3130,54 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -2462,8 +3266,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -2473,8 +3277,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2508,7 +3323,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -2533,6 +3348,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.3" @@ -2599,10 +3424,10 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls", + "rustls 0.23.40", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror", "time", @@ -2642,7 +3467,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", @@ -2665,7 +3490,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -2675,7 +3500,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5", @@ -2686,7 +3511,7 @@ dependencies = [ "rsa", "serde", "sha1", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -2716,7 +3541,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -2727,7 +3552,7 @@ dependencies = [ "rand 0.8.6", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -2944,7 +3769,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] @@ -2960,13 +3785,23 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.40", "tokio", ] @@ -3061,8 +3896,8 @@ dependencies = [ "bitflags", "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", "tokio", "tokio-util", @@ -3157,6 +3992,12 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.0" @@ -3214,7 +4055,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -3283,6 +4124,21 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3701,6 +4557,12 @@ dependencies = [ "time", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yaml-rust2" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index dcebc452..0173e8b3 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/engine", "crates/storage", "crates/storage-postgres", + "crates/storage-dynamodb", "crates/auth", "crates/server", "crates/bin", @@ -26,6 +27,7 @@ extenddb-cache = { path = "crates/cache" } extenddb-engine = { path = "crates/engine" } extenddb-storage = { path = "crates/storage" } extenddb-storage-postgres = { path = "crates/storage-postgres" } +extenddb-storage-dynamodb = { path = "crates/storage-dynamodb" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } @@ -87,6 +89,11 @@ tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] } syslog-tracing = "0.3" metrics = "0.24" +# AWS SDK (DynamoDB-at-home backend) +aws-config = { version = "1", features = ["behavior-version-latest"] } +aws-sdk-dynamodb = "1" +aws-smithy-runtime-api = "1" + # Config clap = { version = "4", features = ["derive"] } config = "0.14" diff --git a/README.md b/README.md index 284bfba1..72eaee6d 100755 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ A DynamoDB-compatible API adapter, ExtendDB speaks the DynamoDB wire protocol - JSON metrics endpoint with DynamoDB CloudWatch-style metric names and dimensions - Daemon mode with syslog logging, plus `--foreground` for container and supervisor environments - PostgreSQL storage — use standard backup, replication, and HA tools +- Optional `dynamodb` storage backend — store your data in actual DynamoDB while running ExtendDB yourself (data plane → DynamoDB, catalog → PostgreSQL); see [Differences from DynamoDB](docs/differences-from-dynamodb.md#the-dynamodb-storage-backend-dynamodb-at-home) ## Quick Start @@ -179,6 +180,7 @@ crates/ engine/ — operation handlers storage/ — storage trait definitions storage-postgres/ — PostgreSQL backend + storage-dynamodb/ — "DynamoDB at home" backend (data → real DynamoDB, catalog → PostgreSQL) auth/ — SigV4 verification, IAM policy engine server/ — HTTP server, management API, web console bin/ — CLI, config, daemon lifecycle diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 9b6755b8..2890f1f2 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" [features] default = ["postgres"] postgres = ["extenddb-storage-postgres"] +dynamodb = ["extenddb-storage-dynamodb"] [dependencies] extenddb-auth = { workspace = true } @@ -22,6 +23,7 @@ extenddb-core = { workspace = true } extenddb-engine = { workspace = true } extenddb-storage = { workspace = true } extenddb-storage-postgres = { workspace = true, optional = true } +extenddb-storage-dynamodb = { workspace = true, optional = true } extenddb-server = { workspace = true } tokio = { workspace = true } anyhow = { workspace = true } diff --git a/crates/bin/src/cmd_init.rs b/crates/bin/src/cmd_init.rs index 92118bb8..50e9854f 100755 --- a/crates/bin/src/cmd_init.rs +++ b/crates/bin/src/cmd_init.rs @@ -16,7 +16,7 @@ use crate::init_helpers::{generate_config, generate_tls_cert_if_needed}; #[derive(Args)] #[allow(clippy::doc_markdown)] // Clap help text, not rustdoc pub struct InitArgs { - /// Storage backend (postgres) (default: postgres) + /// Storage backend (postgres, dynamodb) (default: postgres) #[arg(long, default_value = "postgres")] backend: Option, diff --git a/crates/bin/src/config.rs b/crates/bin/src/config.rs index 607bc1d7..3bf5064e 100755 --- a/crates/bin/src/config.rs +++ b/crates/bin/src/config.rs @@ -138,7 +138,9 @@ impl StorageConfig { } /// Get a reference to the underlying trait object for factory calls. - pub fn as_trait(&self) -> &dyn extenddb_storage::config::StorageConfig { + /// The `+ 'static` (true of the owned `Box` contents) lets factories + /// downcast via `StorageConfig::as_any`. + pub fn as_trait(&self) -> &(dyn extenddb_storage::config::StorageConfig + 'static) { &*self.config } } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index b4f61bf0..824d9d6f 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -25,6 +25,14 @@ mod serve_helpers; mod util; mod workers; +// Force-link optional storage backend crates so their `inventory::submit!` +// registrations are included by the linker. Without an explicit reference, the +// linker drops the otherwise-unused crate and the backend would not register. +// (The postgres backend is linked transitively via a direct symbol reference in +// `config.rs`, so it needs no entry here.) +#[cfg(feature = "dynamodb")] +extern crate extenddb_storage_dynamodb; + use clap::{Parser, Subcommand}; #[derive(Parser)] diff --git a/crates/storage-dynamodb/Cargo.toml b/crates/storage-dynamodb/Cargo.toml new file mode 100644 index 00000000..8e39ab73 --- /dev/null +++ b/crates/storage-dynamodb/Cargo.toml @@ -0,0 +1,29 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "extenddb-storage-dynamodb" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +extenddb-core = { workspace = true } +extenddb-storage = { workspace = true } +extenddb-storage-postgres = { workspace = true } +extenddb-auth = { workspace = true } +aws-config = { workspace = true } +aws-sdk-dynamodb = { workspace = true } +aws-smithy-runtime-api = { workspace = true } +async-trait = { workspace = true } +inventory = { workspace = true } +futures = { workspace = true } +sqlx = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +toml = { workspace = true } +tracing = { workspace = true } +bigdecimal = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } diff --git a/crates/storage-dynamodb/src/backup_engine.rs b/crates/storage-dynamodb/src/backup_engine.rs new file mode 100644 index 00000000..31e1bd50 --- /dev/null +++ b/crates/storage-dynamodb/src/backup_engine.rs @@ -0,0 +1,141 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `BackupEngine` implementation for the DynamoDB-at-home backend — honest stubs. +//! +//! DynamoDB has a native backup and PITR API. This v1 backend does not implement +//! it. Each method names the DynamoDB API call it would map to. + +use futures::future::BoxFuture; + +use extenddb_core::types::{ + BackupDescription, BackupDetails, BackupSummary, ContinuousBackupsDescription, TableDescription, +}; +use extenddb_storage::BackupEngine; +use extenddb_storage::error::StorageError; + +use crate::DynamoEngine; + +impl BackupEngine for DynamoEngine { + fn create_backup( + &self, + _account_id: &str, + _table_name: &str, + _backup_name: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB CreateBackup. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB CreateBackup." + .into(), + )) + }) + } + + fn describe_backup( + &self, + _backup_arn: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB DescribeBackup. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB DescribeBackup." + .into(), + )) + }) + } + + fn list_backups( + &self, + _account_id: &str, + _table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>> { + // Maps to DynamoDB ListBackups. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB ListBackups." + .into(), + )) + }) + } + + fn delete_backup( + &self, + _backup_arn: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB DeleteBackup. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB DeleteBackup." + .into(), + )) + }) + } + + fn restore_table_from_backup( + &self, + _account_id: &str, + _target_table_name: &str, + _backup_arn: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB RestoreTableFromBackup. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB RestoreTableFromBackup." + .into(), + )) + }) + } + + fn describe_continuous_backups( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB DescribeContinuousBackups. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB DescribeContinuousBackups." + .into(), + )) + }) + } + + fn update_continuous_backups( + &self, + _account_id: &str, + _table_name: &str, + _pitr_enabled: bool, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB UpdateContinuousBackups. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB UpdateContinuousBackups." + .into(), + )) + }) + } + + fn restore_table_to_point_in_time( + &self, + _account_id: &str, + _source_table_name: &str, + _target_table_name: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB RestoreTableToPointInTime. + Box::pin(async { + Err(StorageError::Internal( + "Backups are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB RestoreTableToPointInTime." + .into(), + )) + }) + } +} diff --git a/crates/storage-dynamodb/src/bootstrapper.rs b/crates/storage-dynamodb/src/bootstrapper.rs new file mode 100644 index 00000000..b9eab861 --- /dev/null +++ b/crates/storage-dynamodb/src/bootstrapper.rs @@ -0,0 +1,247 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DynamoDB backend implementation of `Bootstrapper`. +//! +//! The DynamoDB backend stores its metadata (catalog, IAM, settings) in a +//! Postgres database, exactly like `extenddb-storage-postgres`. All catalog +//! bootstrap operations are therefore delegated to an inner +//! `PostgresBootstrapper`. DynamoDB-specific "data" bootstrapping is a no-op +//! because DynamoDB tables are created lazily by the CreateTable API — there +//! is no SQL `CREATE DATABASE` equivalent to issue at init time. + +use async_trait::async_trait; +use extenddb_storage::bootstrapper::{AdminBootstrapResult, BootstrapConfig, Bootstrapper}; +use extenddb_storage::error::StorageError; +use extenddb_storage::management_store::OpResult; +use extenddb_storage_postgres::PostgresBootstrapper; + +use crate::config::DynamoStorageConfig; + +/// DynamoDB-at-home bootstrapper. +/// +/// Delegates all catalog operations (user provisioning, migrations, encryption +/// key, admin user, etc.) to an inner `PostgresBootstrapper`. DynamoDB data +/// operations that have no DDL analogue are no-ops. +pub struct DynamoBootstrapper { + /// Delegates catalog/IAM bootstrap to Postgres. + inner: PostgresBootstrapper, + /// Stored for `endpoint_info()` display. + dynamo_config: DynamoStorageConfig, +} + +impl DynamoBootstrapper { + /// Build a `DynamoBootstrapper` from the config file at `config_path`. + /// + /// Reads `storage.dynamodb` from the TOML file, parses it into a + /// `DynamoStorageConfig`, then builds an inner `PostgresBootstrapper` + /// from `catalog_connection_string`. + pub async fn from_config( + config_path: &str, + _cli_args: &[String], + ) -> Result { + // Read and parse the TOML config file. + let config_content = std::fs::read_to_string(config_path) + .map_err(|e| StorageError::Internal(format!("Failed to read config file: {e}")))?; + let app_config: toml::Value = toml::from_str(&config_content) + .map_err(|e| StorageError::Internal(format!("Failed to parse config file: {e}")))?; + + let dynamo_table = app_config + .get("storage") + .and_then(|s| s.get("dynamodb")) + .and_then(|d| d.as_table()) + .ok_or_else(|| { + StorageError::Internal("Missing [storage.dynamodb] section in config".into()) + })?; + + let dynamo_config = DynamoStorageConfig::from_table(dynamo_table).map_err(|e| { + StorageError::Internal(format!("Invalid [storage.dynamodb] config: {e}")) + })?; + + let inner = + Self::build_inner_bootstrapper(&dynamo_config.catalog_connection_string).await?; + + Ok(Self { + inner, + dynamo_config, + }) + } + + /// Build an inner `PostgresBootstrapper` from a Postgres connection string. + /// + /// The `BootstrapConfig` is constructed with `admin_user = app_user` because + /// the connection string encodes the app credentials. This mirrors how the + /// Postgres bootstrapper handles connection strings that already carry + /// app-level credentials. + async fn build_inner_bootstrapper( + catalog_conn: &str, + ) -> Result { + let parts = + extenddb_storage_postgres::parse_connection_string(catalog_conn).map_err(|e| { + StorageError::Internal(format!("invalid catalog connection string: {e}")) + })?; + + // Derive the data_db name: strip the `_catalog` suffix if present. + let data_db = parts + .database + .strip_suffix("_catalog") + .unwrap_or(&parts.database) + .to_owned(); + + let bc = BootstrapConfig { + host: parts.host, + port: parts.port, + // Use the app credentials as admin credentials too. For DynamoDB + // deployments the Postgres instance is typically the catalog-only + // sidecar, and the connection string already carries sufficient + // privileges for DDL. + admin_user: parts.user.clone(), + admin_password: Some(parts.password.clone()), + app_user: parts.user, + app_password: parts.password, + catalog_db: parts.database.clone(), + data_db, + }; + + PostgresBootstrapper::connect(bc) + .await + .map_err(|e| StorageError::Internal(format!("Failed to connect to catalog: {e:?}"))) + } +} + +#[async_trait] +impl Bootstrapper for DynamoBootstrapper { + // ── Delegated to inner PostgresBootstrapper ────────────────────────── + + async fn ensure_app_user(&self) -> OpResult<()> { + self.inner.ensure_app_user().await + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + self.inner.grant_app_role_to_admin().await + } + + async fn create_catalog_db(&self) -> OpResult<()> { + self.inner.create_catalog_db().await + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + self.inner.run_catalog_migrations().await + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + self.inner.bootstrap_encryption_key().await + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + self.inner.bootstrap_default_account().await + } + + async fn bootstrap_admin_user( + &self, + env_user: Option<&str>, + env_password: Option<&str>, + ) -> OpResult { + self.inner + .bootstrap_admin_user(env_user, env_password) + .await + } + + async fn is_catalog_initialized(&self) -> OpResult { + self.inner.is_catalog_initialized().await + } + + async fn read_catalog_version(&self) -> OpResult> { + self.inner.read_catalog_version().await + } + + async fn list_table_names(&self) -> OpResult> { + self.inner.list_table_names().await + } + + async fn drop_databases(&self, data_db: &str) -> OpResult<()> { + self.inner.drop_databases(data_db).await + } + + async fn get_data_db_name(&self) -> OpResult> { + self.inner.get_data_db_name().await + } + + fn expected_catalog_version(&self) -> String { + self.inner.expected_catalog_version() + } + + fn catalog_database_name(&self) -> String { + self.inner.catalog_database_name() + } + + fn catalog_connection_url(&self) -> String { + self.inner.catalog_connection_url() + } + + fn generate_backend_config_section(&self) -> String { + let endpoint_line = match &self.dynamo_config.endpoint_url { + Some(url) => format!("endpoint_url = \"{url}\"\n"), + None => { + "# endpoint_url = \"http://localhost:8000\" # DynamoDB Local, or another ExtendDB endpoint\n" + .to_string() + } + }; + format!( + r#"[storage.dynamodb] +region = "{}" +{}table_prefix = "{}" +catalog_connection_string = "{}""#, + self.dynamo_config.region, + endpoint_line, + self.dynamo_config.table_prefix, + self.inner.catalog_connection_url(), + ) + } + + // ── DynamoDB data no-ops ───────────────────────────────────────────── + + /// DynamoDB has no `CREATE DATABASE` equivalent. + /// + /// Tables are created lazily via the CreateTable API when the user issues + /// their first `CreateTable` request. Nothing to provision at init time. + async fn create_data_db(&self) -> OpResult<()> { + println!("--- [DynamoDB] create_data_db: skipped (tables created lazily via CreateTable)"); + Ok(()) + } + + /// DynamoDB has no SQL schema migrations. + /// + /// The data plane speaks DynamoDB's native wire protocol; there are no + /// `CREATE TABLE` statements to run against DynamoDB itself. + async fn run_data_migrations(&self) -> OpResult<()> { + println!("--- [DynamoDB] run_data_migrations: skipped (no SQL data schema)"); + Ok(()) + } + + /// Record the data connection in the catalog. + /// + /// Delegated to the inner Postgres bootstrapper, which writes a + /// `data_database_connection_string` entry into the catalog `settings` + /// table. For DynamoDB this records the Postgres catalog URL (there is no + /// separate DynamoDB connection string to store), which keeps the catalog + /// coherent for callers that read `get_data_db_name()`. + async fn record_data_connection(&self) -> OpResult<()> { + self.inner.record_data_connection().await + } + + // ── Display ────────────────────────────────────────────────────────── + + /// Return endpoint information combining the DynamoDB endpoint/region and + /// the inner Postgres catalog endpoint. + fn endpoint_info(&self) -> String { + let dynamo_endpoint = self + .dynamo_config + .endpoint_url + .as_deref() + .unwrap_or("aws-dynamodb"); + let region = &self.dynamo_config.region; + let catalog = self.inner.endpoint_info(); + format!("dynamodb endpoint: {dynamo_endpoint} (region {region}), catalog: {catalog}") + } +} diff --git a/crates/storage-dynamodb/src/client.rs b/crates/storage-dynamodb/src/client.rs new file mode 100644 index 00000000..eab57817 --- /dev/null +++ b/crates/storage-dynamodb/src/client.rs @@ -0,0 +1,29 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Builds an aws-sdk-dynamodb client from [`DynamoStorageConfig`]. +//! +//! Setting `endpoint_url` is what lets this point at DynamoDB Local — or at +//! another ExtendDB endpoint, which is a feature, not a bug. + +use crate::config::DynamoStorageConfig; + +/// Build an [`aws_sdk_dynamodb::Client`] from the given config. +/// +/// If `endpoint_url` is set, the client is directed there instead of the +/// standard AWS DynamoDB endpoint — enabling DynamoDB Local or another +/// ExtendDB node as the storage target. +pub async fn build_client(cfg: &DynamoStorageConfig) -> aws_sdk_dynamodb::Client { + use aws_config::BehaviorVersion; + use aws_config::Region; + + let mut loader = + aws_config::defaults(BehaviorVersion::latest()).region(Region::new(cfg.region.clone())); + + if let Some(ep) = &cfg.endpoint_url { + loader = loader.endpoint_url(ep.clone()); + } + + let shared = loader.load().await; + aws_sdk_dynamodb::Client::new(&shared) +} diff --git a/crates/storage-dynamodb/src/config.rs b/crates/storage-dynamodb/src/config.rs new file mode 100644 index 00000000..3bd6af95 --- /dev/null +++ b/crates/storage-dynamodb/src/config.rs @@ -0,0 +1,98 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DynamoDB backend configuration. + +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DynamoStorageConfig { + pub region: String, + #[serde(default)] + pub endpoint_url: Option, + #[serde(default = "default_table_prefix")] + pub table_prefix: String, + pub catalog_connection_string: String, + #[serde(default = "default_pool_size")] + pub pool_size: u32, + #[serde(default)] + pub catalog_pool_size: Option, +} + +fn default_table_prefix() -> String { + "athome_".to_owned() +} + +fn default_pool_size() -> u32 { + 20 +} + +impl DynamoStorageConfig { + /// Deserialize a `DynamoStorageConfig` from a TOML table. + /// + /// # Errors + /// + /// Returns an error string if the table cannot be deserialized into + /// `DynamoStorageConfig` (e.g. missing required fields, unknown fields). + pub fn from_table(t: &toml::Table) -> Result { + t.clone() + .try_into() + .map_err(|e: toml::de::Error| e.to_string()) + } +} + +// ── StorageConfig trait implementation ──────────────────────────────── + +impl extenddb_storage::config::StorageConfig for DynamoStorageConfig { + fn connection_config(&self) -> &str { + &self.catalog_connection_string + } + + fn max_connections(&self) -> u32 { + self.pool_size + } + + fn max_catalog_connections(&self) -> u32 { + self.catalog_pool_size.unwrap_or(self.pool_size) + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_full_section() { + let toml_str = r#" +region = "us-east-1" +endpoint_url = "http://localhost:8000" +table_prefix = "athome_" +catalog_connection_string = "postgresql://u:p@localhost/cat" +"#; + let t: toml::Table = toml::from_str(toml_str).unwrap(); + let c = DynamoStorageConfig::from_table(&t).unwrap(); + assert_eq!(c.region, "us-east-1"); + assert_eq!(c.table_prefix, "athome_"); + assert_eq!(c.endpoint_url.as_deref(), Some("http://localhost:8000")); + } + + #[test] + fn table_prefix_defaults_to_athome() { + let toml_str = r#" +region = "us-east-1" +catalog_connection_string = "postgresql://x" +"#; + let t: toml::Table = toml::from_str(toml_str).unwrap(); + let c = DynamoStorageConfig::from_table(&t).unwrap(); + assert_eq!(c.table_prefix, "athome_"); + } +} diff --git a/crates/storage-dynamodb/src/data_engine.rs b/crates/storage-dynamodb/src/data_engine.rs new file mode 100644 index 00000000..61e074d5 --- /dev/null +++ b/crates/storage-dynamodb/src/data_engine.rs @@ -0,0 +1,599 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DataEngine` implementation for the DynamoDB-at-home backend. +//! +//! Every item-level operation is forwarded to a real DynamoDB endpoint via the +//! AWS SDK. The `stream: Option<&StreamCapture>` parameter is intentionally +//! ignored throughout — DynamoDB generates its own stream records natively; +//! ExtendDB does not synthesise them here. + +use futures::future::BoxFuture; + +use aws_sdk_dynamodb::types::{ + ConditionCheck, Delete, Get, Put, ReturnValue, TransactGetItem, TransactWriteItem, Update, +}; +use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, UpdateAction}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{ + DataEngine, ItemPairResult, QueryResult, StreamCapture, TransactGetOp, TransactWriteOp, +}; + +use crate::DynamoEngine; +use crate::encoding::{item_from_sdk, item_to_sdk}; +use crate::errors::from_sdk_error; +use crate::expression::Renderer; + +// ── Helper — maps a sdk BuildError to StorageError::Internal ───────────────── + +fn sdk_build_err(e: E) -> StorageError { + StorageError::Internal(e.to_string()) +} + +// ── DataEngine ──────────────────────────────────────────────────────────────── + +impl DataEngine for DynamoEngine { + // ── put_item ────────────────────────────────────────────────────────────── + + fn put_item( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + _stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let physical = self + .namer + .physical(&key_info.account_id, &key_info.table_name); + let sdk_item = item_to_sdk(&item); + + // Clone the condition and maps so they can cross the async boundary. + let condition = condition.cloned(); + let maps = maps.clone(); + + Box::pin(async move { + let mut req = self + .client + .put_item() + .table_name(physical) + .set_item(Some(sdk_item)); + + if let Some(cond) = &condition { + let mut r = Renderer::new(); + let expr = r.render_condition(cond, &maps)?; + req = req.condition_expression(expr); + if !r.names().is_empty() { + req = req.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + req = req.set_expression_attribute_values(Some(r.values().clone())); + } + } + + if return_old { + req = req.return_values(ReturnValue::AllOld); + } + + let out = req.send().await.map_err(from_sdk_error)?; + + if return_old { + Ok(out.attributes().map(|m| item_from_sdk(m.clone()))) + } else { + Ok(None) + } + }) + } + + // ── get_item ────────────────────────────────────────────────────────────── + + fn get_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + let physical = self + .namer + .physical(&key_info.account_id, &key_info.table_name); + let sdk_key = item_to_sdk(key); + + Box::pin(async move { + let out = self + .client + .get_item() + .table_name(physical) + .set_key(Some(sdk_key)) + .send() + .await + .map_err(from_sdk_error)?; + + Ok(out.item().map(|m| item_from_sdk(m.clone()))) + }) + } + + // ── delete_item ─────────────────────────────────────────────────────────── + + fn delete_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + _stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let physical = self + .namer + .physical(&key_info.account_id, &key_info.table_name); + let sdk_key = item_to_sdk(key); + let condition = condition.cloned(); + let maps = maps.clone(); + + Box::pin(async move { + let mut req = self + .client + .delete_item() + .table_name(physical) + .set_key(Some(sdk_key)); + + if let Some(cond) = &condition { + let mut r = Renderer::new(); + let expr = r.render_condition(cond, &maps)?; + req = req.condition_expression(expr); + if !r.names().is_empty() { + req = req.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + req = req.set_expression_attribute_values(Some(r.values().clone())); + } + } + + if return_old { + req = req.return_values(ReturnValue::AllOld); + } + + let out = req.send().await.map_err(from_sdk_error)?; + + if return_old { + Ok(out.attributes().map(|m| item_from_sdk(m.clone()))) + } else { + Ok(None) + } + }) + } + + // ── update_item ─────────────────────────────────────────────────────────── + // + // CONCERN: DynamoDB's UpdateItem can return either ALL_OLD or ALL_NEW in a + // single call — not both simultaneously. When both `return_old` and + // `return_new` are true, we prefer ALL_NEW (new slot populated, old = None). + // The caller should not request both unless it can tolerate the missing old. + + #[allow(clippy::too_many_arguments)] + fn update_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + _stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, ItemPairResult> { + let physical = self + .namer + .physical(&key_info.account_id, &key_info.table_name); + let sdk_key = item_to_sdk(key); + let actions = actions.to_vec(); + let condition = condition.cloned(); + let maps = maps.clone(); + + Box::pin(async move { + // Use a single Renderer so update and condition tokens don't collide. + let mut r = Renderer::new(); + let update_expr = r.render_update(&actions, &maps)?; + + let mut req = self + .client + .update_item() + .table_name(physical) + .set_key(Some(sdk_key)) + .update_expression(update_expr); + + if let Some(cond) = &condition { + let cond_expr = r.render_condition(cond, &maps)?; + req = req.condition_expression(cond_expr); + } + + if !r.names().is_empty() { + req = req.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + req = req.set_expression_attribute_values(Some(r.values().clone())); + } + + // DynamoDB supports only one ReturnValues mode per call. + // Prefer ALL_NEW when both are requested (see CONCERN above). + let (rv, want_new, want_old) = if return_new { + (ReturnValue::AllNew, true, false) + } else if return_old { + (ReturnValue::AllOld, false, true) + } else { + (ReturnValue::None, false, false) + }; + + req = req.return_values(rv); + + let out = req.send().await.map_err(from_sdk_error)?; + + let item = out.attributes().map(|m| item_from_sdk(m.clone())); + + if want_new { + Ok((None, item)) + } else if want_old { + Ok((item, None)) + } else { + Ok((None, None)) + } + }) + } + + // ── query ───────────────────────────────────────────────────────────────── + + #[allow(clippy::too_many_arguments)] + fn query( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + let physical = self + .namer + .physical(&key_info.account_id, &key_info.table_name); + let key_condition = key_condition.clone(); + let maps = maps.clone(); + let esk = exclusive_start_key.cloned(); + let index_name = index_name.map(|s| s.to_owned()); + + Box::pin(async move { + let mut r = Renderer::new(); + let kc_expr = r.render_key_condition(&key_condition, &maps)?; + + let mut req = self + .client + .query() + .table_name(physical) + .key_condition_expression(kc_expr) + .scan_index_forward(forward); + + if !r.names().is_empty() { + req = req.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + req = req.set_expression_attribute_values(Some(r.values().clone())); + } + + if let Some(l) = limit { + req = req.limit(i32::try_from(l).unwrap_or(i32::MAX)); + } + + if let Some(k) = esk { + req = req.set_exclusive_start_key(Some(item_to_sdk(&k))); + } + + if let Some(n) = index_name { + req = req.index_name(n); + } + + let out = req.send().await.map_err(from_sdk_error)?; + + let items: Vec = out + .items() + .iter() + .map(|m| item_from_sdk(m.clone())) + .collect(); + + let lek = out.last_evaluated_key().map(|m| item_from_sdk(m.clone())); + + Ok((items, lek)) + }) + } + + // ── scan ────────────────────────────────────────────────────────────────── + + #[allow(clippy::too_many_arguments)] + fn scan( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + let physical = self + .namer + .physical(&key_info.account_id, &key_info.table_name); + let esk = exclusive_start_key.cloned(); + let index_name = index_name.map(|s| s.to_owned()); + + Box::pin(async move { + let mut req = self.client.scan().table_name(physical); + + if let Some(l) = limit { + req = req.limit(i32::try_from(l).unwrap_or(i32::MAX)); + } + + if let Some(k) = esk { + req = req.set_exclusive_start_key(Some(item_to_sdk(&k))); + } + + if let Some(seg) = segment { + req = req.segment(i32::try_from(seg).unwrap_or(0)); + } + + if let Some(ts) = total_segments { + req = req.total_segments(i32::try_from(ts).unwrap_or(1)); + } + + if let Some(n) = index_name { + req = req.index_name(n); + } + + let out = req.send().await.map_err(from_sdk_error)?; + + let items: Vec = out + .items() + .iter() + .map(|m| item_from_sdk(m.clone())) + .collect(); + + let lek = out.last_evaluated_key().map(|m| item_from_sdk(m.clone())); + + Ok((items, lek)) + }) + } + + // ── transact_get_items ──────────────────────────────────────────────────── + + fn transact_get_items( + &self, + ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + // Capture all data before crossing the async boundary. + let items: Result, StorageError> = ops + .iter() + .map(|op| { + let physical = self + .namer + .physical(&op.key_info.account_id, &op.key_info.table_name); + let sdk_key = item_to_sdk(op.key); + let get = Get::builder() + .table_name(physical) + .set_key(Some(sdk_key)) + .build() + .map_err(sdk_build_err)?; + // TransactGetItem::builder().build() is infallible + let tgi = TransactGetItem::builder().get(get).build(); + Ok(tgi) + }) + .collect(); + + Box::pin(async move { + let tgi_vec = items?; + + let out = self + .client + .transact_get_items() + .set_transact_items(Some(tgi_vec)) + .send() + .await + .map_err(from_sdk_error)?; + + let results: Vec> = out + .responses() + .iter() + .map(|resp| resp.item().map(|m| item_from_sdk(m.clone()))) + .collect(); + + Ok(results) + }) + } + + // ── transact_write_items ────────────────────────────────────────────────── + // + // Token interpretation: `token` is `(tok, fp)` where `tok` is the client + // request token (idempotency key) forwarded to DynamoDB's + // `client_request_token`, and `fp` is the fingerprint (request hash) used + // only by Postgres for mismatch detection. DynamoDB manages idempotency + // natively (~10-minute window), so we forward `tok` and ignore `fp`. + // + // This mirrors the postgres interpretation in + // `crates/storage-postgres/src/data/transactions.rs`: + // `if let Some((tok, fp)) = token { check_idempotency_token_in_tx(&mut tx, tok, fp) }` + // where `tok` is the token string passed to the DB and `fp` is its fingerprint. + + fn transact_write_items( + &self, + ops: &[TransactWriteOp<'_>], + token: Option<(&str, &str)>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let client_token = token.map(|(tok, _fp)| tok.to_owned()); + + let twi_vec: Result, StorageError> = ops + .iter() + .map(|op| build_transact_write_item(op, &self.namer)) + .collect(); + + Box::pin(async move { + let twi_vec = twi_vec?; + + let mut req = self + .client + .transact_write_items() + .set_transact_items(Some(twi_vec)); + + if let Some(t) = client_token { + req = req.client_request_token(t); + } + + req.send().await.map_err(from_sdk_error)?; + Ok(()) + }) + } + + // ── cleanup_expired_idempotency_tokens ──────────────────────────────────── + // + // DynamoDB manages its own ~10-minute idempotency window natively. + // There are no ExtendDB-managed idempotency rows to clean up in this backend. + + fn cleanup_expired_idempotency_tokens( + &self, + _max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(0) }) + } +} + +// ── Private helpers ─────────────────────────────────────────────────────────── + +/// Build a single `TransactWriteItem` from a `TransactWriteOp`. +/// +/// Each op gets its own `Renderer` so expression attribute name/value tokens +/// are scoped to the individual SDK item (separate maps per item in the batch). +fn build_transact_write_item( + op: &TransactWriteOp<'_>, + namer: &crate::naming::Namer, +) -> Result { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + .. + } => { + let physical = namer.physical(&key_info.account_id, &key_info.table_name); + let sdk_item = item_to_sdk(item); + let mut put_b = Put::builder().table_name(physical).set_item(Some(sdk_item)); + + if let Some(cond) = condition { + let mut r = Renderer::new(); + let expr = r.render_condition(cond, maps)?; + put_b = put_b.condition_expression(expr); + if !r.names().is_empty() { + put_b = put_b.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + put_b = put_b.set_expression_attribute_values(Some(r.values().clone())); + } + } + + let put = put_b.build().map_err(sdk_build_err)?; + // TransactWriteItem::builder().build() is infallible + let twi = TransactWriteItem::builder().put(put).build(); + Ok(twi) + } + + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + .. + } => { + let physical = namer.physical(&key_info.account_id, &key_info.table_name); + let sdk_key = item_to_sdk(key); + let mut del_b = Delete::builder() + .table_name(physical) + .set_key(Some(sdk_key)); + + if let Some(cond) = condition { + let mut r = Renderer::new(); + let expr = r.render_condition(cond, maps)?; + del_b = del_b.condition_expression(expr); + if !r.names().is_empty() { + del_b = del_b.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + del_b = del_b.set_expression_attribute_values(Some(r.values().clone())); + } + } + + let del = del_b.build().map_err(sdk_build_err)?; + let twi = TransactWriteItem::builder().delete(del).build(); + Ok(twi) + } + + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + .. + } => { + let physical = namer.physical(&key_info.account_id, &key_info.table_name); + let sdk_key = item_to_sdk(key); + let mut r = Renderer::new(); + let update_expr = r.render_update(actions, maps)?; + + let mut upd_b = Update::builder() + .table_name(physical) + .set_key(Some(sdk_key)) + .update_expression(update_expr); + + if let Some(cond) = condition { + let cond_expr = r.render_condition(cond, maps)?; + upd_b = upd_b.condition_expression(cond_expr); + } + + if !r.names().is_empty() { + upd_b = upd_b.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + upd_b = upd_b.set_expression_attribute_values(Some(r.values().clone())); + } + + let upd = upd_b.build().map_err(sdk_build_err)?; + let twi = TransactWriteItem::builder().update(upd).build(); + Ok(twi) + } + + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + .. + } => { + let physical = namer.physical(&key_info.account_id, &key_info.table_name); + let sdk_key = item_to_sdk(key); + let mut r = Renderer::new(); + let cond_expr = r.render_condition(condition, maps)?; + + let mut cc_b = ConditionCheck::builder() + .table_name(physical) + .set_key(Some(sdk_key)) + .condition_expression(cond_expr); + + if !r.names().is_empty() { + cc_b = cc_b.set_expression_attribute_names(Some(r.names().clone())); + } + if !r.values().is_empty() { + cc_b = cc_b.set_expression_attribute_values(Some(r.values().clone())); + } + + let cc = cc_b.build().map_err(sdk_build_err)?; + let twi = TransactWriteItem::builder().condition_check(cc).build(); + Ok(twi) + } + } +} diff --git a/crates/storage-dynamodb/src/encoding.rs b/crates/storage-dynamodb/src/encoding.rs new file mode 100644 index 00000000..5d99f4d8 --- /dev/null +++ b/crates/storage-dynamodb/src/encoding.rs @@ -0,0 +1,148 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Real, round-trip-tested marshalling between ExtendDB's internal item type and +//! the AWS SDK's. This is, structurally, the identity function: ExtendDB already +//! speaks DynamoDB's type system, so every value maps to its exact namesake. +//! The only reason this file is not empty is that ExtendDB's in-memory Rust enum +//! and the SDK's `AttributeValue` enum are *different Rust types* holding the +//! same data. We translate DynamoDB into DynamoDB. It round-trips because of +//! course it does. + +use std::collections::HashMap; + +use aws_sdk_dynamodb::primitives::Blob; +use extenddb_core::types::AttributeValue as CoreAttributeValue; +use extenddb_core::types::Item as CoreItem; + +/// Convert an ExtendDB `AttributeValue` to the AWS SDK `AttributeValue`. +pub fn to_sdk(v: &CoreAttributeValue) -> aws_sdk_dynamodb::types::AttributeValue { + use aws_sdk_dynamodb::types::AttributeValue as Sdk; + match v { + CoreAttributeValue::S(s) => Sdk::S(s.clone()), + CoreAttributeValue::N(n) => Sdk::N(n.clone()), + CoreAttributeValue::B(bytes) => Sdk::B(Blob::new(bytes.clone())), + CoreAttributeValue::SS(set) => Sdk::Ss(set.iter().cloned().collect()), + CoreAttributeValue::NS(set) => Sdk::Ns(set.iter().cloned().collect()), + CoreAttributeValue::BS(set) => Sdk::Bs(set.iter().map(|b| Blob::new(b.clone())).collect()), + CoreAttributeValue::Bool(b) => Sdk::Bool(*b), + CoreAttributeValue::Null => Sdk::Null(true), + CoreAttributeValue::L(list) => Sdk::L(list.iter().map(to_sdk).collect()), + CoreAttributeValue::M(map) => { + Sdk::M(map.iter().map(|(k, v)| (k.clone(), to_sdk(v))).collect()) + } + } +} + +/// Convert an AWS SDK `AttributeValue` to an ExtendDB `AttributeValue`. +/// +/// Unknown / future SDK variants (the `#[non_exhaustive]` catch-all) map to +/// `AttributeValue::Null` and emit a `tracing::warn!`. +pub fn from_sdk(v: &aws_sdk_dynamodb::types::AttributeValue) -> CoreAttributeValue { + use aws_sdk_dynamodb::types::AttributeValue as Sdk; + use std::collections::{BTreeMap, BTreeSet}; + match v { + Sdk::S(s) => CoreAttributeValue::S(s.clone()), + Sdk::N(n) => CoreAttributeValue::N(n.clone()), + Sdk::B(blob) => CoreAttributeValue::B(blob.as_ref().to_vec()), + Sdk::Ss(vec) => CoreAttributeValue::SS(vec.iter().cloned().collect::>()), + Sdk::Ns(vec) => CoreAttributeValue::NS(vec.iter().cloned().collect::>()), + Sdk::Bs(blobs) => CoreAttributeValue::BS( + blobs + .iter() + .map(|b| b.as_ref().to_vec()) + .collect::>(), + ), + Sdk::Bool(b) => CoreAttributeValue::Bool(*b), + Sdk::Null(_) => CoreAttributeValue::Null, + Sdk::L(list) => CoreAttributeValue::L(list.iter().map(from_sdk).collect()), + Sdk::M(map) => CoreAttributeValue::M( + map.iter() + .map(|(k, v)| (k.clone(), from_sdk(v))) + .collect::>(), + ), + _ => { + tracing::warn!("encountered unknown SDK AttributeValue variant; mapping to Null"); + CoreAttributeValue::Null + } + } +} + +/// Convert an ExtendDB `Item` to a DynamoDB SDK item (HashMap). +pub fn item_to_sdk(item: &CoreItem) -> HashMap { + item.iter().map(|(k, v)| (k.clone(), to_sdk(v))).collect() +} + +/// Convert a DynamoDB SDK item (HashMap) to an ExtendDB `Item`. +pub fn item_from_sdk(item: HashMap) -> CoreItem { + item.into_iter().map(|(k, v)| (k, from_sdk(&v))).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::types::AttributeValue as Core; + use std::collections::{BTreeMap, BTreeSet}; + + fn round_trip(v: Core) { + let sdk = to_sdk(&v); + let back = from_sdk(&sdk); + assert_eq!(back, v, "round-trip mismatch"); + } + + #[test] + fn rt_string() { + round_trip(Core::S("hello".into())); + } + + #[test] + fn rt_number() { + round_trip(Core::N("123.45".into())); + } + + #[test] + fn rt_bool() { + round_trip(Core::Bool(true)); + } + + #[test] + fn rt_null() { + round_trip(Core::Null); + } + + #[test] + fn rt_binary() { + round_trip(Core::B(vec![0u8, 1, 2, 255])); + } + + #[test] + fn rt_string_set() { + round_trip(Core::SS(BTreeSet::from(["a".to_string(), "b".to_string()]))); + } + + #[test] + fn rt_number_set() { + round_trip(Core::NS(BTreeSet::from(["1".to_string(), "2".to_string()]))); + } + + #[test] + fn rt_binary_set() { + round_trip(Core::BS(BTreeSet::from([vec![1u8, 2], vec![3u8, 4]]))); + } + + #[test] + fn rt_list_and_map_nested() { + let mut m = BTreeMap::new(); + m.insert("k".to_string(), Core::S("v".into())); + round_trip(Core::L(vec![Core::N("1".into()), Core::M(m)])); + } + + #[test] + fn item_round_trips() { + let mut item: BTreeMap = BTreeMap::new(); + item.insert("pk".to_string(), Core::S("u#1".into())); + item.insert("n".to_string(), Core::N("42".into())); + let sdk = item_to_sdk(&item); + assert_eq!(item_from_sdk(sdk), item); + } +} diff --git a/crates/storage-dynamodb/src/errors.rs b/crates/storage-dynamodb/src/errors.rs new file mode 100644 index 00000000..f35d121a --- /dev/null +++ b/crates/storage-dynamodb/src/errors.rs @@ -0,0 +1,108 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Maps AWS SDK DynamoDB errors to ExtendDB's `StorageError`. + +use aws_sdk_dynamodb::error::ProvideErrorMetadata; +use aws_smithy_runtime_api::client::result::SdkError; +use extenddb_storage::error::StorageError; + +/// Map a DynamoDB error code and message string to the appropriate [`StorageError`] variant. +/// +/// This is a pure function — no SDK types required — and is the primary unit-tested surface. +pub fn classify(error_code: &str, message: &str) -> StorageError { + match error_code { + "ConditionalCheckFailedException" => StorageError::ConditionFailed(None), + "ResourceNotFoundException" => StorageError::TableNotFound(message.to_string()), + "ResourceInUseException" => StorageError::TableAlreadyExists(message.to_string()), + "TransactionCanceledException" | "TransactionConflictException" => { + StorageError::TransactionCanceled(vec![]) + } + "ValidationException" => StorageError::Validation(message.to_string()), + "ProvisionedThroughputExceededException" + | "RequestLimitExceeded" + | "ThrottlingException" => StorageError::Internal(format!("throttled: {message}")), + _ => StorageError::Internal(format!("{error_code}: {message}")), + } +} + +/// Convert a generic AWS SDK [`SdkError`] into a [`StorageError`]. +/// +/// Service errors are routed through [`classify`] using the error code and message extracted via +/// [`ProvideErrorMetadata`]. All other `SdkError` variants (dispatch failures, timeouts, +/// construction errors, response parse errors) are mapped to [`StorageError::Connection`]. +pub fn from_sdk_error(err: SdkError) -> StorageError +where + E: ProvideErrorMetadata, +{ + match err { + SdkError::ServiceError(context) => { + let source = context.into_err(); + let code = source.code().unwrap_or("Unknown"); + let message = source.message().unwrap_or(""); + classify(code, message) + } + other => StorageError::Connection(other.to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn conditional_check_failed_maps_to_condition_failed() { + assert!(matches!( + classify("ConditionalCheckFailedException", ""), + StorageError::ConditionFailed(None) + )); + } + + #[test] + fn resource_not_found_maps_to_table_not_found() { + assert!(matches!( + classify("ResourceNotFoundException", "Requested resource not found"), + StorageError::TableNotFound(_) + )); + } + + #[test] + fn resource_in_use_maps_to_table_already_exists() { + assert!(matches!( + classify("ResourceInUseException", "x"), + StorageError::TableAlreadyExists(_) + )); + } + + #[test] + fn validation_maps_to_validation() { + assert!(matches!( + classify("ValidationException", "bad"), + StorageError::Validation(_) + )); + } + + #[test] + fn transaction_canceled_maps_to_transaction_canceled() { + assert!(matches!( + classify("TransactionCanceledException", ""), + StorageError::TransactionCanceled(_) + )); + } + + #[test] + fn throttle_maps_to_internal() { + assert!(matches!( + classify("ThrottlingException", "slow down"), + StorageError::Internal(_) + )); + } + + #[test] + fn unknown_maps_to_internal() { + assert!(matches!( + classify("SomethingNew", "msg"), + StorageError::Internal(_) + )); + } +} diff --git a/crates/storage-dynamodb/src/expression.rs b/crates/storage-dynamodb/src/expression.rs new file mode 100644 index 00000000..f0537b04 --- /dev/null +++ b/crates/storage-dynamodb/src/expression.rs @@ -0,0 +1,406 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Renders ExtendDB expression ASTs back into DynamoDB expression strings, +//! building fresh `ExpressionAttributeNames` and `ExpressionAttributeValues` +//! maps suitable for forwarding to the AWS SDK DynamoDB client. + +use std::collections::HashMap; + +use extenddb_core::expression::{ + ArithOp, CompareOp, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, + UpdateAction, resolve_name_ref, +}; +use extenddb_storage::error::StorageError; + +use crate::encoding::to_sdk; + +/// Renders ExtendDB AST expressions into DynamoDB expression strings. +/// +/// Allocates fresh `#n{N}` name tokens and `:v{N}` value tokens, and populates +/// the corresponding `ExpressionAttributeNames` and `ExpressionAttributeValues` +/// maps. These maps are forwarded directly to the AWS SDK DynamoDB client. +pub struct Renderer { + names: HashMap, + values: HashMap, + n_counter: usize, + v_counter: usize, +} + +impl Default for Renderer { + fn default() -> Self { + Self::new() + } +} + +impl Renderer { + /// Create a new empty `Renderer`. + pub fn new() -> Self { + Self { + names: HashMap::new(), + values: HashMap::new(), + n_counter: 0, + v_counter: 0, + } + } + + /// The accumulated `ExpressionAttributeNames` map. + pub fn names(&self) -> &HashMap { + &self.names + } + + /// The accumulated `ExpressionAttributeValues` map (SDK type). + pub fn values(&self) -> &HashMap { + &self.values + } + + /// Render a condition/filter expression AST node into a DynamoDB expression string. + /// + /// # Errors + /// + /// Returns `StorageError::Validation` if a name reference or value placeholder + /// cannot be resolved from `maps`. + pub fn render_condition( + &mut self, + e: &Expr, + maps: &ExpressionMaps, + ) -> Result { + self.render_expr(e, maps) + } + + /// Render a `KeyConditionExpression` into a DynamoDB expression string. + /// + /// # Errors + /// + /// Returns `StorageError::Validation` if resolution fails. + pub fn render_key_condition( + &mut self, + kc: &KeyCondition, + maps: &ExpressionMaps, + ) -> Result { + // pk = value + let pk_path_str = self.render_path(&kc.pk_path, maps)?; + let pk_val_str = self.render_expr(&kc.pk_value, maps)?; + let mut parts = vec![format!("{pk_path_str} = {pk_val_str}")]; + + // extra_pk_conditions: rendered as equality conditions + for (path, value) in &kc.extra_pk_conditions { + let p = self.render_path(path, maps)?; + let v = self.render_expr(value, maps)?; + parts.push(format!("{p} = {v}")); + } + + // sk_condition + if let Some(sk) = &kc.sk_condition { + let sk_str = self.render_sk_condition(sk, maps)?; + parts.push(sk_str); + } + + // extra_sk_conditions: rendered as equality conditions + for (path, value) in &kc.extra_sk_conditions { + let p = self.render_path(path, maps)?; + let v = self.render_expr(value, maps)?; + parts.push(format!("{p} = {v}")); + } + + Ok(parts.join(" AND ")) + } + + /// Render a slice of `UpdateAction`s into a DynamoDB `UpdateExpression` string. + /// + /// Groups actions by type (SET, REMOVE, ADD, DELETE) and joins them as + /// `SET a, b REMOVE c ADD d e DELETE f g`. + /// + /// # Errors + /// + /// Returns `StorageError::Validation` if resolution fails. + pub fn render_update( + &mut self, + actions: &[UpdateAction], + maps: &ExpressionMaps, + ) -> Result { + let mut set_parts: Vec = Vec::new(); + let mut remove_parts: Vec = Vec::new(); + let mut add_parts: Vec = Vec::new(); + let mut delete_parts: Vec = Vec::new(); + + for action in actions { + match action { + UpdateAction::Set { path, value } => { + let p = self.render_path(path, maps)?; + let v = self.render_expr(value, maps)?; + set_parts.push(format!("{p} = {v}")); + } + UpdateAction::Remove { path } => { + let p = self.render_path(path, maps)?; + remove_parts.push(p); + } + UpdateAction::Add { path, value } => { + let p = self.render_path(path, maps)?; + let v = self.render_expr(value, maps)?; + add_parts.push(format!("{p} {v}")); + } + UpdateAction::Delete { path, value } => { + let p = self.render_path(path, maps)?; + let v = self.render_expr(value, maps)?; + delete_parts.push(format!("{p} {v}")); + } + } + } + + let mut groups: Vec = Vec::new(); + if !set_parts.is_empty() { + groups.push(format!("SET {}", set_parts.join(", "))); + } + if !remove_parts.is_empty() { + groups.push(format!("REMOVE {}", remove_parts.join(", "))); + } + if !add_parts.is_empty() { + groups.push(format!("ADD {}", add_parts.join(", "))); + } + if !delete_parts.is_empty() { + groups.push(format!("DELETE {}", delete_parts.join(", "))); + } + + Ok(groups.join(" ")) + } + + // --- private helpers --- + + fn render_expr(&mut self, e: &Expr, maps: &ExpressionMaps) -> Result { + match e { + Expr::Path(elements) => self.render_path(elements, maps), + + Expr::Placeholder(name) => { + let core_val = maps.resolve_value(name).map_err( + |err: extenddb_core::error::DynamoDbError| { + StorageError::Validation(err.to_string()) + }, + )?; + let sdk_val = to_sdk(core_val); + let token = format!(":v{}", self.v_counter); + self.v_counter += 1; + self.values.insert(token.clone(), sdk_val); + Ok(token) + } + + Expr::Compare { left, op, right } => { + let l = self.render_expr(left, maps)?; + let r = self.render_expr(right, maps)?; + let op_str = match op { + CompareOp::Eq => "=", + CompareOp::Ne => "<>", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + }; + Ok(format!("{l} {op_str} {r}")) + } + + Expr::And(left, right) => { + let l = self.render_expr(left, maps)?; + let r = self.render_expr(right, maps)?; + Ok(format!("({l} AND {r})")) + } + + Expr::Or(left, right) => { + let l = self.render_expr(left, maps)?; + let r = self.render_expr(right, maps)?; + Ok(format!("({l} OR {r})")) + } + + Expr::Not(inner) => { + let s = self.render_expr(inner, maps)?; + Ok(format!("(NOT {s})")) + } + + Expr::Function { name, args } => { + let mut rendered_args = Vec::with_capacity(args.len()); + for arg in args { + rendered_args.push(self.render_expr(arg, maps)?); + } + Ok(format!("{}({})", name, rendered_args.join(", "))) + } + + Expr::Arithmetic { left, op, right } => { + let l = self.render_expr(left, maps)?; + let r = self.render_expr(right, maps)?; + let op_str = match op { + ArithOp::Add => "+", + ArithOp::Sub => "-", + }; + Ok(format!("{l} {op_str} {r}")) + } + + Expr::Between { operand, low, high } => { + let e_str = self.render_expr(operand, maps)?; + let lo = self.render_expr(low, maps)?; + let hi = self.render_expr(high, maps)?; + Ok(format!("{e_str} BETWEEN {lo} AND {hi}")) + } + + Expr::In { operand, list } => { + let e_str = self.render_expr(operand, maps)?; + let mut items = Vec::with_capacity(list.len()); + for item in list { + items.push(self.render_expr(item, maps)?); + } + Ok(format!("{e_str} IN ({})", items.join(", "))) + } + } + } + + /// Render a document path into a DynamoDB token string. + /// + /// Each `PathElement::Attribute` is resolved to its real name (via + /// `resolve_name_ref`), allocated a fresh `#n{N}` token, and stored in + /// `self.names`. Index elements become `[i]`. + /// + /// Path format: `#n0` for a single attribute; `#n0.#n1[2].#n3` for nested. + fn render_path( + &mut self, + elements: &[PathElement], + maps: &ExpressionMaps, + ) -> Result { + let mut result = String::new(); + + for element in elements { + match element { + PathElement::Attribute(name) => { + let real_name = resolve_name_ref(name, maps).map_err( + |err: extenddb_core::error::DynamoDbError| { + StorageError::Validation(err.to_string()) + }, + )?; + let token = format!("#n{}", self.n_counter); + self.n_counter += 1; + self.names.insert(token.clone(), real_name.into_owned()); + if result.is_empty() { + result.push_str(&token); + } else { + result.push('.'); + result.push_str(&token); + } + } + PathElement::Index(idx) => { + result.push_str(&format!("[{idx}]")); + } + } + } + + Ok(result) + } + + fn render_sk_condition( + &mut self, + sk: &SortKeyCondition, + maps: &ExpressionMaps, + ) -> Result { + match sk { + SortKeyCondition::Compare { path, op, value } => { + let p = self.render_path(path, maps)?; + let v = self.render_expr(value, maps)?; + let op_str = match op { + CompareOp::Eq => "=", + CompareOp::Ne => "<>", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + }; + Ok(format!("{p} {op_str} {v}")) + } + SortKeyCondition::Between { path, low, high } => { + let p = self.render_path(path, maps)?; + let lo = self.render_expr(low, maps)?; + let hi = self.render_expr(high, maps)?; + Ok(format!("{p} BETWEEN {lo} AND {hi}")) + } + SortKeyCondition::BeginsWith { path, prefix } => { + let p = self.render_path(path, maps)?; + let pref = self.render_expr(prefix, maps)?; + Ok(format!("begins_with({p}, {pref})")) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement, UpdateAction}; + use extenddb_core::types::AttributeValue as Av; + use std::collections::HashMap; + + fn maps_with(values: &[(&str, Av)], names: &[(&str, &str)]) -> ExpressionMaps { + let v = values + .iter() + .map(|(k, val)| (k.to_string(), val.clone())) + .collect::>(); + let n = names + .iter() + .map(|(k, val)| (k.to_string(), val.to_string())) + .collect::>(); + ExpressionMaps::new(n, v) + } + + #[test] + fn condition_attribute_exists_bare_name() { + let e = Expr::Function { + name: "attribute_exists".into(), + args: vec![Expr::Path(vec![PathElement::Attribute("pk".into())])], + }; + let mut r = Renderer::new(); + let s = r.render_condition(&e, &ExpressionMaps::default()).unwrap(); + assert_eq!(s, "attribute_exists(#n0)"); + assert_eq!(r.names().get("#n0").map(String::as_str), Some("pk")); + } + + #[test] + fn condition_compare_with_value_placeholder() { + // age >= :min where :min resolves to N "21" + let e = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("age".into())])), + op: CompareOp::Ge, + right: Box::new(Expr::Placeholder("min".into())), + }; + let maps = maps_with(&[("min", Av::N("21".into()))], &[]); + let mut r = Renderer::new(); + let s = r.render_condition(&e, &maps).unwrap(); + assert_eq!(s, "#n0 >= :v0"); + assert_eq!(r.names().get("#n0").map(String::as_str), Some("age")); + assert!(r.values().contains_key(":v0")); + } + + #[test] + fn hash_reference_resolves_via_names_map() { + // path "#a" should resolve through maps.names to the real attribute + let e = Expr::Function { + name: "attribute_not_exists".into(), + args: vec![Expr::Path(vec![PathElement::Attribute("#a".into())])], + }; + let maps = maps_with(&[], &[("a", "status")]); + let mut r = Renderer::new(); + let s = r.render_condition(&e, &maps).unwrap(); + assert_eq!(s, "attribute_not_exists(#n0)"); + assert_eq!(r.names().get("#n0").map(String::as_str), Some("status")); + } + + #[test] + fn update_set_and_remove() { + let actions = vec![ + UpdateAction::Set { + path: vec![PathElement::Attribute("name".into())], + value: Expr::Placeholder("nm".into()), + }, + UpdateAction::Remove { + path: vec![PathElement::Attribute("temp".into())], + }, + ]; + let maps = maps_with(&[("nm", Av::S("Bob".into()))], &[]); + let mut r = Renderer::new(); + let s = r.render_update(&actions, &maps).unwrap(); + assert_eq!(s, "SET #n0 = :v0 REMOVE #n1"); + } +} diff --git a/crates/storage-dynamodb/src/lib.rs b/crates/storage-dynamodb/src/lib.rs new file mode 100644 index 00000000..b78f8565 --- /dev/null +++ b/crates/storage-dynamodb/src/lib.rs @@ -0,0 +1,130 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! "DynamoDB at home" storage backend for ExtendDB. +//! +//! The third entry in the satirical-but-functional backend series, and the only +//! one that actually works the way the marketing implies: ExtendDB speaks the +//! DynamoDB wire protocol, and this backend stores its data in *actual* +//! DynamoDB. The point is not the encoding — there is barely any, because +//! DynamoDB is already a key/value database — it is the deployment posture. Run +//! ExtendDB yourself, pointed at DynamoDB, and you are technically "self-hosted." +//! The execs stop asking. +//! +//! Data plane forwards to DynamoDB; the catalog/IAM/auth plane is delegated to +//! the Postgres backend (`extenddb-storage-postgres`), because DynamoDB has +//! opinions about what a database is and "relational IAM catalog" is not one. + +pub(crate) mod backup_engine; +pub mod bootstrapper; +pub mod client; +pub mod config; +pub(crate) mod data_engine; +pub mod encoding; +pub(crate) mod errors; +pub(crate) mod expression; +pub(crate) mod metadata_engine; +pub mod naming; +pub mod operations; +mod server_components; +pub(crate) mod stream_engine; +pub(crate) mod table_engine; +pub(crate) mod worker_store; + +/// The DynamoDB-at-home storage engine: forwards the data/table plane to a real +/// DynamoDB endpoint. Catalog/auth are composed separately (see server_components, later task). +pub struct DynamoEngine { + pub(crate) client: aws_sdk_dynamodb::Client, + pub(crate) namer: crate::naming::Namer, +} + +/// Compile-time assertion: `DynamoEngine` satisfies the `StorageEngine` supertrait. +/// +/// If any of the six component traits (`TableEngine`, `DataEngine`, `MetadataEngine`, +/// `StreamEngine`, `BackupEngine`, `WorkerStore`) is missing, this function will +/// produce a compiler error. +#[allow(dead_code)] +fn _assert_storage_engine(e: &DynamoEngine) -> &dyn extenddb_storage::StorageEngine { + e +} + +impl DynamoEngine { + /// Build the engine from config (constructs the SDK client and the namer). + pub async fn from_config(cfg: &crate::config::DynamoStorageConfig) -> Self { + Self { + client: crate::client::build_client(cfg).await, + namer: crate::naming::Namer::new(&cfg.table_prefix), + } + } +} + +// ============================================================================ +// Inventory registrations — auto-register the DynamoDB backend at compile time +// ============================================================================ + +// 1. Bootstrapper registration +inventory::submit! { + extenddb_storage::bootstrapper::BackendRegistration { + name: "dynamodb", + factory: |config_path, cli_args| { + Box::pin(async move { + let store = bootstrapper::DynamoBootstrapper::from_config(&config_path, &cli_args).await?; + Ok(Box::new(store) as Box) + }) + }, + } +} + +// 2. Operations engine registration +inventory::submit! { + extenddb_storage::operations::OperationsEngineRegistration { + name: "dynamodb", + operations: &operations::DynamoOperationsEngine, + } +} + +// 3. Storage config deserializer registration +inventory::submit! { + extenddb_storage::config::StorageConfigRegistration { + backend: "dynamodb", + deserializer: |table| { + crate::config::DynamoStorageConfig::from_table(table) + .map(|c| Box::new(c) as Box) + .map_err(|e| format!("Failed to parse dynamodb config: {e}")) + }, + } +} + +// 4. Settings store factory registration (catalog pool -> PostgresCatalogStore) +inventory::submit! { + extenddb_storage::settings_store::SettingsStoreRegistration { + backend: "dynamodb", + factory: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let pool = sqlx::PgPool::connect(&connection_string) + .await + .map_err(|e| extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed(e.to_string()))?; + Ok(Box::new(extenddb_storage_postgres::PostgresCatalogStore::new(pool)) + as Box) + }) + }, + } +} + +// 5. Diagnostics store factory registration (catalog pool -> PostgresCatalogStore) +inventory::submit! { + extenddb_storage::diagnostics_store::DiagnosticsStoreRegistration { + backend: "dynamodb", + factory: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let pool = sqlx::PgPool::connect(&connection_string) + .await + .map_err(|e| extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed(e.to_string()))?; + Ok(Box::new(extenddb_storage_postgres::PostgresCatalogStore::new(pool)) + as Box) + }) + }, + } +} diff --git a/crates/storage-dynamodb/src/metadata_engine.rs b/crates/storage-dynamodb/src/metadata_engine.rs new file mode 100644 index 00000000..1a3adf5c --- /dev/null +++ b/crates/storage-dynamodb/src/metadata_engine.rs @@ -0,0 +1,405 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MetadataEngine` implementation for the DynamoDB-at-home backend. +//! +//! TTL is handled natively by DynamoDB — no ExtendDB TTL worker is needed. +//! Tag methods resolve the ExtendDB ARN to the underlying DynamoDB table ARN +//! via `DescribeTable`, then forward to the DynamoDB Tagging API. +//! Table listing uses `ListTables` filtered to the account prefix. + +use futures::future::BoxFuture; + +use extenddb_core::types::{Item, Tag, TimeToLiveDescription, TimeToLiveStatus}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{MetadataEngine, TtlTableInfo}; + +use crate::DynamoEngine; + +impl MetadataEngine for DynamoEngine { + // ── TTL ─────────────────────────────────────────────────────────────────── + + fn describe_ttl( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let physical = self.namer.physical(&account_id, &table_name); + let out = self + .client + .describe_time_to_live() + .table_name(physical) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + + let (time_to_live_status, attribute_name) = match out.time_to_live_description() { + Some(sdk_ttl) => { + let status = match sdk_ttl.time_to_live_status() { + Some(aws_sdk_dynamodb::types::TimeToLiveStatus::Enabled) + | Some(aws_sdk_dynamodb::types::TimeToLiveStatus::Enabling) => { + TimeToLiveStatus::Enabled + } + _ => TimeToLiveStatus::Disabled, + }; + (status, sdk_ttl.attribute_name().map(str::to_owned)) + } + None => (TimeToLiveStatus::Disabled, None), + }; + Ok(TimeToLiveDescription { + time_to_live_status, + attribute_name, + }) + }) + } + + fn update_ttl( + &self, + account_id: &str, + table_name: &str, + attribute_name: &str, + enabled: bool, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + let attribute_name = attribute_name.to_owned(); + Box::pin(async move { + let physical = self.namer.physical(&account_id, &table_name); + let spec = aws_sdk_dynamodb::types::TimeToLiveSpecification::builder() + .enabled(enabled) + .attribute_name(attribute_name) + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?; + self.client + .update_time_to_live() + .table_name(physical) + .time_to_live_specification(spec) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + Ok(()) + }) + } + + // ── Tags ────────────────────────────────────────────────────────────────── + // + // ExtendDB ARNs follow the same format as real DynamoDB ARNs: + // arn:aws:dynamodb:::table/ + // + // We parse the incoming ARN to recover (account_id, logical_table_name), + // resolve the physical table name, then call DescribeTable to get the + // real DynamoDB table ARN, which is what the Tagging API requires. + + fn tag_resource(&self, arn: &str, tags: &[Tag]) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_owned(); + let tags = tags.to_owned(); + Box::pin(async move { + let real_arn = self.resolve_table_arn_from_extenddb_arn(&arn).await?; + let sdk_tags: Result, _> = tags + .iter() + .map(|t| { + aws_sdk_dynamodb::types::Tag::builder() + .key(t.key.clone()) + .value(t.value.clone()) + .build() + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect(); + self.client + .tag_resource() + .resource_arn(real_arn) + .set_tags(Some(sdk_tags?)) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + Ok(()) + }) + } + + fn untag_resource( + &self, + arn: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_owned(); + let tag_keys = tag_keys.to_owned(); + Box::pin(async move { + let real_arn = self.resolve_table_arn_from_extenddb_arn(&arn).await?; + let mut req = self.client.untag_resource().resource_arn(real_arn); + for key in &tag_keys { + req = req.tag_keys(key.clone()); + } + req.send().await.map_err(crate::errors::from_sdk_error)?; + Ok(()) + }) + } + + fn list_tags(&self, arn: &str) -> BoxFuture<'_, Result, StorageError>> { + let arn = arn.to_owned(); + Box::pin(async move { + let real_arn = self.resolve_table_arn_from_extenddb_arn(&arn).await?; + let out = self + .client + .list_tags_of_resource() + .resource_arn(real_arn) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + let tags = out + .tags() + .iter() + .map(|t| Tag { + key: t.key().to_owned(), + value: t.value().to_owned(), + }) + .collect(); + Ok(tags) + }) + } + + // ── Table listing ───────────────────────────────────────────────────────── + + fn list_active_table_names( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let prefix = self.namer.account_prefix(&account_id); + let mut table_names = Vec::new(); + let mut exclusive_start: Option = None; + + loop { + let mut req = self.client.list_tables(); + if let Some(start) = exclusive_start.take() { + req = req.exclusive_start_table_name(start); + } + let out = req.send().await.map_err(crate::errors::from_sdk_error)?; + + for phys in out.table_names() { + if phys.starts_with(&prefix) + && let Ok(logical) = self.namer.logical(&account_id, phys) + { + table_names.push(logical); + } + } + + match out.last_evaluated_table_name() { + Some(last) => exclusive_start = Some(last.to_owned()), + None => break, + } + } + + Ok(table_names) + }) + } + + fn all_active_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // Collect all physical table names across all accounts. + // Physical table format: _ + // We detect the account_id by stripping the fixed prefix and splitting on '_'. + let table_prefix = &self.namer.account_prefix(""); // prefix without account part: just self.prefix + let mut pairs = Vec::new(); + let mut exclusive_start: Option = None; + + loop { + let mut req = self.client.list_tables(); + if let Some(start) = exclusive_start.take() { + req = req.exclusive_start_table_name(start); + } + let out = req.send().await.map_err(crate::errors::from_sdk_error)?; + + for phys in out.table_names() { + if let Some((account_id, table_name)) = parse_physical_table(phys, table_prefix) + { + pairs.push((account_id, table_name)); + } + } + + match out.last_evaluated_table_name() { + Some(last) => exclusive_start = Some(last.to_owned()), + None => break, + } + } + + Ok(pairs) + }) + } + + fn refresh_table_size( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + // DynamoDB maintains table size and item counts natively via DescribeTable. + // ExtendDB's refresh_table_size exists for backends (Postgres) that cache + // these metrics separately. Here, there is nothing to recompute. + Box::pin(async { Ok(()) }) + } + + // ── TTL worker no-ops ───────────────────────────────────────────────────── + // + // DynamoDB performs TTL deletion itself, asynchronously, in the background. + // ExtendDB's TTL worker (which calls these methods) has nothing to do for + // this backend. All methods below return empty/unit successes immediately. + + fn tables_with_ttl( + &self, + _account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + // DynamoDB handles TTL expiry internally; no table list needed by ExtendDB worker. + Box::pin(async { Ok(vec![]) }) + } + + fn all_tables_with_ttl(&self) -> BoxFuture<'_, Result, StorageError>> { + // DynamoDB handles TTL expiry internally; no cross-account table list needed. + Box::pin(async { Ok(vec![]) }) + } + + fn all_tables_with_ttl_index_ready( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + // DynamoDB handles TTL expiry internally; no TTL-index-ready concept applies. + Box::pin(async { Ok(vec![]) }) + } + + fn create_ttl_index( + &self, + _account_id: &str, + _table_name: &str, + _ttl_attribute: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + // DynamoDB manages its own TTL machinery; no secondary index for expiry needed. + Box::pin(async { Ok(()) }) + } + + fn drop_ttl_index( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + // DynamoDB manages its own TTL machinery; no index to drop. + Box::pin(async { Ok(()) }) + } + + fn find_expired_items_indexed( + &self, + _account_id: &str, + _table_name: &str, + _ttl_attribute: &str, + _limit: usize, + ) -> BoxFuture<'_, Result, StorageError>> { + // DynamoDB expires items itself; ExtendDB's TTL worker does not scan for expired items. + Box::pin(async { Ok(vec![]) }) + } +} + +// ── Internal helpers ────────────────────────────────────────────────────────── + +impl DynamoEngine { + /// Parse an ExtendDB table ARN to extract `(account_id, logical_table_name)`. + /// + /// ExtendDB ARNs use the same structure as real DynamoDB ARNs: + /// `arn:aws:dynamodb:::table/` + /// + /// We then resolve the physical name and call DescribeTable to get the + /// real DynamoDB ARN (needed for the Tagging API). + /// + /// # Concern + /// + /// This is best-effort: if the incoming ARN is a stream or index ARN rather + /// than a table ARN, parsing will fail and an error will be returned. + /// The implementation only handles `arn:aws:dynamodb:...:table/`. + async fn resolve_table_arn_from_extenddb_arn(&self, arn: &str) -> Result { + // ARN format: arn:aws:dynamodb:::table/ + // Split on ':' → ["arn", "aws", "dynamodb", "", "", "table/"] + let segments: Vec<&str> = arn.splitn(6, ':').collect(); + if segments.len() < 6 { + return Err(StorageError::Validation(format!( + "Invalid ExtendDB ARN (too few segments): {arn}" + ))); + } + let account_id = segments[4]; + let resource = segments[5]; // e.g. "table/MyTable" + let logical_table_name = resource.strip_prefix("table/").ok_or_else(|| { + StorageError::Validation(format!( + "Only table ARNs are supported for tag operations (got resource '{resource}')" + )) + })?; + + let physical = self.namer.physical(account_id, logical_table_name); + let out = self + .client + .describe_table() + .table_name(&physical) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + + out.table() + .and_then(|t| t.table_arn()) + .map(str::to_owned) + .ok_or_else(|| { + StorageError::Internal(format!( + "DescribeTable for '{physical}' returned no TableArn" + )) + }) + } +} + +/// Parse a physical table name (`_`) back into +/// `(account_id, logical_table_name)`. +/// +/// `table_prefix` is the fixed prefix without the account part (e.g. `"athome_"`). +/// +/// Returns `None` if the name does not match the expected pattern. +fn parse_physical_table(physical: &str, table_prefix: &str) -> Option<(String, String)> { + let rest = physical.strip_prefix(table_prefix)?; + // rest = "_" + // account_ids are 12 digits, but we split at the first '_' following the prefix. + let sep = rest.find('_')?; + let account_id = &rest[..sep]; + let table_name = &rest[sep + 1..]; + if account_id.is_empty() || table_name.is_empty() { + return None; + } + Some((account_id.to_owned(), table_name.to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_physical_table_round_trips() { + assert_eq!( + parse_physical_table("athome_123456789012_Orders", "athome_"), + Some(("123456789012".to_owned(), "Orders".to_owned())) + ); + } + + #[test] + fn parse_physical_table_handles_underscores_in_name() { + assert_eq!( + parse_physical_table("athome_123456789012_my_orders_v2", "athome_"), + Some(("123456789012".to_owned(), "my_orders_v2".to_owned())) + ); + } + + #[test] + fn parse_physical_table_rejects_wrong_prefix() { + assert_eq!( + parse_physical_table("other_123456789012_Orders", "athome_"), + None + ); + } + + #[test] + fn parse_physical_table_rejects_no_separator() { + assert_eq!(parse_physical_table("athome_nodash", "athome_"), None); + } +} diff --git a/crates/storage-dynamodb/src/naming.rs b/crates/storage-dynamodb/src/naming.rs new file mode 100644 index 00000000..d374acb6 --- /dev/null +++ b/crates/storage-dynamodb/src/naming.rs @@ -0,0 +1,72 @@ +//! Maps ExtendDB's (account_id, logical table name) to a flat physical DynamoDB +//! table name. ExtendDB is multi-tenant; DynamoDB tables are flat per AWS +//! account, so we namespace: `_`. Default prefix is +//! `athome_`, because of course it is. + +#[derive(Debug, Clone)] +pub struct Namer { + prefix: String, +} + +impl Namer { + pub fn new(prefix: &str) -> Self { + Self { + prefix: prefix.to_owned(), + } + } + + /// `_
` + pub fn physical(&self, account_id: &str, table: &str) -> String { + format!("{}{}_{}", self.prefix, account_id, table) + } + + /// Inverse of `physical`, scoped to one account. Errors if `physical` does + /// not belong to `account_id`. + pub fn logical(&self, account_id: &str, physical: &str) -> Result { + let want = format!("{}{}_", self.prefix, account_id); + physical + .strip_prefix(&want) + .map(|s| s.to_owned()) + .ok_or_else(|| format!("physical table '{physical}' not in account '{account_id}'")) + } + + /// The account-scoped prefix used to filter ListTables results. + pub fn account_prefix(&self, account_id: &str) -> String { + format!("{}{}_", self.prefix, account_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn physical_name_combines_prefix_account_table() { + let n = Namer::new("athome_"); + assert_eq!( + n.physical("123456789012", "Orders"), + "athome_123456789012_Orders" + ); + } + + #[test] + fn logical_name_round_trips() { + let n = Namer::new("athome_"); + let phys = n.physical("123456789012", "Orders"); + assert_eq!(n.logical("123456789012", &phys).unwrap(), "Orders"); + } + + #[test] + fn logical_name_rejects_foreign_account() { + let n = Namer::new("athome_"); + let phys = n.physical("111111111111", "Orders"); + assert!(n.logical("222222222222", &phys).is_err()); + } + + #[test] + fn logical_name_preserves_underscores_in_table() { + let n = Namer::new("athome_"); + let phys = n.physical("123456789012", "my_orders_v2"); + assert_eq!(n.logical("123456789012", &phys).unwrap(), "my_orders_v2"); + } +} diff --git a/crates/storage-dynamodb/src/operations.rs b/crates/storage-dynamodb/src/operations.rs new file mode 100644 index 00000000..b98808ac --- /dev/null +++ b/crates/storage-dynamodb/src/operations.rs @@ -0,0 +1,110 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DynamoDB backend implementation of `OperationsEngine`. +//! +//! The DynamoDB backend stores its catalog in Postgres, so +//! `parse_connection_string`, `redact_connection_string`, and `is_sensitive_key` +//! operate on Postgres connection-string semantics. +//! +//! `validate_identifier` uses DynamoDB table-name rules (length 3–255, +//! characters from `[A-Za-z0-9_.-]`). +//! +//! `catalog_version` returns the same version as `PostgresOperationsEngine` +//! because the catalog schema itself is Postgres. + +use extenddb_storage::error::StorageError; +use extenddb_storage::operations::{ConnectionParts, OperationsEngine}; + +/// DynamoDB operations engine for ddbo CLI commands. +/// +/// This is a unit struct (no fields) because all behavior is either +/// pure logic or delegates to `extenddb_storage_postgres`. +pub struct DynamoOperationsEngine; + +impl OperationsEngine for DynamoOperationsEngine { + /// Parse a Postgres catalog connection string. + /// + /// The DynamoDB backend's connection string is a Postgres URL pointing at + /// the catalog database. Delegates to `extenddb_storage_postgres` parsing. + fn parse_connection_string(&self, s: &str) -> Result { + let parts = extenddb_storage_postgres::parse_connection_string(s) + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(ConnectionParts { + host: parts.host, + port: parts.port, + user: parts.user, + password: parts.password, + database: parts.database, + }) + } + + /// Redact the password from a Postgres connection string. + /// + /// Handles `postgresql://user:password@host:port/database` format — same + /// as the Postgres backend because the catalog URL has that shape. + fn redact_connection_string(&self, s: &str) -> String { + // Redact password from postgresql://user:password@host:port/database + if let Some(at) = s.find('@') + && let Some(colon) = s[..at].rfind(':') + { + let scheme_end = s.find("://").map_or(0, |i| i + 3); + if colon >= scheme_end { + return format!("{}:***@{}", &s[..colon], &s[at + 1..]); + } + } + s.to_owned() + } + + /// Validate a DynamoDB table name / identifier. + /// + /// DynamoDB table names must be between 3 and 255 characters and consist + /// solely of letters (`A–Z`, `a–z`), digits (`0–9`), underscores (`_`), + /// hyphens (`-`), and dots (`.`). + fn validate_identifier(&self, name: &str, label: &str) -> Result<(), StorageError> { + let len = name.len(); + if !(3..=255).contains(&len) { + return Err(StorageError::Validation(format!( + "{label} '{name}' is not a valid DynamoDB identifier: \ + length {len} is outside the allowed range 3–255" + ))); + } + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')) + { + return Err(StorageError::Validation(format!( + "{label} '{name}' is not a valid DynamoDB identifier: \ + only [A-Za-z0-9_.-] are allowed" + ))); + } + Ok(()) + } + + /// Return the catalog schema version. + /// + /// The catalog lives in Postgres, so we return the same version string + /// that `PostgresOperationsEngine::catalog_version()` returns — they share + /// the same `CATALOG_VERSION` constant. + fn catalog_version(&self) -> String { + extenddb_storage_postgres::CATALOG_VERSION.to_string() + } + + /// Check whether a configuration key holds sensitive data. + /// + /// Mirrors the Postgres backend's logic: true for keys whose lowercase form + /// contains `"connection_string"`, `"password"`, `"secret"`, `"token"`, or + /// `"encryption_key"`. + fn is_sensitive_key(&self, key: &str) -> bool { + let lower = key.to_lowercase(); + [ + "connection_string", + "password", + "secret", + "token", + "encryption_key", + ] + .iter() + .any(|pattern| lower.contains(pattern)) + } +} diff --git a/crates/storage-dynamodb/src/server_components.rs b/crates/storage-dynamodb/src/server_components.rs new file mode 100644 index 00000000..0cde4ca1 --- /dev/null +++ b/crates/storage-dynamodb/src/server_components.rs @@ -0,0 +1,83 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! ServerComponents factory for the DynamoDB-at-home backend. +//! +//! Data plane: real DynamoDB via `DynamoEngine`. +//! Catalog/IAM/auth plane: PostgreSQL via `extenddb-storage-postgres`. + +use std::sync::Arc; + +use extenddb_auth::CredentialStore; +use extenddb_storage::StorageEngine; +use extenddb_storage::config::StorageConfig as _; +use extenddb_storage::server_components::{ + BackendError, ServerComponents, ServerComponentsRegistration, +}; +use extenddb_storage_postgres::{DbCredentialStore, PostgresCatalogStore}; +use sqlx::postgres::PgPoolOptions; + +use crate::DynamoEngine; + +inventory::submit! { + ServerComponentsRegistration { + backend: "dynamodb", + factory: |config, region| { + let _region = region.to_string(); + let cfg = match config.as_any().downcast_ref::() { + Some(c) => c.clone(), + None => return Box::pin(async { + Err(BackendError::InitializationFailed( + "dynamodb ServerComponents factory received a non-dynamodb config".into(), + )) + }), + }; + Box::pin(async move { + // 1. Data engine -> real DynamoDB + let engine: Arc = + Arc::new(DynamoEngine::from_config(&cfg).await); + + // 2. Catalog pool from the Postgres catalog connection string + let catalog_pool = PgPoolOptions::new() + .max_connections(cfg.max_catalog_connections()) + .connect(&cfg.catalog_connection_string) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "dynamodb".into(), + details: format!("catalog pool: {e}"), + })?; + + // 3. Fetch encryption key from the catalog settings table + let enc_key: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'encryption_key'") + .fetch_optional(&catalog_pool) + .await + .map_err(|e| BackendError::InitializationFailed( + format!("fetch encryption key: {e}"), + ))?; + + let catalog_store = Arc::new(match enc_key { + Some(k) => PostgresCatalogStore::with_encryption_key(catalog_pool.clone(), k), + None => return Err(BackendError::MissingEncryptionKey), + }) as Arc; + + // 4. Credential store (reuse Postgres implementation); the bin + // layer wraps it in CachedCredentialStore and builds the + // auth provider. + let enc_key = + extenddb_storage::CatalogStore::cached_encryption_key(&*catalog_store) + .ok_or(BackendError::MissingEncryptionKey)?; + let cred_store: Arc = + Arc::new(DbCredentialStore::new(catalog_pool.clone(), enc_key)); + + // 5. No background workers needed: DynamoDB drives TTL/streams/control-plane itself. + Ok(ServerComponents { + engine, + catalog_store, + credential_store: cred_store, + runtime_hooks: None, + }) + }) + }, + } +} diff --git a/crates/storage-dynamodb/src/stream_engine.rs b/crates/storage-dynamodb/src/stream_engine.rs new file mode 100644 index 00000000..ed023d30 --- /dev/null +++ b/crates/storage-dynamodb/src/stream_engine.rs @@ -0,0 +1,152 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `StreamEngine` implementation for the DynamoDB-at-home backend — honest stubs. +//! +//! DynamoDB Streams is a separate API surface that this v1 backend does not +//! implement. Each method names the DynamoDB Streams call it would map to. +//! `cleanup_expired_stream_records` is a maintenance no-op (DynamoDB Streams +//! retains records for 24 hours and expires them automatically). + +use futures::future::BoxFuture; + +use extenddb_core::types::{DescribeStreamInput, StreamDescription, StreamRecord}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{StreamEngine, StreamListResult, StreamRecordsResult}; + +use crate::DynamoEngine; + +impl StreamEngine for DynamoEngine { + fn write_stream_record( + &self, + _account_id: &str, + _record: &StreamRecord, + _shard_id: &str, + _table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + // DynamoDB produces stream records natively on every write; ExtendDB does + // not insert them. This would map to DynamoDB Streams (records are produced + // by DynamoDB itself, not via an explicit write API). + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + DynamoDB produces stream records itself — there is no write API." + .into(), + )) + }) + } + + fn get_stream_records( + &self, + _shard_id: &str, + _after_sequence: Option<&str>, + _limit: i64, + ) -> BoxFuture<'_, StreamRecordsResult> { + // Maps to DynamoDB Streams GetShardIterator + GetRecords. + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams GetShardIterator + GetRecords." + .into(), + )) + }) + } + + fn describe_stream( + &self, + _account_id: &str, + _input: &DescribeStreamInput, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB Streams DescribeStream. + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams DescribeStream." + .into(), + )) + }) + } + + fn list_streams( + &self, + _account_id: &str, + _table_name: Option<&str>, + _limit: i64, + _exclusive_start_stream_arn: Option<&str>, + ) -> BoxFuture<'_, StreamListResult> { + // Maps to DynamoDB Streams ListStreams. + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams ListStreams." + .into(), + )) + }) + } + + fn cleanup_expired_stream_records( + &self, + _retention_hours: i64, + ) -> BoxFuture<'_, Result> { + // DynamoDB Streams retains records for 24 hours and expires them automatically. + // No explicit cleanup is needed or possible via the API. + Box::pin(async { Ok(0) }) + } + + fn assign_shard( + &self, + _account_id: &str, + _table_name: &str, + _partition_key: &str, + ) -> BoxFuture<'_, Result> { + // Maps to DynamoDB Streams shard management (shard assignment is implicit in the stream). + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams shard management." + .into(), + )) + }) + } + + fn next_sequence_number(&self, _shard_id: &str) -> BoxFuture<'_, Result> { + // Maps to DynamoDB Streams shard management (sequence numbers are assigned by DynamoDB). + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams shard management." + .into(), + )) + }) + } + + fn validate_shard( + &self, + _account_id: &str, + _stream_arn: &str, + _shard_id: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + // Maps to DynamoDB Streams shard management (DescribeStream to verify shard membership). + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams shard management (DescribeStream)." + .into(), + )) + }) + } + + fn latest_sequence_number( + &self, + _shard_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + // Maps to DynamoDB Streams shard management (GetShardIterator with LATEST). + Box::pin(async { + Err(StorageError::Internal( + "Streams are not implemented in the dynamodb backend (v1). \ + Maps to DynamoDB Streams shard management (GetShardIterator LATEST)." + .into(), + )) + }) + } +} diff --git a/crates/storage-dynamodb/src/table_engine.rs b/crates/storage-dynamodb/src/table_engine.rs new file mode 100644 index 00000000..7fa28b51 --- /dev/null +++ b/crates/storage-dynamodb/src/table_engine.rs @@ -0,0 +1,770 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `TableEngine` implementation for the DynamoDB-at-home backend. +//! +//! Each operation: +//! 1. Resolves the physical DynamoDB table name via `self.namer.physical`. +//! 2. Calls the corresponding AWS SDK operation. +//! 3. Maps the SDK response back to core `extenddb_core` types. + +use futures::future::BoxFuture; + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, + DescribeTableInput, GsiDescription, IndexInfo, KeySchemaElement, KeyType, ListTablesInput, + ListTablesOutput, LsiDescription, OnDemandThroughput, Projection, ProjectionType, + ProvisionedThroughputDescription, ScalarAttributeType, StreamSpecification, StreamViewType, + TableDescription, TableKeyInfo, TableStatus, UpdateTableInput, +}; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; + +use crate::DynamoEngine; + +// ── Trait implementation ───────────────────────────────────────────────────── + +impl TableEngine for DynamoEngine { + fn create_table( + &self, + account_id: &str, + input: CreateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let logical = input.table_name.clone(); + let physical = self.namer.physical(&account_id, &logical); + + // Map core KeySchemaElements → SDK KeySchemaElements + let sdk_key_schema: Result, _> = input + .key_schema + .iter() + .map(|ks| { + aws_sdk_dynamodb::types::KeySchemaElement::builder() + .attribute_name(ks.attribute_name.clone()) + .key_type(map_key_type_to_sdk(&ks.key_type)) + .build() + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect(); + let sdk_key_schema = sdk_key_schema?; + + // Map core AttributeDefinitions → SDK AttributeDefinitions + let sdk_attr_defs: Result, _> = input + .attribute_definitions + .iter() + .map(|ad| { + aws_sdk_dynamodb::types::AttributeDefinition::builder() + .attribute_name(ad.attribute_name.clone()) + .attribute_type(map_scalar_type_to_sdk(&ad.attribute_type)) + .build() + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect(); + let sdk_attr_defs = sdk_attr_defs?; + + // Build the create_table request + let mut req = self + .client + .create_table() + .table_name(physical) + .set_key_schema(Some(sdk_key_schema)) + .set_attribute_definitions(Some(sdk_attr_defs)); + + // Billing mode + let billing_mode = input.billing_mode.unwrap_or(BillingMode::PayPerRequest); + req = req.billing_mode(map_billing_mode_to_sdk(&billing_mode)); + + // Provisioned throughput (only set when billing mode is Provisioned) + if matches!(billing_mode, BillingMode::Provisioned) + && let Some(pt) = &input.provisioned_throughput + { + let sdk_pt = aws_sdk_dynamodb::types::ProvisionedThroughput::builder() + .read_capacity_units(pt.read_capacity_units) + .write_capacity_units(pt.write_capacity_units) + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?; + req = req.provisioned_throughput(sdk_pt); + } + + // GSIs + if let Some(gsis) = &input.global_secondary_indexes { + for gsi in gsis { + let sdk_gsi_ks: Result, _> = gsi + .key_schema + .iter() + .map(|ks| { + aws_sdk_dynamodb::types::KeySchemaElement::builder() + .attribute_name(ks.attribute_name.clone()) + .key_type(map_key_type_to_sdk(&ks.key_type)) + .build() + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect(); + let sdk_gsi_ks = sdk_gsi_ks?; + + let sdk_proj = build_sdk_projection(&gsi.projection); + + let mut gsi_builder = aws_sdk_dynamodb::types::GlobalSecondaryIndex::builder() + .index_name(gsi.index_name.clone()) + .set_key_schema(Some(sdk_gsi_ks)) + .projection(sdk_proj); + + if let Some(pt) = &gsi.provisioned_throughput { + let sdk_pt = aws_sdk_dynamodb::types::ProvisionedThroughput::builder() + .read_capacity_units(pt.read_capacity_units) + .write_capacity_units(pt.write_capacity_units) + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?; + gsi_builder = gsi_builder.provisioned_throughput(sdk_pt); + } + + let sdk_gsi = gsi_builder + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?; + req = req.global_secondary_indexes(sdk_gsi); + } + } + + // LSIs + if let Some(lsis) = &input.local_secondary_indexes { + for lsi in lsis { + let sdk_lsi_ks: Result, _> = lsi + .key_schema + .iter() + .map(|ks| { + aws_sdk_dynamodb::types::KeySchemaElement::builder() + .attribute_name(ks.attribute_name.clone()) + .key_type(map_key_type_to_sdk(&ks.key_type)) + .build() + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect(); + let sdk_lsi_ks = sdk_lsi_ks?; + + let sdk_proj = build_sdk_projection(&lsi.projection); + + let sdk_lsi = aws_sdk_dynamodb::types::LocalSecondaryIndex::builder() + .index_name(lsi.index_name.clone()) + .set_key_schema(Some(sdk_lsi_ks)) + .projection(sdk_proj) + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?; + req = req.local_secondary_indexes(sdk_lsi); + } + } + + // Deletion protection + if let Some(dp) = input.deletion_protection_enabled { + req = req.deletion_protection_enabled(dp); + } + + let out = req.send().await.map_err(crate::errors::from_sdk_error)?; + + match out.table_description() { + Some(t) => to_table_description(t, &account_id, &self.namer), + None => Err(StorageError::Internal( + "create_table: no TableDescription in response".into(), + )), + } + }) + } + + fn delete_table( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let physical = self.namer.physical(&account_id, &input.table_name); + let out = self + .client + .delete_table() + .table_name(physical) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + + match out.table_description() { + Some(t) => to_table_description(t, &account_id, &self.namer), + None => Err(StorageError::Internal( + "delete_table: no TableDescription in response".into(), + )), + } + }) + } + + fn describe_table( + &self, + account_id: &str, + input: DescribeTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let logical = input.table_name.clone(); + let physical = self.namer.physical(&account_id, &logical); + let out = self + .client + .describe_table() + .table_name(physical) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + + match out.table() { + Some(t) => to_table_description(t, &account_id, &self.namer), + None => Err(StorageError::TableNotFound(logical)), + } + }) + } + + fn list_tables( + &self, + account_id: &str, + input: ListTablesInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let account_prefix = self.namer.account_prefix(&account_id); + + // Build the SDK request, honoring pagination parameters + let mut req = self.client.list_tables(); + + if let Some(limit) = input.limit { + // SDK takes i32; core limit is i32 — pass through directly + req = req.limit(limit); + } + if let Some(start) = &input.exclusive_start_table_name { + // The core exclusive_start_table_name is a *logical* name; we must + // translate it to the physical name before passing to DynamoDB. + let physical_start = self.namer.physical(&account_id, start); + req = req.exclusive_start_table_name(physical_start); + } + + let out = req.send().await.map_err(crate::errors::from_sdk_error)?; + + // Filter to this account's tables and strip the physical prefix back to logical + let table_names: Vec = out + .table_names() + .iter() + .filter(|phys| phys.starts_with(&account_prefix)) + .filter_map(|phys| self.namer.logical(&account_id, phys).ok()) + .collect(); + + // last_evaluated_table_name from DynamoDB is a physical name; convert to logical + let last_evaluated_table_name = out + .last_evaluated_table_name() + .and_then(|phys| self.namer.logical(&account_id, phys).ok()); + + Ok(ListTablesOutput { + table_names, + last_evaluated_table_name, + }) + }) + } + + fn update_table( + &self, + account_id: &str, + input: UpdateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let physical = self.namer.physical(&account_id, &input.table_name); + + let mut req = self.client.update_table().table_name(physical); + + // Billing mode change + if let Some(bm) = &input.billing_mode { + req = req.billing_mode(map_billing_mode_to_sdk(bm)); + } + + // Provisioned throughput change + if let Some(pt) = &input.provisioned_throughput { + let sdk_pt = aws_sdk_dynamodb::types::ProvisionedThroughput::builder() + .read_capacity_units(pt.read_capacity_units) + .write_capacity_units(pt.write_capacity_units) + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?; + req = req.provisioned_throughput(sdk_pt); + } + + // Deletion protection change + if let Some(dp) = input.deletion_protection_enabled { + req = req.deletion_protection_enabled(dp); + } + + // GSI updates + // TODO: CreateGsiAction and DeleteGsiAction on UpdateTable are not yet forwarded. + // The SDK's GlobalSecondaryIndexUpdate type supports Create/Update/Delete actions, + // but the core types (CreateGsiAction, DeleteGsiAction, UpdateGsiAction) need a + // full mapping to SDK GlobalSecondaryIndex/UpdateGlobalSecondaryIndexAction types. + // For now we accept the input but only forward billing/throughput/deletion-protection. + if input.global_secondary_index_updates.is_some() { + return Err(StorageError::Internal( + "update_table: GlobalSecondaryIndexUpdates not yet supported in the DynamoDB backend".into(), + )); + } + + let out = req.send().await.map_err(crate::errors::from_sdk_error)?; + + match out.table_description() { + Some(t) => to_table_description(t, &account_id, &self.namer), + None => Err(StorageError::Internal( + "update_table: no TableDescription in response".into(), + )), + } + }) + } + + fn table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let physical = self.namer.physical(&account_id, &table_name); + let out = self + .client + .describe_table() + .table_name(physical) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + + let t = out + .table() + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + // Key schema + let key_schema: Vec = + t.key_schema().iter().map(map_key_schema_from_sdk).collect(); + + // Attribute definitions + let attribute_definitions: Vec = t + .attribute_definitions() + .iter() + .map(map_attr_def_from_sdk) + .collect(); + + // Has LSI? + let has_lsi = !t.local_secondary_indexes().is_empty(); + + // Stream specification + let stream_specification = t.stream_specification().map(map_stream_spec_from_sdk); + + // table_id: prefer SDK TableId, fall back to ARN, then physical name + let physical_fallback = self.namer.physical(&account_id, &table_name); + let table_id = t + .table_id() + .or(t.table_arn()) + .unwrap_or(physical_fallback.as_str()) + .to_owned(); + + Ok(TableKeyInfo { + table_name, + account_id, + table_id, + // Base-table lookup: the base schema IS the key schema. + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + has_lsi, + stream_specification, + }) + }) + } + + fn index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + let index_name = index_name.to_owned(); + Box::pin(async move { + let physical = self.namer.physical(&account_id, &table_name); + let out = self + .client + .describe_table() + .table_name(physical) + .send() + .await + .map_err(crate::errors::from_sdk_error)?; + + let t = out + .table() + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + // Search GSIs first + for gsi in t.global_secondary_indexes() { + if gsi.index_name() == Some(index_name.as_str()) { + let key_schema: Vec = gsi + .key_schema() + .iter() + .map(map_key_schema_from_sdk) + .collect(); + let projection = gsi + .projection() + .map(map_projection_from_sdk) + .unwrap_or_else(default_projection); + let index_id = gsi.index_arn().unwrap_or(&index_name).to_owned(); + return Ok(IndexInfo { + index_name, + index_id, + index_type: extenddb_core::types::IndexType::Gsi, + key_schema, + projection, + }); + } + } + + // Then LSIs + for lsi in t.local_secondary_indexes() { + if lsi.index_name() == Some(index_name.as_str()) { + let key_schema: Vec = lsi + .key_schema() + .iter() + .map(map_key_schema_from_sdk) + .collect(); + let projection = lsi + .projection() + .map(map_projection_from_sdk) + .unwrap_or_else(default_projection); + let index_id = lsi.index_arn().unwrap_or(&index_name).to_owned(); + return Ok(IndexInfo { + index_name, + index_id, + index_type: extenddb_core::types::IndexType::Lsi, + key_schema, + projection, + }); + } + } + + Err(StorageError::IndexNotFound(index_name)) + }) + } + + fn index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + // TODO: index_info_by_table_id is not tractable in the DynamoDB backend without a + // separate mapping from table_id → (account_id, physical_name). DynamoDB does not + // provide a reverse-lookup API for table IDs. A future implementation could maintain + // such a mapping in the catalog (Postgres) layer, but that is out of scope for v1. + let _ = (table_id, index_name); + Box::pin(async move { + Err(StorageError::Internal( + "index_info_by_table_id not supported in the DynamoDB backend".into(), + )) + }) + } +} + +// ── Helpers: core → SDK conversions ───────────────────────────────────────── + +fn map_key_type_to_sdk(kt: &KeyType) -> aws_sdk_dynamodb::types::KeyType { + match kt { + KeyType::Hash => aws_sdk_dynamodb::types::KeyType::Hash, + KeyType::Range => aws_sdk_dynamodb::types::KeyType::Range, + } +} + +fn map_scalar_type_to_sdk( + sat: &ScalarAttributeType, +) -> aws_sdk_dynamodb::types::ScalarAttributeType { + match sat { + ScalarAttributeType::S => aws_sdk_dynamodb::types::ScalarAttributeType::S, + ScalarAttributeType::N => aws_sdk_dynamodb::types::ScalarAttributeType::N, + ScalarAttributeType::B => aws_sdk_dynamodb::types::ScalarAttributeType::B, + } +} + +fn map_billing_mode_to_sdk(bm: &BillingMode) -> aws_sdk_dynamodb::types::BillingMode { + match bm { + BillingMode::PayPerRequest => aws_sdk_dynamodb::types::BillingMode::PayPerRequest, + BillingMode::Provisioned => aws_sdk_dynamodb::types::BillingMode::Provisioned, + } +} + +fn build_sdk_projection(proj: &Projection) -> aws_sdk_dynamodb::types::Projection { + let sdk_pt = match proj.projection_type { + ProjectionType::All => aws_sdk_dynamodb::types::ProjectionType::All, + ProjectionType::KeysOnly => aws_sdk_dynamodb::types::ProjectionType::KeysOnly, + ProjectionType::Include => aws_sdk_dynamodb::types::ProjectionType::Include, + }; + let mut builder = aws_sdk_dynamodb::types::Projection::builder().projection_type(sdk_pt); + if let Some(attrs) = &proj.non_key_attributes { + for attr in attrs { + builder = builder.non_key_attributes(attr.clone()); + } + } + builder.build() +} + +// ── Helpers: SDK → core conversions ───────────────────────────────────────── + +fn map_key_schema_from_sdk(ks: &aws_sdk_dynamodb::types::KeySchemaElement) -> KeySchemaElement { + KeySchemaElement { + attribute_name: ks.attribute_name.clone(), + key_type: match ks.key_type { + aws_sdk_dynamodb::types::KeyType::Hash => KeyType::Hash, + aws_sdk_dynamodb::types::KeyType::Range => KeyType::Range, + _ => KeyType::Hash, // forward-compat: unknown → Hash + }, + } +} + +fn map_attr_def_from_sdk(ad: &aws_sdk_dynamodb::types::AttributeDefinition) -> AttributeDefinition { + AttributeDefinition { + attribute_name: ad.attribute_name.clone(), + attribute_type: match ad.attribute_type { + aws_sdk_dynamodb::types::ScalarAttributeType::S => ScalarAttributeType::S, + aws_sdk_dynamodb::types::ScalarAttributeType::N => ScalarAttributeType::N, + aws_sdk_dynamodb::types::ScalarAttributeType::B => ScalarAttributeType::B, + _ => ScalarAttributeType::S, // forward-compat + }, + } +} + +fn map_projection_from_sdk(proj: &aws_sdk_dynamodb::types::Projection) -> Projection { + let projection_type = match proj.projection_type() { + Some(aws_sdk_dynamodb::types::ProjectionType::All) => ProjectionType::All, + Some(aws_sdk_dynamodb::types::ProjectionType::KeysOnly) => ProjectionType::KeysOnly, + Some(aws_sdk_dynamodb::types::ProjectionType::Include) => ProjectionType::Include, + _ => ProjectionType::All, // forward-compat default + }; + let non_key_attributes = { + let attrs: Vec = proj.non_key_attributes().to_vec(); + if attrs.is_empty() { None } else { Some(attrs) } + }; + Projection { + projection_type, + non_key_attributes, + } +} + +fn default_projection() -> Projection { + Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + } +} + +fn map_stream_spec_from_sdk( + ss: &aws_sdk_dynamodb::types::StreamSpecification, +) -> StreamSpecification { + let stream_view_type = ss.stream_view_type().map(|svt| match svt { + aws_sdk_dynamodb::types::StreamViewType::KeysOnly => StreamViewType::KeysOnly, + aws_sdk_dynamodb::types::StreamViewType::NewImage => StreamViewType::NewImage, + aws_sdk_dynamodb::types::StreamViewType::OldImage => StreamViewType::OldImage, + aws_sdk_dynamodb::types::StreamViewType::NewAndOldImages => StreamViewType::NewAndOldImages, + _ => StreamViewType::KeysOnly, // forward-compat + }); + StreamSpecification { + stream_enabled: ss.stream_enabled(), + stream_view_type, + } +} + +fn map_table_status_from_sdk(ts: &aws_sdk_dynamodb::types::TableStatus) -> TableStatus { + match ts { + aws_sdk_dynamodb::types::TableStatus::Active => TableStatus::Active, + aws_sdk_dynamodb::types::TableStatus::Creating => TableStatus::Creating, + aws_sdk_dynamodb::types::TableStatus::Deleting => TableStatus::Deleting, + aws_sdk_dynamodb::types::TableStatus::Updating => TableStatus::Updating, + _ => TableStatus::Active, // forward-compat: Archived/Archiving/etc → Active + } +} + +// ── Main mapping helper ────────────────────────────────────────────────────── + +/// Map an SDK [`TableDescription`] to a core [`TableDescription`]. +/// +/// `account_id` is used to derive the logical table name via the namer. +fn to_table_description( + t: &aws_sdk_dynamodb::types::TableDescription, + account_id: &str, + namer: &crate::naming::Namer, +) -> Result { + // Logical table name: strip account prefix from physical + let physical_name = t.table_name().unwrap_or(""); + let table_name = namer + .logical(account_id, physical_name) + .unwrap_or_else(|_| physical_name.to_owned()); + + let table_status = t + .table_status() + .map(map_table_status_from_sdk) + .unwrap_or(TableStatus::Active); + + // creation_date_time: seconds since Unix epoch as f64 + let creation_date_time = t + .creation_date_time() + .map(|dt| dt.secs() as f64 + dt.subsec_nanos() as f64 / 1_000_000_000.0) + .unwrap_or(0.0); + + let table_size_bytes = t.table_size_bytes().unwrap_or(0); + let item_count = t.item_count().unwrap_or(0); + + let table_arn = t.table_arn().unwrap_or("").to_owned(); + let table_id = t.table_id().unwrap_or("").to_owned(); + + // Key schema and attribute definitions + let key_schema: Vec = + t.key_schema().iter().map(map_key_schema_from_sdk).collect(); + let attribute_definitions: Vec = t + .attribute_definitions() + .iter() + .map(map_attr_def_from_sdk) + .collect(); + + // ProvisionedThroughputDescription + let provisioned_throughput = t + .provisioned_throughput() + .map(|pt| ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units().unwrap_or(0), + write_capacity_units: pt.write_capacity_units().unwrap_or(0), + number_of_decreases_today: pt.number_of_decreases_today().unwrap_or(0), + last_increase_date_time: pt + .last_increase_date_time() + .map(|dt| dt.secs() as f64 + dt.subsec_nanos() as f64 / 1_000_000_000.0), + last_decrease_date_time: pt + .last_decrease_date_time() + .map(|dt| dt.secs() as f64 + dt.subsec_nanos() as f64 / 1_000_000_000.0), + }) + .unwrap_or(ProvisionedThroughputDescription { + read_capacity_units: 0, + write_capacity_units: 0, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }); + + // BillingModeSummary + let billing_mode_summary = t.billing_mode_summary().map(|bms| BillingModeSummary { + billing_mode: bms + .billing_mode() + .map(|bm| match bm { + aws_sdk_dynamodb::types::BillingMode::PayPerRequest => BillingMode::PayPerRequest, + aws_sdk_dynamodb::types::BillingMode::Provisioned => BillingMode::Provisioned, + _ => BillingMode::PayPerRequest, + }) + .unwrap_or(BillingMode::PayPerRequest), + last_update_to_pay_per_request_date_time: bms + .last_update_to_pay_per_request_date_time() + .map(|dt| dt.secs() as f64 + dt.subsec_nanos() as f64 / 1_000_000_000.0), + }); + + // GSIs + let sdk_gsis = t.global_secondary_indexes(); + let global_secondary_indexes = if sdk_gsis.is_empty() { + None + } else { + Some( + sdk_gsis + .iter() + .map(|gsi| GsiDescription { + index_name: gsi.index_name().unwrap_or("").to_owned(), + key_schema: gsi + .key_schema() + .iter() + .map(map_key_schema_from_sdk) + .collect(), + projection: gsi + .projection() + .map(map_projection_from_sdk) + .unwrap_or_else(default_projection), + index_status: gsi + .index_status() + .map(|s| s.as_str().to_owned()) + .unwrap_or_else(|| "ACTIVE".to_owned()), + provisioned_throughput: gsi.provisioned_throughput().map(|pt| { + ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units().unwrap_or(0), + write_capacity_units: pt.write_capacity_units().unwrap_or(0), + number_of_decreases_today: pt.number_of_decreases_today().unwrap_or(0), + last_increase_date_time: pt.last_increase_date_time().map(|dt| { + dt.secs() as f64 + dt.subsec_nanos() as f64 / 1_000_000_000.0 + }), + last_decrease_date_time: pt.last_decrease_date_time().map(|dt| { + dt.secs() as f64 + dt.subsec_nanos() as f64 / 1_000_000_000.0 + }), + } + }), + index_size_bytes: gsi.index_size_bytes().unwrap_or(0), + item_count: gsi.item_count().unwrap_or(0), + index_arn: gsi.index_arn().unwrap_or("").to_owned(), + }) + .collect(), + ) + }; + + // LSIs + let sdk_lsis = t.local_secondary_indexes(); + let local_secondary_indexes = if sdk_lsis.is_empty() { + None + } else { + Some( + sdk_lsis + .iter() + .map(|lsi| LsiDescription { + index_name: lsi.index_name().unwrap_or("").to_owned(), + key_schema: lsi + .key_schema() + .iter() + .map(map_key_schema_from_sdk) + .collect(), + projection: lsi + .projection() + .map(map_projection_from_sdk) + .unwrap_or_else(default_projection), + index_size_bytes: lsi.index_size_bytes().unwrap_or(0), + item_count: lsi.item_count().unwrap_or(0), + index_arn: lsi.index_arn().unwrap_or("").to_owned(), + }) + .collect(), + ) + }; + + // Stream specification + let stream_specification = t.stream_specification().map(map_stream_spec_from_sdk); + let latest_stream_arn = t.latest_stream_arn().map(str::to_owned); + let latest_stream_label = t.latest_stream_label().map(str::to_owned); + + let deletion_protection_enabled = t.deletion_protection_enabled().unwrap_or(false); + + Ok(TableDescription { + table_name, + key_schema, + attribute_definitions, + table_status, + creation_date_time, + table_size_bytes, + item_count, + table_arn, + table_id, + provisioned_throughput, + billing_mode_summary, + global_secondary_indexes, + local_secondary_indexes, + stream_specification, + latest_stream_arn, + latest_stream_label, + deletion_protection_enabled, + sse_description: None, // Not mapped in v1 + table_class_summary: None, + on_demand_throughput: t.on_demand_throughput().map(|odt| OnDemandThroughput { + max_read_request_units: odt.max_read_request_units(), + max_write_request_units: odt.max_write_request_units(), + }), + }) +} diff --git a/crates/storage-dynamodb/src/worker_store.rs b/crates/storage-dynamodb/src/worker_store.rs new file mode 100644 index 00000000..61e3d718 --- /dev/null +++ b/crates/storage-dynamodb/src/worker_store.rs @@ -0,0 +1,27 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `WorkerStore` implementation for the DynamoDB-at-home backend. +//! +//! DynamoDB drives its own CREATING→ACTIVE transitions internally; DescribeTable +//! reports the real status. ExtendDB's control-plane transition worker has +//! nothing to advance for this backend. + +use futures::future::BoxFuture; + +use extenddb_storage::WorkerStore; +use extenddb_storage::error::StorageError; + +use crate::DynamoEngine; + +impl WorkerStore for DynamoEngine { + /// DynamoDB manages its own table lifecycle transitions (CREATING→ACTIVE, + /// DELETING→deleted). `DescribeTable` reflects the live status, so there + /// are no pending transitions for ExtendDB to advance. Always returns an + /// empty list. + fn process_control_plane_transitions( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(vec![]) }) + } +} diff --git a/crates/storage-dynamodb/tests/integration.rs b/crates/storage-dynamodb/tests/integration.rs new file mode 100644 index 00000000..af2b8b16 --- /dev/null +++ b/crates/storage-dynamodb/tests/integration.rs @@ -0,0 +1,475 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for the DynamoDB-at-home storage backend. +//! +//! These tests exercise a real `DynamoEngine` against a live DynamoDB endpoint. +//! They are gated on the `DDB_LOCAL_ENDPOINT` environment variable: if it is +//! unset, each test prints a skip message and returns immediately, so +//! `cargo test -p extenddb-storage-dynamodb` stays green in CI without the +//! variable. +//! +//! # Running against DynamoDB Local +//! +//! 1. Start DynamoDB Local: +//! ```sh +//! docker run -d -p 8000:8000 amazon/dynamodb-local +//! ``` +//! +//! 2. Run the integration tests (data plane only — no Postgres needed): +//! ```sh +//! DDB_LOCAL_ENDPOINT=http://localhost:8000 \ +//! AWS_ACCESS_KEY_ID=dummy \ +//! AWS_SECRET_ACCESS_KEY=dummy \ +//! AWS_REGION=us-east-1 \ +//! AWS_DEFAULT_REGION=us-east-1 \ +//! cargo test -p extenddb-storage-dynamodb --test integration -- --nocapture +//! ``` +//! +//! Note: `serve` additionally requires a Postgres catalog. These tests only +//! exercise the data plane (table lifecycle + item CRUD + query + conditions). + +use std::collections::HashMap; + +use extenddb_core::expression::{ + ExpressionMaps, parse_condition, parse_key_condition, parse_update, tokenize, +}; +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, CreateTableInput, DeleteTableInput, DescribeTableInput, + Item, KeySchemaElement, KeyType, ScalarAttributeType, TableStatus, +}; +use extenddb_storage::{DataEngine, TableEngine}; +use extenddb_storage_dynamodb::{DynamoEngine, config::DynamoStorageConfig}; + +// --------------------------------------------------------------------------- +// Helper: read DDB_LOCAL_ENDPOINT (returns None if unset → test skips) +// --------------------------------------------------------------------------- + +fn endpoint() -> Option { + std::env::var("DDB_LOCAL_ENDPOINT").ok() +} + +// --------------------------------------------------------------------------- +// Helper: build a DynamoStorageConfig pointing at the local endpoint +// --------------------------------------------------------------------------- + +fn make_config(ep: &str) -> DynamoStorageConfig { + let toml_str = format!( + r#" +region = "us-east-1" +endpoint_url = "{ep}" +table_prefix = "it_" +catalog_connection_string = "postgresql://unused" +pool_size = 20 +"# + ); + let t: toml::Table = toml::from_str(&toml_str).expect("config toml parse"); + DynamoStorageConfig::from_table(&t).expect("DynamoStorageConfig::from_table") +} + +// --------------------------------------------------------------------------- +// Helper: create a simple string Item +// --------------------------------------------------------------------------- + +fn s(v: &str) -> AttributeValue { + AttributeValue::S(v.to_owned()) +} + +fn n(v: &str) -> AttributeValue { + AttributeValue::N(v.to_owned()) +} + +fn item(pairs: &[(&str, AttributeValue)]) -> Item { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect() +} + +// --------------------------------------------------------------------------- +// Helper: build a CreateTableInput for a table with pk (S, HASH) + sk (S, RANGE) +// --------------------------------------------------------------------------- + +fn create_pk_sk_table(logical_name: &str) -> CreateTableInput { + CreateTableInput { + table_name: logical_name.to_owned(), + key_schema: vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ], + attribute_definitions: vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ], + billing_mode: None, // defaults to PAY_PER_REQUEST + provisioned_throughput: None, + global_secondary_indexes: None, + local_secondary_indexes: None, + stream_specification: None, + sse_specification: None, + tags: None, + deletion_protection_enabled: None, + table_class: None, + on_demand_throughput: None, + } +} + +// --------------------------------------------------------------------------- +// Helper: empty ExpressionMaps +// --------------------------------------------------------------------------- + +fn empty_maps() -> ExpressionMaps { + ExpressionMaps::new(HashMap::new(), HashMap::new()) +} + +// --------------------------------------------------------------------------- +// Helper: pre-delete a table if it exists (ignore TableNotFound) +// --------------------------------------------------------------------------- + +async fn try_delete_table(engine: &DynamoEngine, account_id: &str, table_name: &str) { + let _ = engine + .delete_table( + account_id, + DeleteTableInput { + table_name: table_name.to_owned(), + }, + ) + .await; +} + +// ============================================================================ +// Test 1: Table lifecycle + item CRUD +// ============================================================================ +// +// - create_table → assert returns; describe_table → assert status active +// - table_key_info for data ops +// - put_item, get_item, update_item SET, delete_item + +#[tokio::test] +async fn test_table_lifecycle_and_item_crud() { + let Some(ep) = endpoint() else { + eprintln!("skipping test_table_lifecycle_and_item_crud: DDB_LOCAL_ENDPOINT unset"); + return; + }; + + let cfg = make_config(&ep); + let engine = DynamoEngine::from_config(&cfg).await; + let account_id = "000000000001"; + let table_name = "it_crud"; + + // Clean up from a prior run (ignore errors) + try_delete_table(&engine, account_id, table_name).await; + + // --- create_table --- + let desc = engine + .create_table(account_id, create_pk_sk_table(table_name)) + .await + .expect("create_table failed"); + assert_eq!( + desc.table_name, table_name, + "logical table name should match" + ); + eprintln!("[crud] create_table OK, table_name={}", desc.table_name); + + // --- describe_table --- + let desc2 = engine + .describe_table( + account_id, + DescribeTableInput { + table_name: table_name.to_owned(), + }, + ) + .await + .expect("describe_table failed"); + assert_eq!(desc2.table_name, table_name); + assert_eq!(desc2.table_status, TableStatus::Active); + eprintln!("[crud] describe_table OK, status={:?}", desc2.table_status); + + // --- table_key_info --- + let key_info = engine + .table_key_info(account_id, table_name) + .await + .expect("table_key_info failed"); + assert_eq!(key_info.table_name, table_name); + eprintln!("[crud] table_key_info OK"); + + // --- put_item: {pk: "u#1", sk: "p#1", name: "Bob", n: N"42"} --- + let full_item = item(&[ + ("pk", s("u#1")), + ("sk", s("p#1")), + ("name", s("Bob")), + ("n", n("42")), + ]); + engine + .put_item(&key_info, full_item, false, None, &empty_maps(), None) + .await + .expect("put_item failed"); + eprintln!("[crud] put_item OK"); + + // --- get_item by key {pk, sk} → assert Some and name == "Bob" --- + let key = item(&[("pk", s("u#1")), ("sk", s("p#1"))]); + let got = engine + .get_item(&key_info, &key) + .await + .expect("get_item failed"); + let got_item = got.expect("get_item returned None, expected item"); + assert_eq!(got_item.get("name"), Some(&s("Bob")), "name should be Bob"); + eprintln!("[crud] get_item OK: name={:?}", got_item.get("name")); + + // --- update_item: SET n = :v where :v = N"43" --- + let update_tokens = tokenize("SET n = :v").expect("tokenize update"); + let actions = parse_update(&update_tokens).expect("parse_update"); + let mut values: HashMap = HashMap::new(); + values.insert("v".to_owned(), n("43")); + let update_maps = ExpressionMaps::new(HashMap::new(), values); + engine + .update_item( + &key_info, + &key, + &actions, + false, + false, + None, + &update_maps, + None, + ) + .await + .expect("update_item failed"); + eprintln!("[crud] update_item OK"); + + // Verify n changed to 43 + let got2 = engine + .get_item(&key_info, &key) + .await + .expect("get_item after update failed") + .expect("get_item after update returned None"); + assert_eq!(got2.get("n"), Some(&n("43")), "n should be 43 after update"); + eprintln!("[crud] verify update OK: n={:?}", got2.get("n")); + + // --- delete_item --- + engine + .delete_item(&key_info, &key, false, None, &empty_maps(), None) + .await + .expect("delete_item failed"); + eprintln!("[crud] delete_item OK"); + + // Verify item is gone + let got3 = engine + .get_item(&key_info, &key) + .await + .expect("get_item after delete failed"); + assert!(got3.is_none(), "item should be None after delete"); + eprintln!("[crud] verify delete OK: item is None"); + + // --- delete_table --- + engine + .delete_table( + account_id, + DeleteTableInput { + table_name: table_name.to_owned(), + }, + ) + .await + .expect("delete_table failed"); + eprintln!("[crud] delete_table OK"); +} + +// ============================================================================ +// Test 2: Query with begins_with sort condition +// ============================================================================ +// +// - create table "it_query"; put 3 items with same pk "u#1", sk "p#01","p#02","x#03" +// - query pk = :pk AND begins_with(sk, :pre) where :pre = "p#" +// - assert exactly 2 items returned + +#[tokio::test] +async fn test_query_begins_with() { + let Some(ep) = endpoint() else { + eprintln!("skipping test_query_begins_with: DDB_LOCAL_ENDPOINT unset"); + return; + }; + + let cfg = make_config(&ep); + let engine = DynamoEngine::from_config(&cfg).await; + let account_id = "000000000001"; + let table_name = "it_query"; + + try_delete_table(&engine, account_id, table_name).await; + + engine + .create_table(account_id, create_pk_sk_table(table_name)) + .await + .expect("create_table failed"); + eprintln!("[query] table created"); + + let key_info = engine + .table_key_info(account_id, table_name) + .await + .expect("table_key_info failed"); + + // Put 3 items + for (sk_val, extra) in [("p#01", "alpha"), ("p#02", "beta"), ("x#03", "gamma")] { + let full_item = item(&[("pk", s("u#1")), ("sk", s(sk_val)), ("extra", s(extra))]); + engine + .put_item(&key_info, full_item, false, None, &empty_maps(), None) + .await + .expect("put_item failed"); + } + eprintln!("[query] put 3 items OK"); + + // Build key condition: pk = :pk AND begins_with(sk, :pre) + let kc_str = "pk = :pk AND begins_with(sk, :pre)"; + let kc_tokens = tokenize(kc_str).expect("tokenize key condition"); + let kc = parse_key_condition(&kc_tokens).expect("parse_key_condition"); + + let mut values: HashMap = HashMap::new(); + values.insert("pk".to_owned(), s("u#1")); + values.insert("pre".to_owned(), s("p#")); + let kc_maps = ExpressionMaps::new(HashMap::new(), values); + + let (items, lek) = engine + .query(&key_info, &kc, &kc_maps, true, None, None, None) + .await + .expect("query failed"); + + eprintln!( + "[query] query returned {} items, lek={:?}", + items.len(), + lek + ); + assert_eq!( + items.len(), + 2, + "expected exactly 2 items from begins_with(sk, 'p#')" + ); + + // Verify sorted ascending + let sk0 = items[0].get("sk").expect("sk in first item"); + let sk1 = items[1].get("sk").expect("sk in second item"); + assert_eq!(sk0, &s("p#01"), "first item sk should be p#01"); + assert_eq!(sk1, &s("p#02"), "second item sk should be p#02"); + eprintln!("[query] order OK: {:?} < {:?}", sk0, sk1); + + engine + .delete_table( + account_id, + DeleteTableInput { + table_name: table_name.to_owned(), + }, + ) + .await + .expect("delete_table failed"); + eprintln!("[query] delete_table OK"); +} + +// ============================================================================ +// Test 3: Conditional put fails → ConditionFailed +// ============================================================================ +// +// - table "it_cond"; put item; then put SAME key with attribute_not_exists(pk) +// → assert the result is Err(StorageError::ConditionFailed(_)) + +#[tokio::test] +async fn test_conditional_put_fails() { + let Some(ep) = endpoint() else { + eprintln!("skipping test_conditional_put_fails: DDB_LOCAL_ENDPOINT unset"); + return; + }; + + let cfg = make_config(&ep); + let engine = DynamoEngine::from_config(&cfg).await; + let account_id = "000000000001"; + let table_name = "it_cond"; + + try_delete_table(&engine, account_id, table_name).await; + + engine + .create_table(account_id, create_pk_sk_table(table_name)) + .await + .expect("create_table failed"); + eprintln!("[cond] table created"); + + let key_info = engine + .table_key_info(account_id, table_name) + .await + .expect("table_key_info failed"); + + let first_item = item(&[("pk", s("c#1")), ("sk", s("s#1")), ("val", s("first"))]); + + // First put (no condition) — should succeed + engine + .put_item( + &key_info, + first_item.clone(), + false, + None, + &empty_maps(), + None, + ) + .await + .expect("first put_item failed"); + eprintln!("[cond] first put OK"); + + // Second put with attribute_not_exists(pk) condition — should fail + let cond_tokens = tokenize("attribute_not_exists(pk)").expect("tokenize condition"); + let cond_expr = parse_condition(&cond_tokens).expect("parse_condition"); + + let second_item = item(&[("pk", s("c#1")), ("sk", s("s#1")), ("val", s("second"))]); + let result = engine + .put_item( + &key_info, + second_item, + false, + Some(&cond_expr), + &empty_maps(), + None, + ) + .await; + + eprintln!("[cond] conditional put result: {:?}", result); + + match result { + Err(extenddb_storage::error::StorageError::ConditionFailed(_)) => { + eprintln!("[cond] got expected ConditionFailed"); + } + Err(other) => panic!("expected ConditionFailed, got different error: {other:?}"), + Ok(_) => panic!("expected ConditionFailed, but put_item succeeded"), + } + + // Verify original item is still there unchanged + let key = item(&[("pk", s("c#1")), ("sk", s("s#1"))]); + let got = engine + .get_item(&key_info, &key) + .await + .expect("get_item failed") + .expect("item should still exist"); + assert_eq!( + got.get("val"), + Some(&s("first")), + "original value should remain" + ); + eprintln!("[cond] original item intact, val={:?}", got.get("val")); + + engine + .delete_table( + account_id, + DeleteTableInput { + table_name: table_name.to_owned(), + }, + ) + .await + .expect("delete_table failed"); + eprintln!("[cond] delete_table OK"); +} diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index 441f61e8..f9bf0849 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -86,7 +86,9 @@ impl std::error::Error for BackendError {} /// that resolves to `ServerComponents` or `BackendError`. pub type ServerComponentsFactory = fn( - &dyn StorageConfig, + // `+ 'static` so factories can downcast via `StorageConfig::as_any`; + // configs are always owned `Box` data. + &(dyn StorageConfig + 'static), &str, ) -> Pin> + Send>>; @@ -109,7 +111,7 @@ inventory::collect!(ServerComponentsRegistration); /// Returns `UnknownBackend` error if the backend is not registered. pub async fn create_server_components( backend: &str, - config: &dyn StorageConfig, + config: &(dyn StorageConfig + 'static), region: &str, ) -> Result { for reg in inventory::iter:: { diff --git a/docs/design/13-storage-backend-dynamodb.md b/docs/design/13-storage-backend-dynamodb.md new file mode 100644 index 00000000..8670466a --- /dev/null +++ b/docs/design/13-storage-backend-dynamodb.md @@ -0,0 +1,220 @@ +# DynamoDB Storage Backend — Design + +**Date:** 2026-06-17 +**Status:** Approved (design); pending implementation plan +**Backend name:** `dynamodb` · **Crate:** `extenddb-storage-dynamodb` · **Cargo feature:** `dynamodb` + +## Premise + +ExtendDB speaks the DynamoDB wire protocol. This backend stores its data in +*actual DynamoDB*. The point is not the encoding — there is barely any — it is +the deployment posture: run ExtendDB yourself, pointed at DynamoDB, and you are +technically "self-hosted / on-prem," so the execs stop asking you to get off the +cloud. "We have DynamoDB at home." + +It is the third entry in the satirical-but-functional backend series (after the +Route 53 and S3 Object Annotations backends), but it diverges from them in two +ways the others could not: + +1. **It actually works.** The data plane forwards to real DynamoDB rather than + returning a porting-map stub. +2. **The encoding is the anti-joke.** The other backends cram items into a + primitive never meant to hold them. DynamoDB is already a key/value database, + so the encoding is the *identity function* — and the comedy lives in deadpan + documentation of how little there is to do. + +## Approach: Hybrid composition (data → DynamoDB, catalog → Postgres) + +A fully-functional ExtendDB backend must satisfy a large contract: a data plane +(`TableEngine`, `DataEngine`, `MetadataEngine`, `StreamEngine`, `BackupEngine`, +`WorkerStore`) **and** a catalog/management plane (`ManagementStore`, +`AuthorizationStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, +`RateLimitStore`, `DiagnosticsStore`), plus six `inventory` registrations. + +Most of the catalog plane (IAM accounts, users, roles, policies, settings, +metrics, rate-limits, admin users) has **no natural home in DynamoDB** — +DynamoDB has no such concepts. Rather than reimplement ExtendDB's entire +relational-shaped catalog on top of DynamoDB's query model (the rejected +"Approach A"), this backend is a **hybrid composer**: + +- **Data plane → real DynamoDB**, via `aws-sdk-dynamodb`. +- **Catalog plane → the existing Postgres `CatalogStore`, reused wholesale.** + +Deadpan framing: *your data is on-prem; the bureaucracy that proves you're +on-prem still needs a real database.* This is a more honest punchline than +pretending DynamoDB can be a relational catalog, and it sidesteps the part of a +pure-DynamoDB approach that is all toil and no comedy. + +### Approaches considered + +- **A — Pure DynamoDB (data + catalog both on DynamoDB).** Zero non-DynamoDB + dependency; purest on-prem claim. Rejected: reimplements ExtendDB's whole + catalog/IAM layer against DynamoDB's no-joins, GSI-limited model — months of + engineering, and the punchline gets buried. +- **B — Hybrid (chosen).** Data forwards to DynamoDB; catalog delegates to + Postgres. Working `serve --backend dynamodb`, tractable scope, best joke-to- + effort ratio. +- **C — Data-plane MVP, catalog stays stubbed.** Smallest, but `serve` does not + fully come up, so the "I'm really running it on-prem" gag is weaker. + +## Crate layout + +New crate `crates/storage-dynamodb`, depending on `extenddb-storage`, +`extenddb-storage-postgres` (for the reused catalog/auth), `aws-sdk-dynamodb`, +and `aws-config` (the workspace's first AWS SDK dependencies). + +``` +src/ + lib.rs // DynamoEngine struct, inventory registrations, ServerComponents factory + config.rs // DynamoStorageConfig (region, endpoint_url, table_prefix, catalog_connection_string) + encoding.rs // REAL, round-trip-tested Item <-> AttributeValue marshalling (near-identity) + errors.rs // aws-sdk SdkError / operation errors -> StorageError (wire-protocol fidelity) + bootstrapper.rs // CreateTable-based provisioning; delegates catalog bootstrap to postgres + operations.rs // OperationsEngine (delegates catalog_version to postgres) + table_engine.rs // CreateTable/DeleteTable/DescribeTable/ListTables/UpdateTable with account-id namespacing + data_engine.rs // PutItem/GetItem/UpdateItem/DeleteItem/Query/Scan/Transact* + metadata_engine.rs// native TTL, tags, table size + worker_store.rs // control-plane state via DescribeTable polling + catalog_delegate.rs // constructs the postgres CatalogStore + auth for ServerComponents +``` + +## Component design + +### ServerComponents (the composition point) + +`ServerComponentsRegistration` for `"dynamodb"` returns: + +- `engine: Arc` — data/table/metadata/worker traits forward to DynamoDB. +- `catalog_store: Arc` — constructed from the existing Postgres + implementation using `catalog_connection_string`. +- `auth_provider` — the Postgres-backed provider, reused. +- `runtime_hooks` — minimal: a control-plane poller. **No TTL worker** (DynamoDB + performs TTL deletion itself); **no GSI queue** (DynamoDB owns GSI lifecycle). + +### Encoding module — the real, tested piece (near-identity) + +`encoding.rs` maps ExtendDB's internal `Item`/key representation ↔ +`aws-sdk-dynamodb::types::AttributeValue`. Because ExtendDB already speaks +DynamoDB's type system, this is structurally the identity function; the file is +non-empty only because ExtendDB's in-memory Rust type and the SDK's +`AttributeValue` enum are distinct Rust types holding the same data. It is +round-trip tested across every type (S/N/B/M/L/SS/NS/BS/NULL/BOOL). + +It also houses: + +- key extraction (partition/sort key) and the + `exclusive_start_key ↔ ExclusiveStartKey` / `LastEvaluatedKey` conversions; +- the **account-id → physical table name** namespacer. ExtendDB is multi-tenant; + DynamoDB tables are flat per AWS account. Physical names are + `_
`, default prefix `athome_`. + +### Data-plane mapping (v1 functional surface) + +| ExtendDB trait method | DynamoDB call | Notes | +|---|---|---| +| `put_item` (+condition) | `PutItem` + `ConditionExpression` | translate parsed condition AST → expression string | +| `get_item` | `GetItem` | direct | +| `delete_item` (+condition) | `DeleteItem` | direct | +| `update_item` (actions, condition) | `UpdateItem` | translate update AST → `UpdateExpression` | +| `query` | `Query` | key-condition AST → `KeyConditionExpression`; pagination tokens map 1:1 | +| `scan` (segment/total) | `Scan` | `Segment`/`TotalSegments` direct | +| `transact_write_items(ops, token)` | `TransactWriteItems` | `token` → `ClientRequestToken` (idempotency maps perfectly) | +| `transact_get_items` | `TransactGetItems` | direct | +| `cleanup_expired_idempotency_tokens` | no-op | DynamoDB manages its own idempotency window | +| `create/delete/describe/list/update_table` | `CreateTable` / `DeleteTable` / `DescribeTable` / `ListTables` / `UpdateTable` | account-namespaced physical names | +| `describe_ttl` / `update_ttl` | `DescribeTimeToLive` / `UpdateTimeToLive` | native; ExtendDB TTL worker becomes a no-op | +| tags | `TagResource` / `UntagResource` / `ListTagsOfResource` | direct | +| table size | `DescribeTable` | `TableSizeBytes` / `ItemCount` | +| `process_control_plane_transitions` | `DescribeTable` poll | report CREATING→ACTIVE from AWS's real state | + +### Honestly stubbed in v1 (named, not silent) + +`StreamEngine` and `BackupEngine` are honest stubs in v1 — every method errors +naming the DynamoDB API it maps to (`DescribeStream`/`GetRecords`/ +`GetShardIterator`; `CreateBackup`/`RestoreTableFromBackup`/ +`UpdateContinuousBackups`/`DescribeContinuousBackups`). Reason: ExtendDB's stream +model assumes ExtendDB synthesizes records on write, but a passthrough must +instead read DynamoDB's own stream — a real architectural reconciliation not +worth rushing. Both map cleanly and are flagged as fast follow-ups. The gap is +documented in `docs/differences-from-dynamodb.md`. + +### Error handling + +`errors.rs` maps `SdkError` and DynamoDB operation errors → `StorageError`, +preserving wire-protocol fidelity so SDK clients hitting ExtendDB see the errors +they would expect from DynamoDB: + +- `ConditionalCheckFailedException` → ExtendDB's condition-failed error +- `ResourceNotFoundException`, `ResourceInUseException` +- `ProvisionedThroughputExceededException` +- `TransactionCanceledException`, `TransactionConflictException` +- `ItemCollectionSizeLimitExceededException`, `RequestLimitExceeded`, throttling + +### Configuration + +```toml +[storage] +backend = "dynamodb" + +[storage.dynamodb] +region = "us-east-1" +endpoint_url = "https://dynamodb.us-east-1.amazonaws.com" # may point at ANOTHER ExtendDB +table_prefix = "athome_" +catalog_connection_string = "postgresql://user:pass@localhost/extenddb_catalog" +# AWS credentials resolve via the standard provider chain unless overridden here. +``` + +`DynamoStorageConfig` implements the `StorageConfig` trait and is registered via +`StorageConfigRegistration` for parsing the `[storage.dynamodb]` section. + +### Registration & build wiring + +Six `inventory::submit!` registrations under name `"dynamodb"`: +`BackendRegistration`, `OperationsEngineRegistration`, +`StorageConfigRegistration`, `SettingsStoreRegistration`, +`DiagnosticsStoreRegistration`, `ServerComponentsRegistration`. The latter three +delegate to the Postgres catalog. Build wiring: + +- `crates/bin/Cargo.toml`: add `dynamodb = ["extenddb-storage-dynamodb"]` feature. +- `crates/bin/src/main.rs`: add `#[cfg(feature = "dynamodb")] extern crate extenddb_storage_dynamodb;` to force linker inclusion of the inventory submissions. +- `crates/bin/src/cmd_serve.rs`: extend the feature-validation arm to accept `dynamodb`. + +### Catalog version + +`catalog_version()` / `OperationsEngine::catalog_version` delegate to the Postgres +implementation, since the catalog schema lives in Postgres. No separate DynamoDB +catalog version exists. + +## Recursion is a feature + +`endpoint_url` may point at another ExtendDB endpoint. Documented as a legitimate +use case in deadpan voice ("compliance calls this defense in depth; we call it +on-prem"). **No loop guard** — the near-identity encoding is exactly what makes +the stack composable, and it also enables elegant integration testing +(ExtendDB-on-ExtendDB, or ExtendDB-on-DynamoDB-Local). + +## Testing + +- **Unit:** `encoding` round-trip across every `AttributeValue` type; the + error-mapping table with mocked SDK responses; condition/key/update AST → + expression-string translation. +- **Integration:** point the existing external DynamoDB test suite at + `serve --backend dynamodb` backed by **DynamoDB Local**, with the catalog on a + throwaway Postgres. Use the recursion property for an ExtendDB-on-ExtendDB + smoke test. + +## Open questions (resolve during planning; not blockers) + +1. Whether ExtendDB's parsed condition/update ASTs can be losslessly + re-serialized to DynamoDB expression strings, or whether the data-plane traits + need access to the original wire expressions. Affects how much translation + `data_engine.rs` performs. +2. Which Postgres catalog/auth constructors are sufficiently `pub` to reuse + directly, versus needing a small public factory added to + `extenddb-storage-postgres`. + +## Out of scope (v1) + +- Streams and Backups/PITR functional implementations (honest stubs in v1). +- Import/Export. +- A pure-DynamoDB catalog (Approach A). diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 045484a6..267d116b 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -8,11 +8,38 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| -| Storage backend | Proprietary distributed storage | PostgreSQL | +| Storage backend | Proprietary distributed storage | PostgreSQL (default), or the optional `dynamodb` backend (see below) | | Global Tables | CreateGlobalTable, replication | Not implemented (returns UnknownOperationException) | | DAX (Accelerator) | In-memory caching layer | Not applicable | | PartiQL | ExecuteStatement, BatchExecuteStatement | Not implemented (returns UnknownOperationException) | +## The `dynamodb` Storage Backend ("DynamoDB at home") + +ExtendDB ships an optional `dynamodb` storage backend (build with +`--features dynamodb`, select with `[storage] backend = "dynamodb"`). It forwards +the **data plane** to a real DynamoDB endpoint via the AWS SDK, while the +**catalog/IAM/auth plane** is delegated to the PostgreSQL backend. Run it yourself, +point it at DynamoDB, and you are technically self-hosted — the entire point. + +| Area | Behavior with `backend = "dynamodb"` | +|------|--------------------------------------| +| Data plane (PutItem, GetItem, UpdateItem, DeleteItem, Query, Scan, transactions) | Forwarded to real DynamoDB | +| Catalog / IAM / settings / metrics / rate-limits | Stored in PostgreSQL (`catalog_connection_string`) | +| Table namespacing | ExtendDB is multi-tenant; physical DynamoDB tables are named `_
` (default prefix `athome_`) | +| TTL | Native DynamoDB TTL (`UpdateTimeToLive`); ExtendDB's TTL worker is a no-op | +| Tags | Forwarded to DynamoDB `TagResource`/`UntagResource`/`ListTagsOfResource` (table ARNs only) | +| Idempotency tokens | Forwarded as `ClientRequestToken`; DynamoDB manages its own ~10-minute window | +| Streams | **Not implemented in v1** — each method errors naming the DynamoDB Streams call it maps to | +| Backups / PITR | **Not implemented in v1** — each method errors naming the DynamoDB Backup call it maps to | +| `UpdateItem` returning both old and new | DynamoDB returns only one per call; the backend prefers `ALL_NEW` when both are requested | +| `update_table` GSI mutations | Not forwarded in v1 (table create/describe/delete and billing changes are) | +| `index_info_by_table_id` | Not supported (DynamoDB has no TableId→name reverse lookup) | +| `endpoint_url` | May point at DynamoDB Local — or at **another ExtendDB endpoint**, which is documented as a feature, not a bug (the near-identity encoding makes the stack composable) | + +Note that this backend still requires PostgreSQL for the catalog: your data is +on-prem, but the bureaucracy that proves you're on-prem still needs a real +database. + ## Authentication and Authorization (AWS IAM/STS auth surface used by DynamoDB) | Area | DynamoDB | ExtendDB | diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 00e251e0..3c177dd3 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -52,6 +52,13 @@ # DynamoDB request makes concurrent authz queries # — size this to match expected concurrency. +# [storage.dynamodb] +# Data plane stores into real DynamoDB; the catalog/IAM plane lives in Postgres. +# region = "us-east-1" +# endpoint_url = "http://localhost:8000" # DynamoDB Local, or another ExtendDB endpoint (recursion is a feature) +# table_prefix = "athome_" +# catalog_connection_string = "postgresql://extenddb:extenddb-local-dev@localhost:5432/extenddb_catalog" + [auth] # provider = "builtin" # Auth provider: # "builtin" — SigV4 verification with local credential