diff --git a/Cargo.lock b/Cargo.lock index 2009c45..3205e40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,55 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" +[[package]] +name = "astral-tokio-tar" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" +dependencies = [ + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "rustix", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "atoi" version = "2.0.0" @@ -212,7 +261,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn", + "syn 2.0.117", ] [[package]] @@ -242,6 +291,89 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bollard" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" +dependencies = [ + "async-stream", + "base64", + "bitflags", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.9.4", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.52.1-rc.29.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" +dependencies = [ + "base64", + "bollard-buildkit-proto", + "bytes", + "prost", + "serde", + "serde_json", + "serde_repr", + "time", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -370,7 +502,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -554,7 +686,7 @@ dependencies = [ [[package]] name = "cuscuta-chilo" -version = "0.1.8" +version = "0.1.9" dependencies = [ "axum", "base64", @@ -571,10 +703,11 @@ dependencies = [ [[package]] name = "cuscuta-common" -version = "0.1.8" +version = "0.1.9" dependencies = [ "base64", "chrono", + "cuscuta-test", "hex", "redis", "reqwest", @@ -589,7 +722,7 @@ dependencies = [ [[package]] name = "cuscuta-entry" -version = "0.1.8" +version = "0.1.9" dependencies = [ "anyhow", "axum", @@ -629,20 +762,32 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "cuscuta-test" +version = "0.1.9" +dependencies = [ + "cuscuta-common", + "redis", + "testcontainers", + "tokio", +] + [[package]] name = "cuscuta-worker" -version = "0.1.8" +version = "0.1.9" dependencies = [ "anyhow", "axum", "chrono", "cuscuta-common", + "cuscuta-test", "gethostname", "rand 0.10.1", "redis", "reqwest", "serde_json", "sqlx", + "testcontainers", "thiserror", "tokio", "tokio-util", @@ -670,6 +815,40 @@ dependencies = [ "url", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "der" version = "0.7.10" @@ -681,6 +860,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + [[package]] name = "digest" version = "0.10.7" @@ -712,7 +900,18 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64", + "serde", + "serde_json", ] [[package]] @@ -736,6 +935,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.16.0" @@ -781,6 +986,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -792,6 +1007,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "portable-atomic", + "rand 0.10.1", + "web-time", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -836,6 +1062,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -880,6 +1121,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -898,8 +1150,10 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -986,13 +1240,19 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" @@ -1134,6 +1394,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.27.9" @@ -1149,6 +1424,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -1174,6 +1462,21 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1286,6 +1589,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1307,6 +1616,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1382,7 +1702,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -1401,7 +1721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1597,6 +1917,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1623,6 +1957,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -1643,6 +1992,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1700,6 +2060,31 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.117", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1715,6 +2100,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1754,6 +2159,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1763,6 +2174,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1779,7 +2196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -1791,6 +2208,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1980,6 +2429,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "regex" version = "1.12.3" @@ -2119,7 +2588,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2217,6 +2688,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2279,7 +2774,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2306,6 +2801,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -2327,6 +2833,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2499,7 +3037,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap", + "indexmap 2.14.0", "log", "memchr", "once_cell", @@ -2525,7 +3063,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.117", ] [[package]] @@ -2548,7 +3086,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.117", "tokio", "url", ] @@ -2609,7 +3147,7 @@ dependencies = [ "chrono", "crc", "dotenvy", - "etcetera", + "etcetera 0.8.0", "futures-channel", "futures-core", "futures-util", @@ -2682,6 +3220,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.117", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2699,6 +3260,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2716,7 +3288,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2740,6 +3312,37 @@ dependencies = [ "libc", ] +[[package]] +name = "testcontainers" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" +dependencies = [ + "astral-tokio-tar", + "async-trait", + "bollard", + "bytes", + "docker_credential", + "either", + "etcetera 0.11.0", + "ferroid", + "futures", + "http", + "itertools 0.14.0", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -2757,7 +3360,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2769,6 +3372,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -2818,7 +3451,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2861,7 +3494,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime", @@ -2894,6 +3527,46 @@ version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -2902,9 +3575,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2977,7 +3653,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3108,6 +3784,33 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -3118,8 +3821,15 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3241,7 +3951,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -3271,7 +3981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -3284,7 +3994,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "semver", ] @@ -3379,7 +4089,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3390,7 +4100,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3690,9 +4400,9 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap", + "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -3708,7 +4418,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -3721,7 +4431,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -3740,7 +4450,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.14.0", "log", "semver", "serde", @@ -3756,6 +4466,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xxhash-rust" version = "0.8.15" @@ -3781,7 +4501,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -3802,7 +4522,7 @@ checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3822,7 +4542,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -3862,7 +4582,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index feac1de..f8c2832 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,40 +1,40 @@ -[workspace] -members = [ - "cuscuta-common", - "cuscuta-entry", - "cuscuta-worker", - "cuscuta-chilo", - "cuscuta-mock", - "cuscutactl", -] -resolver = "2" - -[workspace.package] -version = "0.1.8" - -[workspace.dependencies] -anyhow = "1.0.102" -axum = "0.8.9" -base64 = "0.22.1" -chrono = { version = "0.4.44", features = ["serde"] } -comfy-table = "7.1" -hex = "0.4.3" -rand = "0.10.1" -redis = { version = "1.2.0", features = ["streams"] } -reqwest = { version = "0.13.2", features = ["json", "query", "form"] } -serde = { version = "1.0.228", features = ["derive"] } -serde_json = "1.0.150" -sha2 = "0.11.0" -sqlx = { version = "0.8.6", features = ["postgres", "chrono", "runtime-tokio"] } -thiserror = "2.0.18" -tokio = { version = "1.52.3", features = ["rt-multi-thread"] } -tokio-util = "0.7.18" -tracing = "0.1.44" -tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } - -[workspace.lints.rust] -missing_docs = "deny" - -[workspace.lints.clippy] -pedantic = "deny" -nursery = "deny" +[workspace] +members = [ + "cuscuta-common", + "cuscuta-entry", + "cuscuta-worker", + "cuscuta-chilo", + "cuscuta-mock", + "cuscutactl", "cuscuta-test", +] +resolver = "2" + +[workspace.package] +version = "0.1.9" + +[workspace.dependencies] +anyhow = "1.0.102" +axum = "0.8.9" +base64 = "0.22.1" +chrono = { version = "0.4.44", features = ["serde"] } +comfy-table = "7.1" +hex = "0.4.3" +rand = "0.10.1" +redis = { version = "1.2.0", features = ["streams"] } +reqwest = { version = "0.13.2", features = ["json", "query", "form"] } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" +sha2 = "0.11.0" +sqlx = { version = "0.8.6", features = ["postgres", "chrono", "runtime-tokio"] } +thiserror = "2.0.18" +tokio = { version = "1.52.3", features = ["rt-multi-thread"] } +tokio-util = "0.7.18" +tracing = "0.1.44" +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } + +[workspace.lints.rust] +missing_docs = "deny" + +[workspace.lints.clippy] +pedantic = "deny" +nursery = "deny" diff --git a/Deployment.en.md b/Deployment.en.md index 3ade9d7..6e5eedd 100644 --- a/Deployment.en.md +++ b/Deployment.en.md @@ -174,6 +174,7 @@ All cuscuta components receive configuration through environment variables. | `WORKER_ACCOUNT_LEASE_TIME_SECS` | 120 | Account lease duration (s) | | `WORKER_ACCOUNT_LEASE_TIME_REFRESH_GAP_SECS` | 30 | Lease refresh interval (s) | | `WORKER_JOB_MAX_WORK_TIME_SECS` | 1200 | Maximum working time for a job (s) | +| `WORKER_EMPTY_FRIENDS_DELAY_TIME_SECS` | 10 | Rate limiting delay time (s) | | `ETA_ENABLE` | true | Enable ETA estimation | | `API_CHILO` | — | chilo service URL | | `API_LOGIN` | — | Login API endpoint | diff --git a/Deployment.md b/Deployment.md index 0a35743..48bd3e8 100644 --- a/Deployment.md +++ b/Deployment.md @@ -169,6 +169,7 @@ cuscuta 所有组件均通过环境变量注入配置。 | `WORKER_ACCOUNT_LEASE_TIME_SECS` | 120 | 账号租约时长（秒） | | `WORKER_ACCOUNT_LEASE_TIME_REFRESH_GAP_SECS` | 30 | 租约续期间隔（秒） | | `WORKER_JOB_MAX_WORK_TIME_SECS` | 1200 | 任务的最长运行时间（秒） | +| `WORKER_EMPTY_FRIENDS_DELAY_TIME_SECS` | 10 | 风控应对延迟（秒） | | `ETA_ENABLE` | true | 是否启用 ETA 预估 | | `API_CHILO` | — | chilo 服务地址 | | `API_LOGIN` | — | 登录 API 地址 | diff --git a/TODO.md b/TODO.md index 166f3fc..d62c0bd 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,11 @@ ## 计划表 +### 杂项 +- [x] tracing重构 +- [ ] 日志重构 + - [ ] 日志细化 + - [ ] 日志JSON开关 + ### cuscuta-worker - [x] 基础查分服务 diff --git a/cuscuta-common/Cargo.toml b/cuscuta-common/Cargo.toml index 4cb4401..88bde49 100644 --- a/cuscuta-common/Cargo.toml +++ b/cuscuta-common/Cargo.toml @@ -19,5 +19,8 @@ thiserror = { workspace = true } tokio = { workspace = true, features = ["signal", "macros"] } tokio-util = { workspace = true } +[dev-dependencies] +cuscuta-test = { path = "../cuscuta-test" } + [lints] workspace = true diff --git a/cuscuta-common/src/api/chilo.rs b/cuscuta-common/src/api/chilo.rs index 984a7d9..dd852da 100644 --- a/cuscuta-common/src/api/chilo.rs +++ b/cuscuta-common/src/api/chilo.rs @@ -40,5 +40,7 @@ pub async fn chilo_generate(timestamp: &str, path: &str, kind: &str) -> Result() .await - .map_err(|e| Error::Decode(e.to_string())) + .map_err(|e| Error::Decode { + message: e.to_string(), + }) } diff --git a/cuscuta-common/src/api/github.rs b/cuscuta-common/src/api/github.rs index 0d6ec8f..8824c63 100644 --- a/cuscuta-common/src/api/github.rs +++ b/cuscuta-common/src/api/github.rs @@ -1,70 +1,90 @@ -use base64::Engine; -use reqwest::StatusCode; -use serde::{Deserialize, de::DeserializeOwned}; - -use crate::api::{Error, ErrorForStatusWithResponse}; - -#[derive(Debug, Deserialize)] -struct GitHubFileInternal { - content: String, - download_url: String, -} - -type Result = core::result::Result; - -/// 调用Github Api，从Github上拉取资源 -/// -/// # Errors -/// - 当环境变量配置无效时，返回[`Error::Env`] -/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] -/// - 当请求发送失败时，返回[`Error::Network`] -/// - 当返回值不为2xx时，返回[`Error::BadStatus`] -/// - 当Json反序列化失败时，返回[`Error::Decode`] -pub async fn fetch_github_resource(repo: &str, path: &str, token: &str) -> Result -where - T: DeserializeOwned, -{ - let file_object = reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .get(format!( - "https://api.github.com/repos/{repo}/contents/{path}" - )) - .header("Accept", "application/vnd.github.object") - .header("Authorization", format!("Bearer {token}")) - .header("X-GitHub-Api-Version", "2026-03-10") - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(format!("phase1: {e}")))?; - if file_object.content.is_empty() { - reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .get(file_object.download_url) - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(format!("phase b_1: {e}"))) - } else { - base64::prelude::BASE64_STANDARD - .decode(file_object.content.replace('\n', "")) - .map_err(|e| Error::Decode(format!("phase a_2: {e}"))) - .map(|it| String::from_utf8(it).map_err(|e| Error::Decode(format!("phase a_3: {e}"))))? - .map(|it| { - serde_json::from_str::(&it).map_err(|e| Error::Decode(format!("phase a_4: {e}"))) - })? - } -} +use base64::Engine; +use reqwest::StatusCode; +use serde::{Deserialize, de::DeserializeOwned}; + +use crate::api::{Error, ErrorForStatusWithResponse}; + +#[derive(Debug, Deserialize)] +struct GitHubFileInternal { + content: String, + download_url: String, +} + +type Result = core::result::Result; + +/// 调用Github Api，从Github上拉取资源 +/// +/// # Errors +/// - 当环境变量配置无效时，返回[`Error::Env`] +/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] +/// - 当请求发送失败时，返回[`Error::Network`] +/// - 当返回值不为2xx时，返回[`Error::BadStatus`] +/// - 当Json反序列化失败时，返回[`Error::Decode`] +pub async fn fetch_github_resource(repo: &str, path: &str, token: &str) -> Result +where + T: DeserializeOwned, +{ + let file_object = reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .get(format!( + "https://api.github.com/repos/{repo}/contents/{path}" + )) + .header("Accept", "application/vnd.github.object") + .header("Authorization", format!("Bearer {token}")) + .header("X-GitHub-Api-Version", "2026-03-10") + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await + .map_err(|(s, e)| Error::BadStatus { + status_code: e.status().unwrap_or_else(StatusCode::default), + message: s, + extra_error_code: None, + })? + .json::() + .await + .map_err(|e| Error::Decode { + message: format!("phase1: {e}"), + })?; + if file_object.content.is_empty() { + reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .get(file_object.download_url) + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await + .map_err(|(s, e)| Error::BadStatus { + status_code: e.status().unwrap_or_else(StatusCode::default), + message: s, + extra_error_code: None, + })? + .json::() + .await + .map_err(|e| Error::Decode { + message: format!("phase b_1: {e}"), + }) + } else { + base64::prelude::BASE64_STANDARD + .decode(file_object.content.replace('\n', "")) + .map_err(|e| Error::Decode { + message: format!("phase a_2: {e}"), + }) + .map(|it| { + String::from_utf8(it).map_err(|e| Error::Decode { + message: format!("phase a_3: {e}"), + }) + })? + .map(|it| { + serde_json::from_str::(&it).map_err(|e| Error::Decode { + message: format!("phase a_4: {e}"), + }) + })? + } +} diff --git a/cuscuta-common/src/api/mod.rs b/cuscuta-common/src/api/mod.rs index 1568e34..46c926b 100644 --- a/cuscuta-common/src/api/mod.rs +++ b/cuscuta-common/src/api/mod.rs @@ -12,6 +12,9 @@ pub mod xxxxxx; pub mod chilo; /// Api调用可能引发的错误 +/// +/// 注：这里的错误处理可能很脏，因为这个错误类型包含了过于特化的[`Self::TooManyRetries`] +/// 以及并非所有Api系函数均支持的[`Self::ApiError`] #[derive(Debug, thiserror::Error)] pub enum Error { /// `reqwest`客户端初始化失败 @@ -23,24 +26,41 @@ pub enum Error { Network(reqwest::Error), /// Api的`HTTP`返回码不为2xx或1xx - #[error("bad return status: {0}: {1}")] - BadStatus(StatusCode, String), + #[error("bad return status: HTTP {status_code} {extra_error_code:?}: {message}")] + BadStatus { + /// 错误码 + status_code: StatusCode, + + /// 错误描述 + message: String, + + /// Api错误码（如果有） + extra_error_code: Option, + }, /// Json反序列化失败 - #[error("failed to decode response: {0}")] - Decode(String), + #[error("failed to decode response: {message}")] + Decode { + /// 错误描述 + message: String, + }, /// 环境变量未配置或配置无效 - #[error("failed to read env::var: {0}:{1}")] - Env(env::VarError, String), + #[error("failed to read env::var: {error}:{message}")] + Env { + /// 环境变量错误 + error: env::VarError, - /// 重试次数过多 - #[error("too many retries: inner: {0}")] - TooManyRetries(String), + /// 错误描述 + message: String, + }, } fn try_get_env_var(var: &str) -> Result { - env::var(var).map_err(|e| Error::Env(e, var.to_string())) + env::var(var).map_err(|error| Error::Env { + error, + message: var.to_string(), + }) } trait ErrorForStatusWithResponse diff --git a/cuscuta-common/src/api/xxxxxx.rs b/cuscuta-common/src/api/xxxxxx.rs index 4a2c6cc..f89c496 100644 --- a/cuscuta-common/src/api/xxxxxx.rs +++ b/cuscuta-common/src/api/xxxxxx.rs @@ -1,499 +1,720 @@ -use std::collections::HashSet; -use std::fmt::Write; - -use reqwest::StatusCode; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::{ - api::{Error, ErrorForStatusWithResponse, try_get_env_var}, - data::BundleData, -}; - -type Result = core::result::Result; - -/// xxxxxx api登录成功时的适配数据模型 -#[derive(Debug, Deserialize, Clone)] -pub struct LoginResult { - /// 用户id - pub user_id: i64, - - /// 访问密钥 - pub access_token: String, - - /// 密钥类型，通常为`Bearer` - pub token_type: String, -} - -/// xxxxxx api好友变更时的适配数据模型（顶层） -#[derive(Debug, Deserialize, Clone)] -pub struct FriendListResult { - /// 值 - pub value: FriendListResult1, -} - -/// xxxxxx api好友变更时的适配数据模型（第二层） -#[derive(Debug, Deserialize, Clone)] -pub struct FriendListResult1 { - /// 好友信息 - pub friends: Vec, -} - -/// xxxxxx api好友信息的适配数据模型 -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] -pub struct FriendInfo { - /// 好友游戏名 - pub name: String, - - /// 好友用户id - pub user_id: i64, - - /// 好友的评级（实际显示评级为`rating/10.0`） - pub rating: i64, - - /// 好友设置的搭档 - pub character: i64, - - /// 好友搭档状态1 - pub is_char_uncapped: bool, - - /// 好友搭档状态2 - pub is_char_uncapped_override: bool, -} - -/// xxxxxx api曲目成绩的适配数据模型（顶层） -#[derive(Debug, Serialize, Deserialize)] -pub struct SongScoreResult { - ///曲目信息 - pub value: Vec, -} - -/// xxxxxx api曲目成绩的适配数据模型 -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct SongScore { - /// 曲目id - pub song_id: String, - - /// 玩家的id - pub user_id: i64, - - /// 难度等级（如0、1、2） - pub difficulty: i64, - - /// 游玩分数 - pub score: i64, - - /// 游玩分数距最高分的差距 - pub score_below_max: i64, - - /// 大P数目 - pub shiny_perfect_count: i64, - - /// 小P数目 - pub perfect_count: i64, - - /// Far数目 - pub near_count: i64, - - /// Lost数目 - pub miss_count: i64, - - /// 通关类型 - pub clear_type: i64, - - /// 最高纪录的通关类型 - pub best_clear_type: i64, - - /// 通关时搭档血量 - pub health: i64, - - /// 游玩时间戳 - pub time_played: i64, - - /// 玩家名字 - #[serde(rename = "name")] - pub player_name: String, -} - -/// 通过xxxxxx api登录 -/// -/// # Errors -/// - 当环境变量配置无效时，返回[`Error::Env`] -/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] -/// - 当请求发送失败时，返回[`Error::Network`] -/// - 当返回值不为2xx时，返回[`Error::BadStatus`] -/// - 当Json反序列化失败时，返回[`Error::Decode`] -pub async fn api_login( - bundle_data: &BundleData, - email: &str, - password: &str, - random_challenge: &str, -) -> Result { - reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .post(try_get_env_var("API_LOGIN")?) - .header("X-Random-Challenge", random_challenge) - .header("AppVersion", bundle_data.application_version_number.clone()) - .header("ContentBundle", bundle_data.version_number.clone()) - .header("DeviceId", generate_device_id(email)) - .basic_auth(email, Some(password)) - .form(&[("grant_type", "client_credentials")]) - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(e.to_string())) -} - -/// 通过xxxxxx api查询好友 -/// -/// # Errors -/// - 当环境变量配置无效时，返回[`Error::Env`] -/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] -/// - 当请求发送失败时，返回[`Error::Network`] -/// - 当返回值不为2xx时，返回[`Error::BadStatus`] -/// - 当Json反序列化失败时，返回[`Error::Decode`] -pub async fn api_list_friend( - bundle_data: &BundleData, - email: &str, - user_id: &str, - token: &str, -) -> Result { - reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .get(try_get_env_var("API_LIST_FRIENDS")?) - .header("X-Random-Challenge", generate_random_challenge()) - .header("Platform", "android") - .header("AppVersion", bundle_data.application_version_number.clone()) - .header("ContentBundle", bundle_data.version_number.clone()) - .header("DeviceId", generate_device_id(email)) - .header("i", user_id) - .bearer_auth(token) - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(e.to_string())) - .map(|it| it.value) -} - -/// 通过xxxxxx api添加好友 -/// -/// # Errors -/// - 当环境变量配置无效时，返回[`Error::Env`] -/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] -/// - 当请求发送失败时，返回[`Error::Network`] -/// - 当返回值不为2xx时，返回[`Error::BadStatus`] -/// - 当Json反序列化失败时，返回[`Error::Decode`] -pub async fn api_add_friend( - bundle_data: &BundleData, - email: &str, - user_id: &str, - token: &str, - friend_code: &str, -) -> Result { - reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .post(try_get_env_var("API_ADD_FRIENDS")?) - .header("X-Random-Challenge", generate_random_challenge()) - .header("Platform", "android") - .header("AppVersion", bundle_data.application_version_number.clone()) - .header("ContentBundle", bundle_data.version_number.clone()) - .header("DeviceId", generate_device_id(email)) - .header("i", user_id) - .bearer_auth(token) - .form(&[("friend_code", friend_code)]) - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(e.to_string())) - .map(|it| it.value) -} - -/// 通过xxxxxx api删除好友 -/// -/// # Errors -/// - 当环境变量配置无效时，返回[`Error::Env`] -/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] -/// - 当请求发送失败时，返回[`Error::Network`] -/// - 当返回值不为2xx时，返回[`Error::BadStatus`] -/// - 当Json反序列化失败时，返回[`Error::Decode`] -pub async fn api_delete_friend( - bundle_data: &BundleData, - email: &str, - user_id: &str, - token: &str, - friend_id: &str, -) -> Result { - reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .post(try_get_env_var("API_DELETE_FRIENDS")?) - .header("X-Random-Challenge", generate_random_challenge()) - .header("Platform", "android") - .header("AppVersion", bundle_data.application_version_number.clone()) - .header("ContentBundle", bundle_data.version_number.clone()) - .header("DeviceId", generate_device_id(email)) - .header("i", user_id) - .bearer_auth(token) - .form(&[("friend_id", friend_id)]) - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(e.to_string())) - .map(|it| it.value) -} - -/// 通过xxxxxx api查询排行榜 -/// -/// # Errors -/// - 当环境变量配置无效时，返回[`Error::Env`] -/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] -/// - 当请求发送失败时，返回[`Error::Network`] -/// - 当返回值不为2xx时，返回[`Error::BadStatus`] -/// - 当Json反序列化失败时，返回[`Error::Decode`] -#[allow(clippy::too_many_arguments)] -pub async fn api_get_rank_list( - bundle_data: &BundleData, - email: &str, - user_id: &str, - token: &str, - song_id: &str, - difficulty: &str, - start: &str, - limit: &str, -) -> Result> { - reqwest::Client::builder() - .user_agent("curl/7.88.1") - .build() - .map_err(Error::ClientSetup)? - .get(try_get_env_var("API_GET_RANK")?) - .query(&[ - ("song_id", song_id), - ("difficulty", difficulty), - ("start", start), - ("limit", limit), - ]) - .header("X-Random-Challenge", generate_random_challenge()) - .header("Platform", "android") - .header("AppVersion", bundle_data.application_version_number.clone()) - .header("ContentBundle", bundle_data.version_number.clone()) - .header("DeviceId", generate_device_id(email)) - .header("i", user_id) - .bearer_auth(token) - .send() - .await - .map_err(Error::Network)? - .error_for_status_with_response() - .await - .map_err(|(s, e)| Error::BadStatus(e.status().unwrap_or_else(StatusCode::default), s))? - .json::() - .await - .map_err(|e| Error::Decode(e.to_string())) - .map(|it| it.value) -} - -fn generate_device_id(email: &str) -> String { - // Warn: 这只是为了保证唯一性 - let mut result = String::with_capacity(16); - for it in Sha256::digest(email).into_iter().take(8) { - let _ = write!(&mut result, "{it:02x}"); - } - result -} - -/// 生成Random Challenge的占位符 -/// -/// 在不没有硬性检查的端点，就全部使用占位符替代 -fn generate_random_challenge() -> String { - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".into() -} - -/// [`calc_friend_delta`]所使用的好友Delta量 -pub enum FriendDelta { - /// 好友相较`before`增加了一个 - Add(FriendInfo), - /// 好友相较`before`减少了一个 - Remove(FriendInfo), - /// 好友相较`before`没有变化 - Same, -} - -/// 计算好友变化，返回好友信息 -/// -/// 当好友仅变化1个或不变化时，返回`Ok` -/// -/// # Errors -/// 当好友变化量不为0或1时，返回`String`的`Err`\ -/// （TODO：计划重构来改善错误处理） -/// -/// # Panics -/// 所有可能的panic的代码来自于`set.iter().next().expect(...)`，而根据条件，这些set的长度在执行它之前均为1，故本函数理论上永远不会panic -pub fn calc_friend_delta( - before: &[FriendInfo], - after: &[FriendInfo], -) -> core::result::Result { - let before: HashSet<_> = before.iter().collect(); - let after: HashSet<_> = after.iter().collect(); - let delta_add: HashSet<_> = after.difference(&before).collect(); - if delta_add.len() == 1 { - return Ok(FriendDelta::Add( - (**delta_add - .iter() - .next() - .expect("first element of delta_add is None when len==1, this should not happen")) - .clone(), - )); - } - if !delta_add.is_empty() { - return Err("bad add delta".into()); - } - let delta_rem: HashSet<_> = before.difference(&after).collect(); - if delta_rem.len() == 1 { - return Ok(FriendDelta::Remove( - (**delta_rem - .iter() - .next() - .expect("first element of delta_rem is None when len==1, this should not happen")) - .clone(), - )); - } - if !delta_rem.is_empty() { - return Err("bad rem delta".into()); - } - Ok(FriendDelta::Same) -} - -/// 这个模块提供了便捷的xxxxxx api调用包装 -pub mod auto { - use std::time::Duration; - - use reqwest::StatusCode; - use tokio::time::{Sleep, sleep}; - - use crate::api; - - /// 一个包装，用于为一定量的[`StatusCode`]错误和网络错误提供弹性 - /// 使用简化的错误判断，任何错误均重试 - /// - /// # Errors - /// 当本函数因重试次数过多而失败时，返回[`api::Error::TooManyRetries`]\ - /// 否则，返回的错误由指定函数所可能引发的错误决定 - #[allow(clippy::cast_possible_truncation)] - pub async fn xxxxxx_safe_call<'a, F, R, Fut>( - max_retries: u64, - exponential_backoff_base_millis: u64, - exponential_backoff_multiplier: u64, - exponential_backoff_max_delay_millis: u64, - f: F, - ) -> Result - where - Fut: Future> + 'a + Send, - R: Send + 'a, - F: Fn() -> Fut, - { - xxxxxx_safe_call_ex( - max_retries, - exponential_backoff_base_millis, - exponential_backoff_multiplier, - exponential_backoff_max_delay_millis, - |_| false, - f, - ) - .await - } - - /// 一个包装，用于为一定量的[`StatusCode`]错误和网络错误提供弹性 - /// - /// # Errors - /// 当本函数因重试次数过多而失败时，返回[`api::Error::TooManyRetries`]\ - /// 否则，返回的错误由指定函数所可能引发的错误决定 - #[allow(clippy::cast_possible_truncation)] - pub async fn xxxxxx_safe_call_ex<'a, F, R, T, Fut>( - max_retries: u64, - exponential_backoff_base_millis: u64, - exponential_backoff_multiplier: u64, - exponential_backoff_max_delay_millis: u64, - fail_cond: T, - f: F, - ) -> Result - where - Fut: Future> + 'a + Send, - R: Send + 'a, - F: Fn() -> Fut, - T: Fn(StatusCode) -> bool, - { - //TODO add re-login - fn wait( - exponential_backoff_base_millis: u64, - exponential_backoff_multiplier: u64, - exponential_backoff_max_delay_millis: u64, - retries: u64, - ) -> Sleep { - sleep(Duration::from_millis( - (exponential_backoff_base_millis - * exponential_backoff_multiplier.pow(retries as u32)) - .min(exponential_backoff_max_delay_millis), - )) - } - let mut retries = 0; - let mut latest_error = None; - while retries <= max_retries { - let result = f().await; - if let Err(err) = &result { - latest_error = Some(format!("{err:?}")); - } - match result { - Ok(result) => return Ok(result), - Err(api::Error::Network(_)) => { - wait( - exponential_backoff_base_millis, - exponential_backoff_multiplier, - exponential_backoff_max_delay_millis, - retries, - ) - .await; - } - Err(api::Error::BadStatus(code, _)) if !fail_cond(code) => { - wait( - exponential_backoff_base_millis, - exponential_backoff_multiplier, - exponential_backoff_max_delay_millis, - retries, - ) - .await; - } - Err(e) => return Err(e), - } - retries += 1; - } - Err(api::Error::TooManyRetries(latest_error.unwrap_or_else( - || "too many retries without message, this should not happen".to_owned(), - ))) - } -} +use std::collections::HashSet; +use std::fmt::Write; + +use reqwest::{Response, StatusCode}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::castable_enum; +use crate::{ + api::{Error, try_get_env_var}, + data::BundleData, +}; + +type Result = core::result::Result; + +castable_enum!( + /// 各种Api错误的转换 + #[allow(missing_docs)] + #[derive(Debug, thiserror::Error, PartialEq, Eq)] + #repr(i64) + pub enum ApiErrorMap { + #[error( + "An error occurred completing purchases. Please try restarting your device or Xxxxxx and ensuring that you\"re logged in ,to." + )] + PurchaseFailed = 0, + #[error("This item is currently unavailable to purchase.")] + PurchaseUnavailable = 1, + #[error("All songs are already downloaded!")] + AllSongsAlreadyDownloaded = 2, + #[error("You have been logged out by another device. Please restart Xxxxxx.")] + LoggedOutByOtherDevice = 3, + #[error("Could not connect to online server.")] + CouldNotConnectToServer4 = 4, + #[error("Incorrect app version.")] + IncorrectAppVersion = 5, + #[error("An unknown error has occurred.")] + Unknown6 = 6, + #[error("An unknown error has occurred.")] + Unknown7 = 7, + #[error("An unknown error has occurred.")] + Unknown8 = 8, + #[error("The Xxxxxx network is currently under maintenance.")] + ServerInMaintenance = 9, + #[error("An unknown error has occured.")] + Unknown10 = 10, + #[error("An unknown error has occured.")] + Unknown11 = 11, + #[error("Please update Xxxxxx to the latest version.")] + UpdateVersionRequested12 = 12, + #[error( + "Registrations from this IP address are restricted.\nTry again later or contact support@yyyyyy.com." + )] + RegistrationsIPRestricted = 100, + #[error("This username is already in use.")] + DuplicatedUserName = 101, + #[error("This email address is already in use.")] + DuplicatedUserEmail = 102, + #[error("An account has already been made from this device.")] + RegistrationsDevicesRestricted = 103, + #[error("Username or password incorrect.")] + IncorrectUsernameOrPassword = 104, + #[error( + "You\"ve logged into over 2 devices in 24 hours. Please wait before using this new device." + )] + TooManyDevicesLogged = 105, + #[error("This account is locked.")] + UserLocked106 = 106, + #[error("You do not have enough stamina.")] + NoEnoughStamina = 107, + #[error("An unknown error has occurred.")] + Unknown108 = 108, + #[error("An unknown error has occurred.")] + Unknown109 = 109, + #[error("An unknown error has occurred.")] + Unknown110 = 110, + #[error("An unknown error has occurred.")] + Unknown111 = 111, + #[error("World map not unlocked.")] + WorldMapLocked = 112, + #[error("This event map has ended and is no longer available.")] + EventMapEnded = 113, + #[error("An unknown error has occurred.")] + Unknown114 = 114, + #[error("An unknown error has occurred.")] + Unknown115 = 115, + #[error("An unknown error has occurred.")] + Unknown116 = 116, + #[error("An unknown error has occurred.")] + Unknown117 = 117, + #[error("An unknown error has occurred.")] + Unknown118 = 118, + #[error("An unknown error has occurred.")] + Unknown119 = 119, + #[error( + "WARNING! You are using a modified version of Xxxxxx.\nContinued use will result in the banning of your account.\nThis ,is a final warning." + )] + ModifiedAppDetected = 120, + #[error("This account is locked.")] + UserLocked121 = 121, + #[error( + "A temporary hold has been placed on your account.\nPlease visit the official website to resolve the issue." + )] + UserTemporaryLocked = 122, + #[error( + "This feature has been restricted for your account.\nIf you are unsure why, please contact support@yyyyyy.com" + )] + FeatureRestricted = 150, + #[error("This user does not exist.")] + UserNotExist = 401, + #[error("Could not connect to online server.")] + CouldNotConnectToServer403 = 403, + #[error("This item is currently unavailable to purchase.")] + ItemUnavailableToPurchase501 = 501, + #[error("This item is currently unavailable to purchase.")] + ItemUnavailableToPurchase502 = 502, + #[error("An unknown error has occured.")] + Unknown503 = 503, + #[error("Invalid Code")] + InvalidCode = 504, + #[error("This code has already been claimed.")] + CodeAlreadyClaimed = 505, + #[error("You already own this item.")] + ItemAlreadyOwn = 506, + #[error("You can\"t be friends with yourself ;-;")] + AddSelfAsFriend = 604, + #[error("Your friends list is full.")] + FriendListIsFull = 601, + #[error("This user is already your friend.")] + UserIsAlreadyFriend = 602, + #[error( + "There was a problem receiving the server response. Please check your progress after re-entering World Mode." + )] + WorldServerProblem = 801, + #[error("This score could not be submitted online. Please restart or update Xxxxxx.")] + ScoreCouldNotBeSubmitted = 802, + #[error( + "There was a problem submitting this score online. WARNING!Stamina has already been consumed. Exiting will lose World ,Mode progress." + )] + ScoreCouldNotBeSubmittedWithStaminaLost = 803, + #[error("Password reset expired. Please request a new reset link.")] + PasswordResetExpired = 804, + #[error("")] + Unknown805 = 805, + #[error("Max downloads exceeded. Please wait 24 hours and try again.")] + MaxDownloadExceeded = 903, + #[error("Please wait 24 hours before using this feature again.")] + FeatureCoolingDown = 905, + #[error( + "Game data is out of sync due to another device. Please check your progress after re-entering World Mode." + )] + WorldDataOutOfSync = 9701, + #[error("An error occured downloading the song.Please try again.")] + DownloadSongFailed = 9801, + #[error("There was a problem saving the song.Please check storage.")] + SaveSongFailed = 9802, + #[error("No data found to sync.")] + NoDataFoundToSync = 9905, + #[error( + "Sync failed due to conflicting data from another device. Please perform sync from Main Menu > Network." + )] + SyncFailedDueToConflictingData = 9906, + #[error("A problem occured updating data...")] + UpdatingDataFailed = 9907, + #[error("There is a new version of Xxxxxx available.Please update.")] + UpdateVersionRequested9908 = 9908, + #[error("cuscuta does not know this error")] + Unknown = -1, + } +); + +/// xxxxxx api的通用错误 +#[derive(Debug, Deserialize, Clone)] +pub struct ApiError { + /// 错误码 + pub error_code: i64, +} + +/// xxxxxx api登录成功时的适配数据模型 +#[derive(Debug, Deserialize, Clone)] +pub struct LoginResult { + /// 用户id + pub user_id: i64, + + /// 访问密钥 + pub access_token: String, + + /// 密钥类型，通常为`Bearer` + pub token_type: String, +} + +/// xxxxxx api好友变更时的适配数据模型（顶层） +#[derive(Debug, Deserialize, Clone)] +pub struct FriendListResult { + /// 值 + pub value: FriendListResult1, +} + +/// xxxxxx api好友变更时的适配数据模型（第二层） +#[derive(Debug, Deserialize, Clone)] +pub struct FriendListResult1 { + /// 好友信息 + pub friends: Vec, +} + +/// xxxxxx api好友信息的适配数据模型 +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] +pub struct FriendInfo { + /// 好友游戏名 + pub name: String, + + /// 好友用户id + pub user_id: i64, + + /// 好友的评级（实际显示评级为`rating/10.0`） + pub rating: i64, + + /// 好友设置的搭档 + pub character: i64, + + /// 好友搭档状态1 + pub is_char_uncapped: bool, + + /// 好友搭档状态2 + pub is_char_uncapped_override: bool, +} + +/// xxxxxx api曲目成绩的适配数据模型（顶层） +#[derive(Debug, Serialize, Deserialize)] +pub struct SongScoreResult { + ///曲目信息 + pub value: Vec, +} + +/// xxxxxx api曲目成绩的适配数据模型 +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SongScore { + /// 曲目id + pub song_id: String, + + /// 玩家的id + pub user_id: i64, + + /// 难度等级（如0、1、2） + pub difficulty: i64, + + /// 游玩分数 + pub score: i64, + + /// 游玩分数距最高分的差距 + pub score_below_max: i64, + + /// 大P数目 + pub shiny_perfect_count: i64, + + /// 小P数目 + pub perfect_count: i64, + + /// Far数目 + pub near_count: i64, + + /// Lost数目 + pub miss_count: i64, + + /// 通关类型 + pub clear_type: i64, + + /// 最高纪录的通关类型 + pub best_clear_type: i64, + + /// 通关时搭档血量 + pub health: i64, + + /// 游玩时间戳 + pub time_played: i64, + + /// 玩家名字 + #[serde(rename = "name")] + pub player_name: String, +} + +/// 通过xxxxxx api登录 +/// +/// # Errors +/// - 当环境变量配置无效时，返回[`Error::Env`] +/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] +/// - 当请求发送失败时，返回[`Error::Network`] +/// - 当返回值不为2xx时，返回[`Error::BadStatus`] +/// - 当Json反序列化失败时，返回[`Error::Decode`] +pub async fn api_login( + bundle_data: &BundleData, + email: &str, + password: &str, + random_challenge: &str, +) -> Result { + reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .post(try_get_env_var("API_LOGIN")?) + .header("X-Random-Challenge", random_challenge) + .header("AppVersion", bundle_data.application_version_number.clone()) + .header("ContentBundle", bundle_data.version_number.clone()) + .header("DeviceId", generate_device_id(email)) + .basic_auth(email, Some(password)) + .form(&[("grant_type", "client_credentials")]) + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await? + .json::() + .await + .map_err(|e| Error::Decode { + message: e.to_string(), + }) +} + +/// 通过xxxxxx api查询好友 +/// +/// # Errors +/// - 当环境变量配置无效时，返回[`Error::Env`] +/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] +/// - 当请求发送失败时，返回[`Error::Network`] +/// - 当返回值不为2xx时，返回[`Error::BadStatus`] +/// - 当Json反序列化失败时，返回[`Error::Decode`] +pub async fn api_list_friend( + bundle_data: &BundleData, + email: &str, + user_id: &str, + token: &str, +) -> Result { + reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .get(try_get_env_var("API_LIST_FRIENDS")?) + .header("X-Random-Challenge", generate_random_challenge()) + .header("Platform", "android") + .header("AppVersion", bundle_data.application_version_number.clone()) + .header("ContentBundle", bundle_data.version_number.clone()) + .header("DeviceId", generate_device_id(email)) + .header("i", user_id) + .bearer_auth(token) + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await? + .json::() + .await + .map_err(|e| Error::Decode { + message: e.to_string(), + }) + .map(|it| it.value) +} + +/// 通过xxxxxx api添加好友 +/// +/// # Errors +/// - 当环境变量配置无效时，返回[`Error::Env`] +/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] +/// - 当请求发送失败时，返回[`Error::Network`] +/// - 当返回值不为2xx时，返回[`Error::BadStatus`] +/// - 当Json反序列化失败时，返回[`Error::Decode`] +pub async fn api_add_friend( + bundle_data: &BundleData, + email: &str, + user_id: &str, + token: &str, + friend_code: &str, +) -> Result { + reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .post(try_get_env_var("API_ADD_FRIENDS")?) + .header("X-Random-Challenge", generate_random_challenge()) + .header("Platform", "android") + .header("AppVersion", bundle_data.application_version_number.clone()) + .header("ContentBundle", bundle_data.version_number.clone()) + .header("DeviceId", generate_device_id(email)) + .header("i", user_id) + .bearer_auth(token) + .form(&[("friend_code", friend_code)]) + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await? + .json::() + .await + .map_err(|e| Error::Decode { + message: e.to_string(), + }) + .map(|it| it.value) +} + +/// 通过xxxxxx api删除好友 +/// +/// # Errors +/// - 当环境变量配置无效时，返回[`Error::Env`] +/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] +/// - 当请求发送失败时，返回[`Error::Network`] +/// - 当返回值不为2xx时，返回[`Error::BadStatus`] +/// - 当Json反序列化失败时，返回[`Error::Decode`] +pub async fn api_delete_friend( + bundle_data: &BundleData, + email: &str, + user_id: &str, + token: &str, + friend_id: &str, +) -> Result { + reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .post(try_get_env_var("API_DELETE_FRIENDS")?) + .header("X-Random-Challenge", generate_random_challenge()) + .header("Platform", "android") + .header("AppVersion", bundle_data.application_version_number.clone()) + .header("ContentBundle", bundle_data.version_number.clone()) + .header("DeviceId", generate_device_id(email)) + .header("i", user_id) + .bearer_auth(token) + .form(&[("friend_id", friend_id)]) + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await? + .json::() + .await + .map_err(|e| Error::Decode { + message: e.to_string(), + }) + .map(|it| it.value) +} + +/// 通过xxxxxx api查询排行榜 +/// +/// # Errors +/// - 当环境变量配置无效时，返回[`Error::Env`] +/// - 当`reqwest`客户端初始化失败时，返回[`Error::ClientSetup`] +/// - 当请求发送失败时，返回[`Error::Network`] +/// - 当返回值不为2xx时，返回[`Error::BadStatus`] +/// - 当Json反序列化失败时，返回[`Error::Decode`] +#[allow(clippy::too_many_arguments)] +pub async fn api_get_rank_list( + bundle_data: &BundleData, + email: &str, + user_id: &str, + token: &str, + song_id: &str, + difficulty: &str, + start: &str, + limit: &str, +) -> Result> { + reqwest::Client::builder() + .user_agent("curl/7.88.1") + .build() + .map_err(Error::ClientSetup)? + .get(try_get_env_var("API_GET_RANK")?) + .query(&[ + ("song_id", song_id), + ("difficulty", difficulty), + ("start", start), + ("limit", limit), + ]) + .header("X-Random-Challenge", generate_random_challenge()) + .header("Platform", "android") + .header("AppVersion", bundle_data.application_version_number.clone()) + .header("ContentBundle", bundle_data.version_number.clone()) + .header("DeviceId", generate_device_id(email)) + .header("i", user_id) + .bearer_auth(token) + .send() + .await + .map_err(Error::Network)? + .error_for_status_with_response() + .await? + .json::() + .await + .map_err(|e| Error::Decode { + message: e.to_string(), + }) + .map(|it| it.value) +} + +fn generate_device_id(email: &str) -> String { + // Warn: 这只是为了保证唯一性 + let mut result = String::with_capacity(16); + for it in Sha256::digest(email).into_iter().take(8) { + let _ = write!(&mut result, "{it:02x}"); + } + result +} + +/// 生成Random Challenge的占位符 +/// +/// 在不没有硬性检查的端点，就全部使用占位符替代 +fn generate_random_challenge() -> String { + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".into() +} + +/// [`calc_friend_delta`]所使用的好友Delta量 +pub enum FriendDelta { + /// 好友相较`before`增加了一个 + Add(FriendInfo), + /// 好友相较`before`减少了一个 + Remove(FriendInfo), + /// 好友相较`before`没有变化 + Same, +} + +/// 计算好友变化，返回好友信息 +/// +/// 当好友仅变化1个或不变化时，返回`Ok` +/// +/// # Errors +/// 当好友变化量不为0或1时，返回`String`的`Err`\ +/// （TODO：计划重构来改善错误处理） +/// +/// # Panics +/// 所有可能的panic的代码来自于`set.iter().next().expect(...)`，而根据条件，这些set的长度在执行它之前均为1，故本函数理论上永远不会panic +pub fn calc_friend_delta( + before: &[FriendInfo], + after: &[FriendInfo], +) -> core::result::Result { + let before: HashSet<_> = before.iter().collect(); + let after: HashSet<_> = after.iter().collect(); + let delta_add: HashSet<_> = after.difference(&before).collect(); + if delta_add.len() == 1 { + return Ok(FriendDelta::Add( + (**delta_add + .iter() + .next() + .expect("first element of delta_add is None when len==1, this should not happen")) + .clone(), + )); + } + if !delta_add.is_empty() { + return Err("bad add delta".into()); + } + let delta_rem: HashSet<_> = before.difference(&after).collect(); + if delta_rem.len() == 1 { + return Ok(FriendDelta::Remove( + (**delta_rem + .iter() + .next() + .expect("first element of delta_rem is None when len==1, this should not happen")) + .clone(), + )); + } + if !delta_rem.is_empty() { + return Err("bad rem delta".into()); + } + Ok(FriendDelta::Same) +} + +trait ErrorForStatusWithResponseXxxxxx +where + Self: Sized, +{ + fn error_for_status_with_response( + self, + ) -> impl Future>; +} + +impl ErrorForStatusWithResponseXxxxxx for Response { + async fn error_for_status_with_response(self) -> std::result::Result { + fn parse(api_error: &ApiError, status_code: StatusCode) -> Error { + let error_code = api_error.error_code; + let error_map = ApiErrorMap::from(error_code); + Error::BadStatus { + status_code, + extra_error_code: Some(error_code), + message: if error_map == ApiErrorMap::Unknown { + format!("failed to map error description: code {error_code}") + } else { + error_map.to_string() + }, + } + } + match self.error_for_status_ref() { + Ok(_) => Ok(self), + Err(e) => { + let status_code = self.status(); + let text = self.text().await.map_err(|it| Error::BadStatus { + status_code, + message: format!("[FAILED TO GET BODY: {it} {e}]"), + extra_error_code: None, + })?; + let api_error = serde_json::from_str::(text.as_str()).map_err(|it| { + Error::BadStatus { + status_code, + message: format!("[FAILED TO PARSE JSON: {it} {e}]"), + extra_error_code: None, + } + })?; + Err(parse(&api_error, status_code)) + } + } + } +} + +/// 这个模块提供了便捷的xxxxxx api调用包装 +pub mod auto { + use std::time::Duration; + + use reqwest::StatusCode; + use tokio::time::{Sleep, sleep}; + + use crate::api; + + /// 一个包装，用于为一定量的[`StatusCode`]错误和网络错误提供弹性 + /// 使用简化的错误判断，任何错误均重试 + /// + /// # Errors + /// 当本函数因重试次数过多而失败时，返回[`api::Error::TooManyRetries`]\ + /// 否则，返回的错误由指定函数所可能引发的错误决定 + #[allow(clippy::cast_possible_truncation)] + pub async fn xxxxxx_safe_call<'a, F, R, Fut>( + max_retries: u64, + exponential_backoff_base_millis: u64, + exponential_backoff_multiplier: u64, + exponential_backoff_max_delay_millis: u64, + f: F, + ) -> Result + where + Fut: Future> + 'a + Send, + R: Send + 'a, + F: Fn() -> Fut, + { + xxxxxx_safe_call_ex( + max_retries, + exponential_backoff_base_millis, + exponential_backoff_multiplier, + exponential_backoff_max_delay_millis, + |_| false, + f, + ) + .await + } + + /// 一个包装，用于为一定量的[`StatusCode`]错误和网络错误提供弹性 + /// + /// # Errors + /// 当本函数因重试次数过多而失败时，返回[`api::Error::TooManyRetries`]\ + /// 否则，返回的错误由指定函数所可能引发的错误决定 + /// + /// # Panics + /// 本函数不会因为除不合理传入`max_retries`或`exponential_backoff_multiplier`以外的情况下panic， + /// 本函数的panic来自最后返回的`expect`，当`latest_error`为`None`时panic， + /// 但由于`retries < max_retries.max(1)`，故循环体至少会被执行一次， + /// 故在最后一行可达的情况下，`latest_error`不可能为`None`， + /// 故一般使用情况下，本函数不会panic ~（除非调用者故意找茬）~ + #[allow(clippy::cast_possible_truncation)] + pub async fn xxxxxx_safe_call_ex<'a, F, R, T, Fut>( + max_retries: u64, + exponential_backoff_base_millis: u64, + exponential_backoff_multiplier: u64, + exponential_backoff_max_delay_millis: u64, + fail_cond: T, + f: F, + ) -> Result + where + Fut: Future> + 'a + Send, + R: Send + 'a, + F: Fn() -> Fut, + T: Fn(StatusCode) -> bool, + { + //TODO add re-login + fn wait( + exponential_backoff_base_millis: u64, + exponential_backoff_multiplier: u64, + exponential_backoff_max_delay_millis: u64, + retries: u64, + ) -> Sleep { + sleep(Duration::from_millis( + (exponential_backoff_base_millis + * exponential_backoff_multiplier.saturating_pow(retries as u32)) + .min(exponential_backoff_max_delay_millis), + )) + } + let mut retries = 0; + let mut latest_error = None; + while retries < max_retries.max(1) { + let result = f().await; + match result { + Ok(result) => return Ok(result), + Err(e) => { + match &e { + api::Error::Network(_) => { + wait( + exponential_backoff_base_millis, + exponential_backoff_multiplier, + exponential_backoff_max_delay_millis, + retries, + ) + .await; + } + api::Error::BadStatus { status_code, .. } if !fail_cond(*status_code) => { + wait( + exponential_backoff_base_millis, + exponential_backoff_multiplier, + exponential_backoff_max_delay_millis, + retries, + ) + .await; + } + _ => return Err(e), + } + latest_error = Some(e); + } + } + retries += 1; + } + Err(latest_error.expect("this should not happen")) + } +} diff --git a/cuscuta-common/src/db/job/enqueue.rs b/cuscuta-common/src/db/job/enqueue.rs index be57716..bb63ce6 100644 --- a/cuscuta-common/src/db/job/enqueue.rs +++ b/cuscuta-common/src/db/job/enqueue.rs @@ -2,7 +2,7 @@ use redis::{Client, TypedCommands}; use crate::db::{job::JobEssential, redis::job_sub_queue_redis_key}; -/// 向任务队列写入新任务 +/// 向任务队列写入新任务，如果成功，返回新任务的redis stream id /// /// # Errors /// 本函数的错误全部来自[`redis::RedisError`] diff --git a/cuscuta-common/src/db/job/mod.rs b/cuscuta-common/src/db/job/mod.rs index 7a8aa51..dd992a4 100644 --- a/cuscuta-common/src/db/job/mod.rs +++ b/cuscuta-common/src/db/job/mod.rs @@ -1,375 +1,365 @@ -use std::{collections::HashMap, ops::Range}; - -use chrono::Utc; -use redis::{Client, FromRedisValue, ScanOptions, TypedCommands, streams::StreamId}; -use serde::{Deserialize, Serialize}; -use sha2::Digest; - -use crate::{api::xxxxxx::FriendInfo, db::redis::Error}; - -/// 剩余时间相关功能 -pub mod eta; - -/// 任务完成情况相关功能 -pub mod fetch; - -/// 任务入列相关功能 -pub mod enqueue; - -/// 任跟踪相关功能 -pub mod track; - -/// 代表一个任务分片，对应Redis数据库中的分任务队列 -/// -/// 一个任务队列的Key格式如下：\ -/// `cuscuta:jobs:chunk_hash_timestamp_from_to` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SubQueue { - /// 这个任务队列的全名，例如`cuscuta:jobs:chunk_00000000_123456789_114_514` - pub name: String, - - /// \[WIP\] 这个任务队列所对应的`song_list`的hash，目前处于未完成状态 - pub hash: String, - - /// 这个任务队列创建时的时间戳，对应任务队列名字的`timestamp` - pub timestamp: u64, - - /// 这个任务队列所占有的分块，对应队列名字的`from`和`to` - pub segment: Range, -} - -/// 一个Worker负责的`Job`实例，包含`Job`的关键信息和临时状态信息 -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Job { - // From Redis - /// 任务在Redis队列（Stream）中的id - pub job_id: String, - - /// 任务的关键信息 - pub essential: JobEssential, - - // States - /// 源任务队列 - pub sub_queue: SubQueue, - - /// Job的内部状态 - pub state: JobState, -} - -/// 记录任务的状态 -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum JobState { - /// 任务刚刚被拉取，还没有加好友 - Pulled { - /// 任务开始时的时间戳 - start_timestamp: i64, - }, - - /// 任务已加好友，正在进行 - Pending { - /// 任务的好友信息 - friend_info: FriendInfo, - - /// 任务目前进行的长度 - current_length: usize, - - /// 任务开始时的时间戳 - start_timestamp: i64, - }, - - /// 任务已经完成，等待清理 - Finished { - /// 任务的好友信息 - friend_info: FriendInfo, - - /// 任务开始时的时间戳 - start_timestamp: i64, - }, - - /// 任务失败 - Failed { - /// 任务的好友信息（可能有） - friend_info: Option, - - /// 任务开始时的时间戳 - start_timestamp: i64, - - /// 任务失败情况 - failure_info: JobFailure, - }, - - /// 任务已被清理 - Cleaned, -} - -/// 任务的失败情况 -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct JobFailure { - /// Job 失败的原因 - pub fail_type: JobFailureType, - - /// Job 失败的恢复策略 - pub resume_strategy: JobFailureResuming, - - /// Job 失败的时间 - pub timestamp_millis: i64, -} - -/// 失败时的恢复策略 -#[derive(Debug, Serialize, Deserialize, Clone)] -pub enum JobFailureResuming { - /// 任务直接失败 - Drop, - - /// 无操作 - NoOp, -} - -macro_rules! castable_enum { - ( - $(#[$meta:meta])* - #repr($repr:ty) - $vis:vis enum $name:ident { - $( - $(#[$vmeta:meta])* - $Variant:ident$(($($v:tt)*))? = $code:expr, - )* - } - ) => { - $(#[$meta])* - $vis enum $name { - $( - $(#[$vmeta])* - $Variant $(($($v)*))?, - )* - } - impl $name { - #[allow(missing_docs)] - $vis const fn get_repr(&self) -> $repr { - match self { - $(Self::$Variant {..} => $code,)* - } - } - } - }; -} - -castable_enum! { - /// 任务的失败信息 - #[derive(Debug, Serialize, Deserialize, thiserror::Error, Clone)] - #repr(i32) - pub enum JobFailureType { - /// 好友找不到，一般是好友码无效 - #[error("friend not found")] - FriendNotFound = -1, - - /// 无法找到对应的tracking条目，尝试创建新的 - #[error("worker failed to find exist target key: when {0}")] - TargetKeyNotFound(String) = -2, - - /// Job 重新入队 - #[error("job Reenqueued")] - Reenqueued = -3, - } -} - -/// `Job`的关键信息 -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct JobEssential { - /// 查询对象的好友代码 - pub friend_code: String, - - /// 任务入列时的时间戳 - pub timestamp: String, - - /// 任务开始时的游标 - pub cursor_start: i32, - - /// 任务需要轮询的曲目长度 - pub cursor_length: i32, - - /// \[WIP\] 任务的重试次数，正在实现 - pub retry_count: i32, - - /// 任务的唯一ID，不应随任务重新入队而改变 - pub job_uid: String, -} - -impl JobFailure { - /// 新建一个[`JobTrackFailure`]，自动填写当前时间戳 - #[must_use] - pub fn new(fail_type: JobFailureType, resume_strategy: JobFailureResuming) -> Self { - Self { - fail_type, - resume_strategy, - timestamp_millis: Utc::now().timestamp_millis(), - } - } -} - -impl TryFrom<(SubQueue, StreamId)> for Job { - type Error = Error; - fn try_from((sub_queue, id): (SubQueue, StreamId)) -> Result { - let map = id.map; - let timestamp = Utc::now().timestamp_millis(); - Ok(Self { - job_id: id.id, - essential: JobEssential::try_from(&map)?, - sub_queue, - state: JobState::Pulled { - start_timestamp: timestamp, - }, - }) - } -} - -impl PartialEq for Job { - fn eq(&self, other: &Self) -> bool { - self.essential == other.essential - } -} - -impl Job { - /// 获取Job对应的结果类队列id - #[must_use] - pub fn get_stream_key_postfix(&self) -> String { - self.essential.get_stream_key_postfix() - } -} - -impl SubQueue { - /// 获取任务的后缀 - #[must_use] - pub fn get_postfix(&self) -> String { - format!( - "chunk_{}_{}_{}_{}", - self.hash, self.timestamp, self.segment.start, self.segment.end - ) - } -} - -impl JobEssential { - /// 创建新任务重要信息 - /// - /// 提供任务基本参数，生成新的`JobEssential`\ - /// 其中`job_uid`由这个函数自动使用SHA256生成 - #[must_use] - pub fn new( - friend_code: String, - timestamp: String, - cursor_start: i32, - cursor_length: i32, - retry_count: i32, - ) -> Self { - let traits: Vec<_> = friend_code - .as_bytes() - .iter() - .chain(timestamp.as_bytes().iter()) - .chain(cursor_start.to_le_bytes().iter()) - .chain(cursor_length.to_le_bytes().iter()) - .copied() - .collect(); - let digest = hex::encode(sha2::Sha256::digest(traits)); - Self { - friend_code, - timestamp, - cursor_start, - cursor_length, - retry_count, - job_uid: digest, - } - } - - /// 获取Job对应的结果类队列id - #[must_use] - pub fn get_stream_key_postfix(&self) -> String { - format!("{}-{}", self.friend_code.clone(), self.timestamp.clone()) - } -} - -impl TryFrom<&str> for SubQueue { - type Error = Error; - - fn try_from(name: &str) -> Result { - let chunk_info: Vec<&str> = name.split('_').collect(); - if chunk_info.len() != 5 { - return Err(Error::BadData(format!("bad queue name: {name}"))); - } - let segment_from = chunk_info[3].parse::().map_err(|e| { - Error::BadData(format!( - "bad segment start \"{}\" of {} ({e})", - chunk_info[3], name - )) - })?; - let segment_to = chunk_info[4].parse::().map_err(|e| { - Error::BadData(format!( - "bad segment end \"{}\" of {} ({e})", - chunk_info[4], name - )) - })?; - if segment_from > segment_to { - return Err(Error::BadData(format!("bad segment (end().map_err(|e| { - Error::BadData(format!( - "bad timestamp \"{}\" of {} ({e})", - chunk_info[2], name - )) - })?, - segment: segment_from..segment_to, - }) - } -} - -impl TryFrom<&HashMap> for JobEssential { - type Error = Error; - - fn try_from(map: &HashMap) -> Result { - Ok(Self::new( - from_redis(map, "job:friend_code")?, - from_redis(map, "job:timestamp")?, - from_redis(map, "job:cursor_start")?, - from_redis(map, "job:cursor_length")?, - from_redis(map, "job:retry_count")?, - )) - } -} - -/// 不保证同步性，搜索工作队列分片 -/// -/// # Errors -/// 这个函数产生的错误来自Redis的错误，以及读取内容不符合预期的错误，参见[`crate::db::job::Error`] -pub fn scan_sub_queue(redis_client: &Client) -> Result, Error> { - let mut connection = redis_client.get_connection().map_err(Error::Redis)?; - let mut sub_queues = Vec::new(); - for it in connection - .scan_options::( - ScanOptions::default() - .with_count(100) - .with_pattern("cuscuta:jobs:*") - .with_type("stream"), - ) - .map_err(Error::Redis)? - { - let name = it.map_err(Error::Redis)?; - sub_queues.push(SubQueue::try_from(name.as_str())?); - } - sub_queues.sort_by_key(|it| it.timestamp); - Ok(sub_queues) -} - -fn from_redis(value: &HashMap, key: &str) -> Result -where - T: FromRedisValue, -{ - T::from_redis_value( - value - .get(key) - .ok_or(Error::BadData(format!("no key found {key}")))? - .clone(), - ) - .map_err(|e| Error::BadData(format!("failed to parse redis value: {key} : {e}"))) -} +use std::{collections::HashMap, ops::Range}; + +use chrono::Utc; +use redis::{Client, FromRedisValue, ScanOptions, TypedCommands, streams::StreamId}; +use serde::{Deserialize, Serialize}; +use sha2::Digest; + +use crate::{api::xxxxxx::FriendInfo, castable_enum_with_arg, db::redis::Error}; + +/// 剩余时间相关功能 +pub mod eta; + +/// 任务完成情况相关功能 +pub mod fetch; + +/// 任务入列相关功能 +pub mod enqueue; + +/// 任跟踪相关功能 +pub mod track; + +/// 代表一个任务分片，对应Redis数据库中的分任务队列 +/// +/// 一个任务队列的Key格式如下：\ +/// `cuscuta:jobs:chunk_hash_timestamp_from_to` +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SubQueue { + /// 这个任务队列的全名，例如`cuscuta:jobs:chunk_00000000_123456789_114_514` + pub name: String, + + /// \[WIP\] 这个任务队列所对应的`song_list`的hash，目前处于未完成状态 + pub hash: String, + + /// 这个任务队列创建时的时间戳，对应任务队列名字的`timestamp` + pub timestamp: u64, + + /// 这个任务队列所占有的分块，对应队列名字的`from`和`to` + pub segment: Range, +} + +/// 一个Worker负责的`Job`实例，包含`Job`的关键信息和临时状态信息 +#[derive(Debug, Clone, Serialize, Deserialize, Eq)] +pub struct Job { + // From Redis + /// 任务在Redis队列（Stream）中的id + pub job_id: String, + + /// 任务的关键信息 + pub essential: JobEssential, + + // States + /// 源任务队列 + pub sub_queue: SubQueue, + + /// Job的内部状态 + pub state: JobState, +} + +/// 记录任务的状态 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum JobState { + /// 任务刚刚被拉取，还没有加好友 + Pulled { + /// 任务开始时的时间戳 + start_timestamp: i64, + }, + + /// 任务已加好友，正在进行 + Pending { + /// 任务的好友信息 + friend_info: FriendInfo, + + /// 任务目前进行的长度 + current_length: usize, + + /// 任务开始时的时间戳 + start_timestamp: i64, + }, + + /// 任务已经完成，等待清理 + Finished { + /// 任务的好友信息 + friend_info: FriendInfo, + + /// 任务开始时的时间戳 + start_timestamp: i64, + }, + + /// 任务失败 + Failed { + /// 任务的好友信息（可能有） + friend_info: Option, + + /// 任务开始时的时间戳 + start_timestamp: i64, + + /// 任务失败情况 + failure_info: JobFailure, + }, + + /// 任务已被清理 + Cleaned, +} + +/// 任务的失败情况 +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct JobFailure { + /// Job 失败的原因 + pub fail_type: JobFailureType, + + /// Job 失败的恢复策略 + pub resume_strategy: JobFailureResuming, + + /// Job 失败的时间 + pub timestamp_millis: i64, +} + +/// 失败时的恢复策略 +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum JobFailureResuming { + /// 任务直接失败 + Drop, + + /// 无操作 + NoOp, +} + +castable_enum_with_arg! { + /// 任务的失败信息 + #[derive(Debug, Serialize, Deserialize, thiserror::Error, Clone, PartialEq, Eq)] + #repr(i64) + pub enum JobFailureType { + /// 好友找不到，一般是好友码无效 + #[error("friend not found")] + FriendNotFound = -1, + + /// 无法找到对应的tracking条目，尝试创建新的 + #[error("worker failed to find exist target key: when {0}")] + TargetKeyNotFound(String) = -2, + + /// Job 重新入队 + #[error("job Reenqueued")] + Reenqueued = -3, + + /// 远程Api错误 + #[error("remote xxxxxx api error, HTTP {0}: {1:?}")] + XxxxxxApiError(u16, Option) = -4, + + /// 其它Api错误 + #[error("other api error: {0:?}")] + ApiError(String) = -5, + } +} + +/// `Job`的关键信息 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct JobEssential { + /// 查询对象的好友代码 + pub friend_code: String, + + /// 任务入列时的时间戳 + pub timestamp: String, + + /// 任务开始时的游标 + pub cursor_start: i32, + + /// 任务需要轮询的曲目长度 + pub cursor_length: i32, + + /// \[WIP\] 任务的重试次数，正在实现 + pub retry_count: i32, + + /// 任务的唯一ID，不应随任务重新入队而改变 + pub job_uid: String, +} + +impl JobFailure { + /// 新建一个[`JobTrackFailure`]，自动填写当前时间戳 + #[must_use] + pub fn new(fail_type: JobFailureType, resume_strategy: JobFailureResuming) -> Self { + Self { + fail_type, + resume_strategy, + timestamp_millis: Utc::now().timestamp_millis(), + } + } +} + +impl TryFrom<(SubQueue, StreamId)> for Job { + type Error = Error; + fn try_from((sub_queue, id): (SubQueue, StreamId)) -> Result { + let map = id.map; + let timestamp = Utc::now().timestamp_millis(); + Ok(Self { + job_id: id.id, + essential: JobEssential::try_from(&map)?, + sub_queue, + state: JobState::Pulled { + start_timestamp: timestamp, + }, + }) + } +} + +impl PartialEq for Job { + fn eq(&self, other: &Self) -> bool { + self.essential == other.essential + } +} + +impl Job { + /// 获取Job对应的结果类队列id + #[must_use] + pub fn get_stream_key_postfix(&self) -> String { + self.essential.get_stream_key_postfix() + } +} + +impl SubQueue { + /// 获取任务的后缀 + #[must_use] + pub fn get_postfix(&self) -> String { + format!( + "chunk_{}_{}_{}_{}", + self.hash, self.timestamp, self.segment.start, self.segment.end + ) + } +} + +impl JobEssential { + /// 创建新任务重要信息 + /// + /// 提供任务基本参数，生成新的`JobEssential`\ + /// 其中`job_uid`由这个函数自动使用SHA256生成 + #[must_use] + pub fn new( + friend_code: String, + timestamp: String, + cursor_start: i32, + cursor_length: i32, + retry_count: i32, + ) -> Self { + Self { + friend_code, + timestamp, + cursor_start, + cursor_length, + retry_count, + job_uid: "temp".to_owned(), + } + .generate_uid() + } + + /// 从其它字段推断`job_uid` + #[must_use] + pub fn generate_uid(self) -> Self { + let traits: Vec<_> = self + .friend_code + .as_bytes() + .iter() + .chain(self.timestamp.as_bytes().iter()) + .chain(self.cursor_start.to_le_bytes().iter()) + .chain(self.cursor_length.to_le_bytes().iter()) + .copied() + .collect(); + Self { + job_uid: hex::encode(sha2::Sha256::digest(traits)), + ..self + } + } + + /// 获取Job对应的结果类队列id + #[must_use] + pub fn get_stream_key_postfix(&self) -> String { + format!("{}-{}", self.friend_code.clone(), self.timestamp.clone()) + } +} + +impl TryFrom<&str> for SubQueue { + type Error = Error; + + fn try_from(name: &str) -> Result { + let chunk_info: Vec<&str> = name.split('_').collect(); + if chunk_info.len() != 5 { + return Err(Error::BadData(format!("bad queue name: {name}"))); + } + let segment_from = chunk_info[3].parse::().map_err(|e| { + Error::BadData(format!( + "bad segment start \"{}\" of {} ({e})", + chunk_info[3], name + )) + })?; + let segment_to = chunk_info[4].parse::().map_err(|e| { + Error::BadData(format!( + "bad segment end \"{}\" of {} ({e})", + chunk_info[4], name + )) + })?; + if segment_from > segment_to { + return Err(Error::BadData(format!("bad segment (end().map_err(|e| { + Error::BadData(format!( + "bad timestamp \"{}\" of {} ({e})", + chunk_info[2], name + )) + })?, + segment: segment_from..segment_to, + }) + } +} + +impl TryFrom<&HashMap> for JobEssential { + type Error = Error; + + fn try_from(map: &HashMap) -> Result { + Ok(Self::new( + from_redis(map, "job:friend_code")?, + from_redis(map, "job:timestamp")?, + from_redis(map, "job:cursor_start")?, + from_redis(map, "job:cursor_length")?, + from_redis(map, "job:retry_count")?, + )) + } +} + +/// 不保证同步性，搜索工作队列分片 +/// +/// # Errors +/// 这个函数产生的错误来自Redis的错误，以及读取内容不符合预期的错误，参见[`crate::db::job::Error`] +pub fn scan_sub_queue(redis_client: &Client) -> Result, Error> { + let mut connection = redis_client.get_connection().map_err(Error::Redis)?; + let mut sub_queues = Vec::new(); + for it in connection + .scan_options::( + ScanOptions::default() + .with_count(100) + .with_pattern("cuscuta:jobs:*") + .with_type("stream"), + ) + .map_err(Error::Redis)? + { + let name = it.map_err(Error::Redis)?; + sub_queues.push(SubQueue::try_from(name.as_str())?); + } + sub_queues.sort_by_key(|it| it.timestamp); + sub_queues.sort_by_key(|it| it.segment.start); + Ok(sub_queues) +} + +fn from_redis(value: &HashMap, key: &str) -> Result +where + T: FromRedisValue, +{ + T::from_redis_value( + value + .get(key) + .ok_or(Error::BadData(format!("no key found {key}")))? + .clone(), + ) + .map_err(|e| Error::BadData(format!("failed to parse redis value: {key} : {e}"))) +} diff --git a/cuscuta-common/src/db/mod.rs b/cuscuta-common/src/db/mod.rs index ed53245..af0f69f 100644 --- a/cuscuta-common/src/db/mod.rs +++ b/cuscuta-common/src/db/mod.rs @@ -111,4 +111,15 @@ pub mod redis { pub fn job_eta_redis_key() -> String { "cuscuta:eta:record".to_string() } + + /// 任务分片后缀 + #[must_use] + pub fn sub_queue_postfix( + hash: &str, + timestamp: &str, + range_start: usize, + range_end: usize, + ) -> String { + format!("chunk_{hash}_{timestamp}_{range_start}_{range_end}") + } } diff --git a/cuscuta-common/src/lib.rs b/cuscuta-common/src/lib.rs index 77d7ce7..84500b4 100644 --- a/cuscuta-common/src/lib.rs +++ b/cuscuta-common/src/lib.rs @@ -1,22 +1,90 @@ -//! cuscuta的通用组件 -//! -//! # 速览 -//! cuscuta-common定义了一些通用的函数、数据模型和一些实用函数，包括各种API，数据库的通用业务操作，定时操作的简易注册和管理，和一些全局变量的线程安全的便捷操作 -//! -//! # 局限 -//! 很明显，cuscuta并没有做数据层兼容，即cuscuta目前**强绑定于**`Redis`和`PostgreSQL`，这将在未来必要时重构 - -/// api相关 -pub mod api; - -/// 通用数据结构相关 -pub mod data; - -/// 数据库相关 -pub mod db; - -/// 便捷变量操作相关 -pub mod quick_fetch; - -/// 定时操作相关 -pub mod scheduled_job; +//! cuscuta的通用组件 +//! +//! # 速览 +//! cuscuta-common定义了一些通用的函数、数据模型和一些实用函数，包括各种API，数据库的通用业务操作，定时操作的简易注册和管理，和一些全局变量的线程安全的便捷操作 +//! +//! # 局限 +//! 很明显，cuscuta并没有做数据层兼容，即cuscuta目前**强绑定于**`Redis`和`PostgreSQL`，这将在未来必要时重构 + +/// api相关 +pub mod api; + +/// 通用数据结构相关 +pub mod data; + +/// 数据库相关 +pub mod db; + +/// 便捷变量操作相关 +pub mod quick_fetch; + +/// 定时操作相关 +pub mod scheduled_job; + +/// 为有参数的枚举类型添加数字转换 +#[macro_export] +macro_rules! castable_enum_with_arg { + ( + $(#[$meta:meta])* + #repr($repr:ty) + $vis:vis enum $name:ident { + $( + $(#[$vmeta:meta])* + $Variant:ident$(($($v:tt)*))? = $code:expr, + )* + } + ) => { + $(#[$meta])* + $vis enum $name { + $( + $(#[$vmeta])* + $Variant $(($($v)*))?, + )* + } + impl From<$name> for $repr { + fn from(v: $name) -> Self { + match v { + $($name::$Variant {..} => $code,)* + } + } + } + }; +} + +/// 为无参数的枚举类型添加数字转换 +#[macro_export] +macro_rules! castable_enum { + ( + $(#[$meta:meta])* + #repr($repr:ty) + $vis:vis enum $name:ident { + $( + $(#[$vmeta:meta])* + $Variant:ident = $code:expr, + )* + } + ) => { + $(#[$meta])* + $vis enum $name { + $( + $(#[$vmeta])* + $Variant, + )* + } + impl From<$name> for $repr { + fn from(v: $name) -> Self { + match v { + $($name::$Variant => $code,)* + } + } + } + impl From<$repr> for $name { + fn from(v: $repr) -> Self { + match v { + $($code => $name::$Variant,)* + _ => Self::Unknown + } + } + } + }; +} diff --git a/cuscuta-entry/src/endpoints/enqueue.rs b/cuscuta-entry/src/endpoints/enqueue.rs index 6dd8b0c..c73cb1b 100644 --- a/cuscuta-entry/src/endpoints/enqueue.rs +++ b/cuscuta-entry/src/endpoints/enqueue.rs @@ -13,11 +13,14 @@ use crate::{ }; use cuscuta_common::{ - db::job::{ - JobEssential, SubQueue, - enqueue::write_job, - scan_sub_queue, - track::{JobTrackQueueStatus, JobTrackTag, batch_write_job_tracking_tag}, + db::{ + job::{ + JobEssential, SubQueue, + enqueue::write_job, + scan_sub_queue, + track::{JobTrackQueueStatus, JobTrackTag, batch_write_job_tracking_tag}, + }, + redis::sub_queue_postfix, }, quick_fetch::QuickFetch, }; @@ -111,10 +114,7 @@ async fn op(form: EnqueueBody) -> anyhow::Result { let (queue_name, exist) = target_queue.map_or_else( || { ( - format!( - "chunk_{}_{}_{}_{}", - "00000000", timestamp, range.start, range.end - ), + sub_queue_postfix("00000000", ×tamp, range.start, range.end), false, ) }, diff --git a/cuscuta-entry/src/endpoints/query.rs b/cuscuta-entry/src/endpoints/query.rs index ca13832..d8142db 100644 --- a/cuscuta-entry/src/endpoints/query.rs +++ b/cuscuta-entry/src/endpoints/query.rs @@ -184,10 +184,10 @@ fn check_evidence( .failures .first() { - let failure_type = first_failed.fail_type.get_repr(); + let failure_type = i64::from(first_failed.fail_type.clone()); ( EvidenceCheckResult::JobFailed { - code: failure_type.into(), + code: failure_type, message: format!("{:?}", first_failed.fail_type), }, Some(job_tracks), diff --git a/cuscuta-test/Cargo.toml b/cuscuta-test/Cargo.toml new file mode 100644 index 0000000..6cb0d78 --- /dev/null +++ b/cuscuta-test/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cuscuta-test" +edition = "2024" +version.workspace = true + +[lib] + +[dependencies] +testcontainers = "0.27.3" +cuscuta-common = { path = "../cuscuta-common" } +tokio = { workspace = true, features = ["macros"] } +redis = { workspace = true } + +[lints] +workspace = true diff --git a/cuscuta-test/src/lib.rs b/cuscuta-test/src/lib.rs new file mode 100644 index 0000000..653c071 --- /dev/null +++ b/cuscuta-test/src/lib.rs @@ -0,0 +1,173 @@ +//! > [!CAUTION] +//! > +//! > 测试相关的组件 +//! > +//! > 这个文件内存放的mod, trait, fn 皆为 **测试** 相关 +//! > +//! > 它们的实现细节相比真实情况做出了可能合理但 **绝对高度的简化** +//! > +//! > 所以它们 **不应** 被用于任何除测试以外的场景 + +/// mock +pub mod mocks; + +/// mock相关 +pub mod mock { + /// 一个简单的`mock`包装trait + pub trait SimpleMockable { + /// 生成一个mock + fn mock() -> Self; + } +} + +/// 测试容器创建相关 +pub mod container { + /// 启动一个redis测试容器并获取[`redis::Client`] + #[macro_export] + macro_rules! redis_client_image { + () => {{ + use redis::Client; + use testcontainers::{ + GenericImage, ImageExt, + core::{ContainerPort, WaitFor}, + runners::AsyncRunner, + }; + let redis_container = GenericImage::new("redis", "7.2.4") + .with_exposed_port(ContainerPort::Tcp(6379)) + .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")) + .start() + .await + .expect("failed to boot image"); + ClientWrap( + Client::open(format!( + "redis://{}:{}", + redis_container + .get_host() + .await + .expect("failed to get host"), + redis_container + .get_host_port_ipv4(6379) + .await + .expect("failed to get host port") + )) + .expect("failed to open redis client"), + redis_container, + ) + }}; + } +} + +/// redis相关 +pub mod redis { + + use redis::Client; + + use cuscuta_common::db::{ + job::{JobEssential, enqueue::write_job}, + redis::sub_queue_postfix, + }; + use testcontainers::{ContainerAsync, Image}; + + /// 为防止[`ContainerAsync`]提前回收的包装 + pub struct ClientWrap(pub Client, pub ContainerAsync); + + /// 不检查任何参数，入队模拟查分任务，仅用作测试 + /// + /// 就像这样： + /// ``` + /// use cuscuta_common::db::{ + /// job::{Job, JobEssential, SubQueue}, redis::{job_sub_queue_redis_key, sub_queue_postfix}, + /// }; + /// use cuscuta_test::{ + /// mock::SimpleMockable, + /// mocks::mock::job_state, + /// redis::{ClientWrap, fake_enqueue}, + /// redis_client_image, + /// }; + /// use redis::{TypedCommands, streams::StreamReadOptions}; + /// #[tokio::main] + /// async fn main() { + /// let ClientWrap(client, _container) = redis_client_image!(); + /// let hash = "0000000"; + /// let friend_code = "123456789"; + /// let job_timestamp = "1784475024"; + /// let queue_timestamp = "1784475023"; + /// fake_enqueue(&client, hash, friend_code, job_timestamp, queue_timestamp) + /// .expect("failed to enqueue fake jobs"); + /// let mut connection = client.get_connection().expect("failed to open connection"); + /// let sub_queue_postfix = sub_queue_postfix(hash, queue_timestamp, 0, 10); + /// let sub_queue = + /// SubQueue::try_from(sub_queue_postfix.as_str()).expect("failed to parse SubQueue str"); + /// let reply = connection + /// .xread_options( + /// &[&job_sub_queue_redis_key(&sub_queue_postfix)], + /// &[">"], + /// &StreamReadOptions::default().group("default_group", "mock_id"), + /// ) + /// .expect("failed to read from stream") + /// .expect("result should be `Some`"); + /// let stream_id = reply + /// .keys + /// .first() + /// .expect("should have result in stream") + /// .ids + /// .first() + /// .expect("should have result in key"); + /// let job_deserialized = + /// Job::try_from((sub_queue.clone(), stream_id.clone())).expect("failed to parse job"); + /// assert_eq!( + /// job_deserialized.essential, + /// JobEssential::new(friend_code.to_string(), job_timestamp.to_string(), 0, 10, 0) + /// ); + /// assert_eq!( + /// job_deserialized.sub_queue, + /// sub_queue + /// ); + /// } + /// ``` + /// + /// # Errors + /// 本函数的错误全部来自[`redis::RedisError`] + /// + pub fn fake_enqueue( + client: &Client, + hash: &str, + friend_code: &str, + job_timestamp: &str, + queue_timestamp: &str, + ) -> Result, redis::RedisError> { + [ + JobEssential::new(friend_code.to_string(), job_timestamp.to_string(), 0, 10, 0), + JobEssential::new( + friend_code.to_string(), + job_timestamp.to_string(), + 10, + 10, + 0, + ), + JobEssential::new( + friend_code.to_string(), + job_timestamp.to_string(), + 20, + 10, + 0, + ), + ] + .map(|it| { + write_job( + client, + &it, + &sub_queue_postfix( + hash, + queue_timestamp, + it.cursor_start.cast_unsigned() as usize, + (it.cursor_start + it.cursor_length).cast_unsigned() as usize, + ), + true, + 60, + ) + }) + .into_iter() + .collect::, _>>() + } +} diff --git a/cuscuta-test/src/mocks.rs b/cuscuta-test/src/mocks.rs new file mode 100644 index 0000000..667fa39 --- /dev/null +++ b/cuscuta-test/src/mocks.rs @@ -0,0 +1,169 @@ +/// 生成各种结构体的简易mock +pub mod mock { + use crate::mock::SimpleMockable; + use cuscuta_common::{ + api::xxxxxx::FriendInfo, + data::{Difficulty, Song}, + db::job::{Job, JobEssential, JobFailure, JobFailureResuming, JobFailureType, SubQueue}, + }; + + impl SimpleMockable for Job { + fn mock() -> Self { + Self { + job_id: "mock".to_owned(), + essential: JobEssential::mock(), + sub_queue: SubQueue::mock(), + state: job_state::mock_cleaned(), + } + } + } + + impl SimpleMockable for FriendInfo { + fn mock() -> Self { + Self { + name: "nofyso".to_owned(), + user_id: 123_456, + rating: 16, + character: 0, + is_char_uncapped: false, + is_char_uncapped_override: false, + } + } + } + + impl SimpleMockable for Vec { + fn mock() -> Self { + vec![ + Song { + idx: 0, + id: "mock0".to_owned(), + difficulties: vec![ + Difficulty { + rating_class: 0, + rating: 2, + }, + Difficulty { + rating_class: 1, + rating: 2, + }, + ], + }, + Song { + idx: 1, + id: "mock1".to_owned(), + difficulties: vec![ + Difficulty { + rating_class: 0, + rating: 2, + }, + Difficulty { + rating_class: 1, + rating: 2, + }, + ], + }, + Song { + idx: 2, + id: "mock2".to_owned(), + difficulties: vec![ + Difficulty { + rating_class: 0, + rating: 2, + }, + Difficulty { + rating_class: 1, + rating: 2, + }, + ], + }, + ] + } + } + + /// [`cuscuta_common::db::job::JobState`]的各种mock + pub mod job_state { + use cuscuta_common::{ + api::xxxxxx::FriendInfo, + db::job::{JobFailure, JobState}, + }; + + use crate::mock::SimpleMockable; + + /// 生成一个mock的[`JobState`] + #[must_use] + pub const fn mock_cleaned() -> JobState { + JobState::Cleaned + } + + /// 生成一个mock的[`JobState`] + #[must_use] + pub const fn mock_pulled() -> JobState { + JobState::Pulled { + start_timestamp: 1_784_475_024, + } + } + + /// 生成一个mock的[`JobState`] + #[must_use] + pub fn mock_pending() -> JobState { + JobState::Pending { + friend_info: FriendInfo::mock(), + current_length: 0, + start_timestamp: 1_784_475_024, + } + } + + /// 生成一个mock的[`JobState`] + #[must_use] + pub fn mock_finished() -> JobState { + JobState::Finished { + friend_info: FriendInfo::mock(), + start_timestamp: 1_784_475_024, + } + } + + /// 生成一个mock的[`JobState`] + #[must_use] + pub fn mock_failed() -> JobState { + JobState::Failed { + friend_info: Some(FriendInfo::mock()), + start_timestamp: 1_784_475_024, + failure_info: JobFailure::mock(), + } + } + } + + impl SimpleMockable for JobFailure { + fn mock() -> Self { + Self { + fail_type: JobFailureType::FriendNotFound, + resume_strategy: JobFailureResuming::Drop, + timestamp_millis: 1_784_475_024, + } + } + } + + impl SimpleMockable for SubQueue { + fn mock() -> Self { + Self { + name: "mock".to_owned(), + hash: "mock".to_owned(), + timestamp: 1_784_475_024, + segment: 0..5, + } + } + } + + impl SimpleMockable for JobEssential { + fn mock() -> Self { + Self { + friend_code: "123456789".to_owned(), + timestamp: "1784475024".to_owned(), + cursor_start: 0, + cursor_length: 0, + retry_count: 0, + job_uid: "abcde".to_owned(), + } + } + } +} diff --git a/cuscuta-worker/Cargo.toml b/cuscuta-worker/Cargo.toml index 6d978aa..f6469b3 100644 --- a/cuscuta-worker/Cargo.toml +++ b/cuscuta-worker/Cargo.toml @@ -20,5 +20,9 @@ tokio = { workspace = true, features = ["signal"] } tokio-util = { workspace = true } gethostname = "1.1.0" +[dev-dependencies] +testcontainers = "0.27.3" +cuscuta-test = { path = "../cuscuta-test" } + [lints] workspace = true diff --git a/cuscuta-worker/src/data.rs b/cuscuta-worker/src/data.rs index 6f2c97d..324d2c9 100644 --- a/cuscuta-worker/src/data.rs +++ b/cuscuta-worker/src/data.rs @@ -17,6 +17,7 @@ pub struct Config { pub worker_account_lease_time_secs: u64, pub _worker_account_lease_time_refresh_gap_secs: u64, pub worker_job_max_work_time_secs: u64, + pub worker_empty_friends_delay_time_secs: u64, } pub static BUNDLE_DATA: OnceLock>> = OnceLock::new(); @@ -26,3 +27,27 @@ pub static SONG_LIST: OnceLock>>> = OnceLock::new(); pub static ACCOUNT_ROW: OnceLock>> = OnceLock::new(); pub static WORKER_ID: OnceLock = OnceLock::new(); + +#[cfg(test)] +pub mod mock { + use cuscuta_test::mock::SimpleMockable; + + use crate::data::Config; + + impl SimpleMockable for Config { + fn mock() -> Self { + Self { + worker_max_jobs: 8, + worker_max_retry_count: 5, + worker_exponential_backoff_base_millis: 100, + worker_exponential_backoff_multiplier: 2, + worker_exponential_backoff_max_delay_millis: 500, + redis_stream_refresh_ttl: 600, + worker_account_lease_time_secs: 60, + _worker_account_lease_time_refresh_gap_secs: 5, + worker_job_max_work_time_secs: 600, + worker_empty_friends_delay_time_secs: 10, + } + } + } +} diff --git a/cuscuta-worker/src/db/account.rs b/cuscuta-worker/src/db/account.rs index 813de0e..da8ff8a 100644 --- a/cuscuta-worker/src/db/account.rs +++ b/cuscuta-worker/src/db/account.rs @@ -15,9 +15,11 @@ pub async fn perform_login( bundle_data: &BundleData, account_row: &AccountRow, ) -> Result { - let url = - Url::from_str(&env::var("API_LOGIN").map_err(|e| api::Error::Env(e, "API_LOGIN".into()))?) - .unwrap_or_else(|_| Url::from_str("http://nofyso:11451/auth/login").unwrap()); + let url = Url::from_str(&env::var("API_LOGIN").map_err(|error| api::Error::Env { + error, + message: "API_LOGIN".into(), + })?) + .unwrap_or_else(|_| Url::from_str("http://nofyso:11451/auth/login").unwrap()); let timestamp = Local::now().timestamp_millis().to_string(); let random_challenge = match chilo_generate( ×tamp, @@ -27,12 +29,13 @@ pub async fn perform_login( .await? { api::chilo::ChiloResult::Success { value, .. } => value, - api::chilo::ChiloResult::Failed { message, .. } => { + api::chilo::ChiloResult::Failed { message } => { tracing::warn!("login_interface: failed to generate challenge: {message}"); - return Err(api::Error::BadStatus( - StatusCode::INTERNAL_SERVER_ERROR, + return Err(api::Error::BadStatus { + status_code: StatusCode::INTERNAL_SERVER_ERROR, message, - )); + extra_error_code: None, + }); } }; tracing::info!("login_interface: challenge: {random_challenge}"); diff --git a/cuscuta-worker/src/db/mod.rs b/cuscuta-worker/src/db/mod.rs index e59d545..a848afa 100644 --- a/cuscuta-worker/src/db/mod.rs +++ b/cuscuta-worker/src/db/mod.rs @@ -24,6 +24,6 @@ macro_rules! worker_write_event { use cuscuta_common::try_write_event; use $crate::data::WORKER_ID; use $crate::db::redis::REDIS_CLIENT; - try_write_event!(REDIS_CLIENT, WORKER_ID, $event_type, $message); + try_write_event!(REDIS_CLIENT, WORKER_ID, $event_type, $message.to_string()); }}; } diff --git a/cuscuta-worker/src/init.rs b/cuscuta-worker/src/init.rs index eb1f118..2f57943 100644 --- a/cuscuta-worker/src/init.rs +++ b/cuscuta-worker/src/init.rs @@ -35,106 +35,152 @@ impl From<(Level, String)> for Error { } } -#[allow(clippy::too_many_lines)] -pub async fn cuscuta_init(service_token: &CancellationToken, init_token: &CancellationToken) { - async fn get_friend_result( - bundle_data: &BundleData, - account_row: &AccountRow, - ) -> Result { - let first_result = try_update_token(bundle_data, account_row, false).await; - match first_result { - Ok(result) => return Ok(result), - Err((Level::DirtyAccount, _)) => {} - Err(e) => return Err(e.into()), - } - let second_result = try_update_token(bundle_data, account_row, true).await; - match second_result { - Ok(result) => Ok(result), - Err((Level::DirtyAccount, m)) => Err((Level::Halt, m).into()), - Err(e) => Err(e.into()), - } +async fn get_friend_result( + bundle_data: &BundleData, + account_row: &AccountRow, +) -> Result { + let first_result = try_update_token(bundle_data, account_row, false).await; + match first_result { + Ok(result) => return Ok(result), + Err((Level::DirtyAccount, _)) => {} + Err(e) => return Err(e.into()), } - async fn try_init() -> Result<(), Error> { - tracing::info!("init: cuscuta-worker initializing..."); - let bundle_data = BUNDLE_DATA - .try_read(std::clone::Clone::clone) - .map_err(|e| (Level::Retry, format!("failed to read BUNDLE_DATA: {e}")))?; - let worker_account_lease_time_secs = CONFIG - .try_read(|it| it.worker_account_lease_time_secs) - .map_err(|e| (Level::Retry, format!("failed to read BUNDLE_DATA: {e}")))?; - REDIS_CLIENT - .get() - .ok_or_else(|| (Level::Retry, "redis is not ready".to_string()))? - .get_connection() - .map_err(|e| { - ( - Level::Retry, - format!("failed to connection to redis server: {e}"), - ) - })?; - let account_row = try_lock_account( - try_open_transaction() - .await - .map_err(|e| (Level::Retry, format!("failed to open transaction: {e}")))?, - worker_account_lease_time_secs, - ) - .await + let second_result = try_update_token(bundle_data, account_row, true).await; + match second_result { + Ok(result) => Ok(result), + Err((Level::DirtyAccount, m)) => Err((Level::Halt, m).into()), + Err(e) => Err(e.into()), + } +} + +async fn try_init() -> Result<(), Error> { + tracing::info!("init: cuscuta-worker initializing..."); + let bundle_data = BUNDLE_DATA + .try_read(std::clone::Clone::clone) + .map_err(|e| (Level::Retry, format!("failed to read BUNDLE_DATA: {e}")))?; + let worker_account_lease_time_secs = CONFIG + .try_read(|it| it.worker_account_lease_time_secs) + .map_err(|e| (Level::Retry, format!("failed to read BUNDLE_DATA: {e}")))?; + REDIS_CLIENT + .get() + .ok_or_else(|| (Level::Retry, "redis is not ready".to_string()))? + .get_connection() .map_err(|e| { ( Level::Retry, - format!("failed to operate postgresql database: {e}"), - ) - })? - .ok_or_else(|| (Level::Halt, "no account found".to_string()))?; - tracing::info!("init: locked {}", account_row.id); - ACCOUNT_ROW - .try_write(|_| Some(account_row.clone())) - .map_err(|e| (Level::Retry, format!("failed to write ACCOUNT_ROW: {e}")))?; - let TokenUpdateResult { - account_row, - friends, - } = get_friend_result(&bundle_data, &account_row).await?; - tracing::info!("init: found {} existing friends", friends.friends.len()); - for friend in friends.friends { - api_delete_friend( - &bundle_data, - &account_row.account_email, - &account_row - .user_id - .ok_or_else(|| (Level::Halt, "unexpected data #1".to_string()))? - .to_string(), - &account_row - .temp_token - .clone() - .ok_or_else(|| (Level::Halt, "unexpected data #2".to_string()))?, - &friend.user_id.to_string(), + format!("failed to connection to redis server: {e}"), ) + })?; + let account_row = try_lock_account( + try_open_transaction() .await - .map_err(|e| (Level::Halt, format!("failed to clean friends: {e}")))?; - tracing::info!("init: removed friend: {friend:?}"); - } + .map_err(|e| (Level::Retry, format!("failed to open transaction: {e}")))?, + worker_account_lease_time_secs, + ) + .await + .map_err(|e| { + ( + Level::Retry, + format!("failed to operate postgresql database: {e}"), + ) + })? + .ok_or_else(|| (Level::Halt, "no account found".to_string()))?; + tracing::info!("init: locked {}", account_row.id); + ACCOUNT_ROW + .try_write(|_| Some(account_row.clone())) + .map_err(|e| (Level::Retry, format!("failed to write ACCOUNT_ROW: {e}")))?; + let TokenUpdateResult { + account_row, + friends, + } = get_friend_result(&bundle_data, &account_row).await?; + tracing::info!("init: found {} existing friends", friends.friends.len()); + for friend in friends.friends { + api_delete_friend( + &bundle_data, + &account_row.account_email, + &account_row + .user_id + .ok_or_else(|| (Level::Halt, "unexpected data #1".to_string()))? + .to_string(), + &account_row + .temp_token + .clone() + .ok_or_else(|| (Level::Halt, "unexpected data #2".to_string()))?, + &friend.user_id.to_string(), + ) + .await + .map_err(|e| (Level::Halt, format!("failed to clean friends: {e}")))?; + tracing::info!("init: removed friend: {friend:?}"); + } + ACCOUNT_ROW + .try_write(|_| Some(account_row.clone())) + .map_err(|e| (Level::Retry, format!("failed to write ACCOUNT_ROW: {e}")))?; + Ok(()) +} + +async fn try_failed_resume() -> Result<(), Error> { + let account_row = ACCOUNT_ROW.try_read(std::clone::Clone::clone).ok(); + if let Some(account_row) = account_row { + let transaction = try_open_transaction() + .await + .map_err(|e| (Level::Retry, format!("failed to open transaction: {e}")))?; + try_release_account(transaction, account_row.id) + .await + .map_err(|e| (Level::Retry, format!("failed to release account: {e}")))?; ACCOUNT_ROW - .try_write(|_| Some(account_row.clone())) - .map_err(|e| (Level::Retry, format!("failed to write ACCOUNT_ROW: {e}")))?; - Ok(()) + .try_write(|_| None) + .map_err(|e| (Level::Retry, format!("failed to clear account row: {e}")))?; + } else { + tracing::info!("init_resume: no account found, skip"); } - async fn try_failed_resume() -> Result<(), Error> { - let account_row = ACCOUNT_ROW.try_read(std::clone::Clone::clone).ok(); - if let Some(account_row) = account_row { - let transaction = try_open_transaction() - .await - .map_err(|e| (Level::Retry, format!("failed to open transaction: {e}")))?; - try_release_account(transaction, account_row.id) - .await - .map_err(|e| (Level::Retry, format!("failed to release account: {e}")))?; - ACCOUNT_ROW - .try_write(|_| None) - .map_err(|e| (Level::Retry, format!("failed to clear account row: {e}")))?; - } else { - tracing::info!("init_resume: no account found, skip"); + Ok(()) +} + +async fn try_update_token( + bundle_data: &BundleData, + account_row: &AccountRow, + account_dirty: bool, +) -> Result { + let friends_result = check_and_update_token(bundle_data, account_row, account_dirty).await; + if let Err(db::account::auto::Error::Api(api::Error::BadStatus { + status_code, + extra_error_code, + message, + })) = friends_result + { + if status_code == 500 { + return Err(( + Level::Halt, + format!( + "bad hash(HTTP {status_code}:{extra_error_code:?} {message}), is chilo out of dated?" + ), + )); } - Ok(()) + update_account_rate( + try_open_transaction() + .await + .map_err(|e| (Level::Halt, format!("failed to {e}")))?, + account_row, + -1, + ) + .await + .map_err(|e| (Level::Retry, format!("failed to update account rate: {e}")))?; + return Err(( + if account_dirty { + Level::Halt + } else { + Level::DirtyAccount + }, + format!("failed to login: HTTP: {status_code} {extra_error_code:?}"), + )); + } + match friends_result { + Ok(result) => Ok(result), + Err(e) => Err((Level::Retry, format!("failed to fetch friend result: {e}"))), } +} + +pub async fn cuscuta_init(service_token: &CancellationToken, init_token: &CancellationToken) { if let Err(Error { level, description }) = try_init().await { if level == Level::Halt { tracing::error!("init: serious error occurred in initializing: {description}, halting"); @@ -171,41 +217,3 @@ pub async fn cuscuta_init(service_token: &CancellationToken, init_token: &Cancel tracing::info!("init: let the cuscuta spread..."); init_token.cancel(); } - -async fn try_update_token( - bundle_data: &BundleData, - account_row: &AccountRow, - account_dirty: bool, -) -> Result { - let friends_result = check_and_update_token(bundle_data, account_row, account_dirty).await; - if let Err(db::account::auto::Error::Api(api::Error::BadStatus(code, message))) = friends_result - { - if code == 500 { - return Err(( - Level::Halt, - format!("bad hash({code}: {message}), is chilo out of dated?"), - )); - } - update_account_rate( - try_open_transaction() - .await - .map_err(|e| (Level::Halt, format!("failed to {e}")))?, - account_row, - -1, - ) - .await - .map_err(|e| (Level::Retry, format!("failed to update account rate: {e}")))?; - return Err(( - if account_dirty { - Level::Halt - } else { - Level::DirtyAccount - }, - format!("failed to login: {code}"), - )); - } - match friends_result { - Ok(result) => Ok(result), - Err(e) => Err((Level::Retry, format!("failed to fetch friend result: {e}"))), - } -} diff --git a/cuscuta-worker/src/loop_tasks.rs b/cuscuta-worker/src/loop_tasks.rs index 7fdb3ac..cf219ef 100644 --- a/cuscuta-worker/src/loop_tasks.rs +++ b/cuscuta-worker/src/loop_tasks.rs @@ -84,6 +84,9 @@ pub async fn sync_config(_: &CancellationToken) { "WORKER_ACCOUNT_LEASE_TIME_REFRESH_GAP_SECS", )?, worker_job_max_work_time_secs: read_as_number("WORKER_JOB_MAX_WORK_TIME_SECS")?, + worker_empty_friends_delay_time_secs: read_as_number( + "WORKER_EMPTY_FRIENDS_DELAY_TIME_SECS", + )?, }; CONFIG .try_write(move |_| config.into()) diff --git a/cuscuta-worker/src/main.rs b/cuscuta-worker/src/main.rs index c1082a6..eec3769 100644 --- a/cuscuta-worker/src/main.rs +++ b/cuscuta-worker/src/main.rs @@ -9,6 +9,7 @@ //! - `WORKER_EXPONENTIAL_BACKOFF_MAX_DELAY_MILLIS` //! - `WORKER_ACCOUNT_LEASE_TIME_SECS` //! - `WORKER_ACCOUNT_LEASE_TIME_REFRESH_GAP_SECS` +//! - `WORKER_EMPTY_FRIENDS_DELAY_TIME_SECS` //! - `REDIS_STREAM_REFRESH_TTL` //! - `GITHUB_BUNDLE_REPOSITORY` //! - `GITHUB_BUNDLE_PATH` diff --git a/cuscuta-worker/src/worker/clean.rs b/cuscuta-worker/src/worker/clean.rs index 5a7ae76..75e0397 100644 --- a/cuscuta-worker/src/worker/clean.rs +++ b/cuscuta-worker/src/worker/clean.rs @@ -111,8 +111,9 @@ pub async fn clean_jobs( WorkerEventType::Warn, format!("failed to delete friend: {e}") ); + } else { + friends.retain(|it| it.user_id != friend_info.user_id); } - friends.retain(|it| it.user_id != friend_info.user_id); } let cursor_length = i64::from(finished_job.essential.cursor_length); if cursor_length != 0 { diff --git a/cuscuta-worker/src/worker/mod.rs b/cuscuta-worker/src/worker/mod.rs index 989dfc7..f304982 100644 --- a/cuscuta-worker/src/worker/mod.rs +++ b/cuscuta-worker/src/worker/mod.rs @@ -7,8 +7,10 @@ use std::time::Duration; use cuscuta_common::{ api::{self, xxxxxx::FriendInfo}, + data::{BundleData, Song}, db::{ self, + account::AccountRow, job::{ Job, JobFailure, JobFailureResuming, JobFailureType, enqueue::write_job, @@ -50,20 +52,26 @@ pub struct WorkerResult { #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("some data is not ready: {0}")] - NotReady(String), + #[error("some data is not ready: {message}")] + NotReady { message: String }, + #[error("redis error: {0}")] Redis(redis::RedisError), + #[error("redis extended error: {0}")] RedisExtend(cuscuta_common::db::redis::Error), + #[error("job parse error: {0}")] JobParse(db::redis::Error), + #[error("json parsing error: {0}")] JsonParse(serde_json::Error), + #[error("api error: {0}")] Api(api::Error), - #[error("bad state error: {0}")] - BadState(String), + + #[error("bad state error: {message}")] + BadState { message: String }, } pub async fn worker_loop(cancellation_token: &CancellationToken) -> WorkerResult { @@ -109,33 +117,77 @@ pub async fn worker_loop(cancellation_token: &CancellationToken) -> WorkerResult worker_result } -async fn internal_loop( - current_jobs: &mut Vec, - cursor: &mut usize, - friends: &mut Vec, -) -> anyhow::Result<(), Error> { - let worker_id = WORKER_ID - .get() - .ok_or(Error::NotReady("worker_id... what?".to_string()))?; - let redis_client = REDIS_CLIENT - .get() - .ok_or(Error::NotReady("redis client".to_string()))?; +struct Args<'a> { + worker_id: &'a String, + redis_client: &'a Client, + account_row: AccountRow, + user_id: String, + token: String, + config: Config, + bundle_data: BundleData, + song_list: Vec, +} + +fn get_args<'a>() -> Result, Error> { + let worker_id = WORKER_ID.get().ok_or(Error::NotReady { + message: "worker_id... what?".to_string(), + })?; + let redis_client = REDIS_CLIENT.get().ok_or(Error::NotReady { + message: "redis client".to_string(), + })?; let account_row = ACCOUNT_ROW .try_read(std::clone::Clone::clone) - .map_err(|e| Error::NotReady(format!("account row ({e})")))?; + .map_err(|e| Error::NotReady { + message: format!("account row ({e})"), + })?; let (user_id, token) = account_row .check_log_info() .map(|(id, token)| (id.to_string(), token)) - .ok_or(Error::NotReady("user is not login".to_string()))?; + .ok_or(Error::NotReady { + message: "user is not login".to_string(), + })?; let config = CONFIG .try_read(std::clone::Clone::clone) - .map_err(|e| Error::NotReady(format!("config ({e})")))?; + .map_err(|e| Error::NotReady { + message: format!("config ({e})"), + })?; let bundle_data = BUNDLE_DATA .try_read(std::clone::Clone::clone) - .map_err(|e| Error::NotReady(format!("bundle data ({e})")))?; + .map_err(|e| Error::NotReady { + message: format!("bundle data ({e})"), + })?; let song_list = SONG_LIST .try_read(std::clone::Clone::clone) - .map_err(|e| Error::NotReady(format!("song list ({e})")))?; + .map_err(|e| Error::NotReady { + message: format!("song list ({e})"), + })?; + Ok(Args { + worker_id, + redis_client, + account_row, + user_id, + token, + config, + bundle_data, + song_list, + }) +} + +async fn internal_loop( + current_jobs: &mut Vec, + cursor: &mut usize, + friends: &mut Vec, +) -> anyhow::Result<(), Error> { + let Args { + worker_id, + redis_client, + account_row, + user_id, + token, + config, + bundle_data, + song_list, + } = get_args()?; let Some(current_segments) = scan_sub_queue_and_pull_job( redis_client, current_jobs, @@ -256,9 +308,9 @@ where pub fn resume_state(worker_result: WorkerResult) { // TODO: complete error handling here fn resume_jobs(worker_result: WorkerResult) -> Result<(), Error> { - let redis_client = REDIS_CLIENT - .get() - .ok_or(Error::NotReady("redis client".to_string()))?; + let redis_client = REDIS_CLIENT.get().ok_or(Error::NotReady { + message: "redis client".to_string(), + })?; let redis_stream_refresh_ttl = CONFIG .try_read(|it| it.redis_stream_refresh_ttl) .unwrap_or(300); diff --git a/cuscuta-worker/src/worker/pending_friend.rs b/cuscuta-worker/src/worker/pending_friend.rs index 4edda1a..5909c9f 100644 --- a/cuscuta-worker/src/worker/pending_friend.rs +++ b/cuscuta-worker/src/worker/pending_friend.rs @@ -1,4 +1,6 @@ -use std::collections::HashMap; +//! 这个文件写的好脏…… + +use std::{collections::HashMap, time::Duration}; use cuscuta_common::{ api::{ @@ -15,9 +17,16 @@ use cuscuta_common::{ }; use redis::{Client, Connection, TypedCommands}; use reqwest::StatusCode; +use tokio::time::sleep; use crate::{data::Config, worker::Error, worker_write_event}; +#[derive(Debug)] +enum AddFriendError { + Api(api::Error), + Wait, +} + #[allow(clippy::cast_possible_truncation, clippy::too_many_arguments)] pub async fn try_add_friends( config: &Config, @@ -57,32 +66,25 @@ pub async fn try_add_friends( job.essential.cursor_start = cursor.cast_signed() as i32; continue; } - let friends_new = - match try_modify_remote_friend(config, bundle_data, account_row, user_id, token, job) - .await - { - Ok(x) => x, - Err(e) => { - job.state = JobState::Failed { - start_timestamp, - failure_info: JobFailure::new( - JobFailureType::FriendNotFound, - JobFailureResuming::Drop, - ), - friend_info: None, - }; - worker_write_event!( - WorkerEventType::Warn, - format!("failed to add friend: {e:?}",) - ); - continue; - } - }; - let friend_delta = calc_friend_delta(friends, &friends_new) - .map_err(|e| Error::BadState(format!("failed to resolve friend delta: {e}")))?; + let friends_new = try_modify_remote_friend( + config, + bundle_data, + user_id, + token, + account_row, + start_timestamp, + job, + ) + .await + .map_err(Error::Api)?; + let friend_delta = + calc_friend_delta(friends, &friends_new).map_err(|e| Error::BadState { + message: format!("failed to resolve friend delta: {e}"), + })?; let friend_add = match friend_delta { FriendDelta::Add(it) => it, FriendDelta::Remove(e) => { + tracing::warn!("pending_friends: friend conflict detected(remove): {e:?}"); worker_write_event!( WorkerEventType::Info, format!("friend conflict detected: {e:?}") @@ -90,9 +92,8 @@ pub async fn try_add_friends( continue; } FriendDelta::Same => { - return Err(Error::BadState( - "bad friend delta (nothing changed)".to_string(), - )); + tracing::warn!("pending_friends: friends keep same, may triggered something"); + continue; } }; ids.insert(job.essential.friend_code.clone(), friend_add.clone()); @@ -109,13 +110,106 @@ pub async fn try_add_friends( } async fn try_modify_remote_friend( + config: &Config, + bundle_data: &BundleData, + user_id: &str, + token: &str, + account_row: &AccountRow, + start_timestamp: i64, + job: &mut Job, +) -> Result, api::Error> { + match try_add_friend(config, bundle_data, account_row, user_id, token, job).await { + Ok(x) => Ok(x), + Err(AddFriendError::Api(e)) => { + let failure_info = if let api::Error::BadStatus { + status_code, + extra_error_code, + .. + } = &e + { + if *status_code == 404 { + JobFailure::new(JobFailureType::FriendNotFound, JobFailureResuming::Drop) + } else { + JobFailure::new( + JobFailureType::XxxxxxApiError(status_code.as_u16(), *extra_error_code), + JobFailureResuming::Drop, + ) + } + } else { + JobFailure::new( + JobFailureType::ApiError(format!("{e:?}")), + JobFailureResuming::Drop, + ) + }; + job.state = JobState::Failed { + start_timestamp, + failure_info, + friend_info: None, + }; + worker_write_event!( + WorkerEventType::Warn, + format!("failed to add friend: {e:?}",) + ); + Err(e) + } + Err(AddFriendError::Wait) => { + worker_write_event!(WorkerEventType::Warn, "triggered friend modify waiting"); + // To reviewers: Due to the target's rate limiting strategy, + // blocking work queue is expected behavior here + sleep(Duration::from_secs( + config.worker_empty_friends_delay_time_secs, + )) + .await; + try_re_get_friend(config, bundle_data, user_id, token, account_row).await + } + } +} + +async fn try_re_get_friend( + config: &Config, + bundle_data: &BundleData, + user_id: &str, + token: &str, + account_row: &AccountRow, +) -> Result, api::Error> { + loop { + let result = xxxxxx_safe_call_ex( + config.worker_max_retry_count, + config.worker_exponential_backoff_base_millis, + config.worker_exponential_backoff_multiplier, + config.worker_exponential_backoff_max_delay_millis, + |it| it != StatusCode::TOO_MANY_REQUESTS, + || { + api::xxxxxx::api_list_friend( + bundle_data, + &account_row.account_email, + user_id, + token, + ) + }, + ) + .await + .map(|it| it.friends)?; + if !result.is_empty() { + return Ok(result); + } + // To reviewers: Due to the target's rate limiting strategy, + // blocking work queue is expected behavior here + sleep(Duration::from_secs( + config.worker_empty_friends_delay_time_secs, + )) + .await; + } +} + +async fn try_add_friend( config: &Config, bundle_data: &BundleData, account_row: &AccountRow, user_id: &str, token: &str, job: &Job, -) -> Result, api::Error> { +) -> Result, AddFriendError> { let result = xxxxxx_safe_call_ex( config.worker_max_retry_count, config.worker_exponential_backoff_base_millis, @@ -135,40 +229,33 @@ async fn try_modify_remote_friend( .await; match result { Err(e) => { - if let api::Error::BadStatus(code, message) = &e { - tracing::warn!("pending_friends: failed to call friend_add: {message}"); - if *code == 400 { - tracing::warn!( - "pending_friends: friend is already exist but cache is out-of-date! trying readd" - ); - xxxxxx_safe_call_ex( - config.worker_max_retry_count, - config.worker_exponential_backoff_base_millis, - config.worker_exponential_backoff_multiplier, - config.worker_exponential_backoff_max_delay_millis, - |it| it != StatusCode::TOO_MANY_REQUESTS, - || { - api::xxxxxx::api_delete_friend( - bundle_data, - &account_row.account_email, - user_id, - token, - &job.essential.friend_code, - ) - }, + if let api::Error::BadStatus { + status_code, + extra_error_code, + message, + } = &e + { + tracing::warn!( + "pending_friends: failed to call friend_add: HTTP {status_code} {message}" + ); + worker_write_event!( + WorkerEventType::Warn, + format!( + "failed to add friend: HTTP {status_code}: {extra_error_code:?}: {message}" ) - .await - .map(|it| it.friends) - } else { - tracing::warn!("pending_friends: failed to add friend: {e}: code: {code}"); - Err(e) - } + ); } else { tracing::warn!("pending_friends: unexpected error: {e}"); - Err(e) } + Err(AddFriendError::Api(e)) + } + Ok(it) => { + if !it.friends.is_empty() { + return Ok(it.friends); + } + tracing::warn!("pending_friends: returning friends is empty, waiting"); + Err(AddFriendError::Wait) } - Ok(it) => Ok(it.friends), } } diff --git a/cuscuta-worker/src/worker/pending_gather.rs b/cuscuta-worker/src/worker/pending_gather.rs index 6a97f95..cfd84fb 100644 --- a/cuscuta-worker/src/worker/pending_gather.rs +++ b/cuscuta-worker/src/worker/pending_gather.rs @@ -52,8 +52,9 @@ pub fn write_result_to_redis( ) -> Result<(), Error> { let mut connection = redis_client.get_connection().map_err(Error::Redis)?; for (key, score) in score_pairs { - let json = serde_json::to_string(score) - .map_err(|e| Error::BadState(format!("failed to serialize data to json: {e}")))?; + let json = serde_json::to_string(score).map_err(|e| Error::BadState { + message: format!("failed to serialize data to json: {e}"), + })?; connection.lpush(key, &json).map_err(Error::Redis)?; } Ok(()) diff --git a/cuscuta-worker/src/worker/pull.rs b/cuscuta-worker/src/worker/pull/mod.rs similarity index 93% rename from cuscuta-worker/src/worker/pull.rs rename to cuscuta-worker/src/worker/pull/mod.rs index 0d04956..7bdf457 100644 --- a/cuscuta-worker/src/worker/pull.rs +++ b/cuscuta-worker/src/worker/pull/mod.rs @@ -1,243 +1,247 @@ -use std::collections::HashMap; - -use cuscuta_common::{ - data::Song, - db::{ - self, - job::{ - Job, JobFailure, JobFailureResuming, JobFailureType, JobState, SubQueue, - scan_sub_queue, - track::{JobTrackQueueStatus, JobTrackTag}, - }, - log::WorkerEventType, - }, -}; -use redis::{ - Client, TypedCommands, - streams::{StreamAutoClaimOptions, StreamId, StreamReadOptions}, -}; - -use crate::{ - data::Config, - worker::{Error, update_job_track_info}, - worker_write_event, -}; - -// 好吧我承认这里写的有点脏了 -pub async fn scan_sub_queue_and_pull_job( - redis_client: &Client, - current_jobs: &mut Vec, - cursor: &mut usize, - config: &Config, - song_list: &[Song], - random: &str, -) -> Result, Error> { - let current_sub_queue = current_jobs.first().map(|it| it.sub_queue.clone()); - let song_list_len = song_list.len(); - let (new_jobs, current_segments) = if let Some(s) = current_sub_queue { - ( - pull_jobs( - current_jobs, - &s, - config, - redis_client, - random, - song_list_len, - )?, - s, - ) - } else { - let sub_queues = scan_sub_queue(redis_client).map_err(|e| match e { - db::redis::Error::Redis(redis_error) => Error::Redis(redis_error), - db::redis::Error::BadData(e) => Error::BadState(format!("bad data: {e}")), - })?; - let Some((jobs, sub_queue)) = discover_sub_queue( - current_jobs, - config, - redis_client, - random, - &sub_queues, - song_list_len, - )? - else { - return Ok(None); - }; - *cursor = sub_queue.segment.start; - (Some(jobs), sub_queue) - }; - if let Some(new_jobs) = new_jobs { - for it in new_jobs { - tracing::info!("worker_loop: pulled job: {it:?}"); - worker_write_event!(WorkerEventType::Trace, format!("pulled: {it:?}")); - update_job_track_info( - redis_client, - &it, - |it| { - it.status = JobTrackQueueStatus::Pending; - }, - || JobTrackTag { - status: JobTrackQueueStatus::Queueing, - job_ids: vec![it.job_id.clone()], - queue: it.sub_queue.clone(), - job_essential: it.essential.clone(), - failures: vec![JobFailure::new( - JobFailureType::TargetKeyNotFound("pulling job".to_owned()), - JobFailureResuming::NoOp, - )], - }, - )?; - current_jobs.push(it); - } - } - Ok(Some(current_segments)) -} - -#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] -fn claim_redis_jobs( - connection: &mut redis::Connection, - key: &str, - consumer: &str, - min_idle_time_secs: f64, - target_counts: usize, -) -> Result, Error> { - if target_counts == 0 { - return Ok(Vec::new()); - } - Ok(connection - .xautoclaim_options( - key, - "default_group", - consumer, - (min_idle_time_secs * 1000.0) as i64, - "0-0", - StreamAutoClaimOptions::default().count(target_counts), - ) - .map_err(Error::Redis)? - .claimed) -} - -fn fetch_redis_jobs( - connection: &mut redis::Connection, - key: &str, - consumer: &str, - target_counts: usize, -) -> Result, Error> { - if target_counts == 0 { - return Ok(Vec::new()); - } - Ok(connection - .xread_options( - &[&key], - &[">"], - &StreamReadOptions::default() - .group("default_group", consumer) - .count(target_counts), - ) - .map_err(Error::Redis)? - .map_or(Vec::new(), |it| { - it.keys.into_iter().next().map_or(Vec::new(), |it| it.ids) - })) -} - -#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] -fn pull_jobs( - jobs: &[Job], - sub_queue: &SubQueue, - config: &Config, - redis_client: &Client, - pod_uid: &str, - total_length: usize, -) -> Result>, Error> { - // TODO: 添加无GROUP找不到的错误处理（跳过） - let valid_jobs = valid_jobs(jobs); - let mut connection = redis_client.get_connection().map_err(Error::Redis)?; - let max_jobs = config.worker_max_jobs.try_into().expect("wait... what?"); - if valid_jobs >= 1 { - let _ = connection.expire(sub_queue.name.clone(), config.redis_stream_refresh_ttl); - } - if valid_jobs >= max_jobs { - return Ok(Option::None); - } - let divisions = jobs - .first() - .map_or(1, |it| { - (total_length as f64 / it.sub_queue.segment.len() as f64) as i32 - }) - .max(1); - let min_idle_time = - (config.worker_job_max_work_time_secs as f64 / f64::from(divisions)).round(); - let claimed_jobs = match claim_redis_jobs( - &mut connection, - &sub_queue.name, - pod_uid, - min_idle_time, - max_jobs.saturating_sub(valid_jobs), - ) { - Ok(o) => o, - Err(e) => { - tracing::warn!("worker_loop_pull_jobs: failed to claim jobs: {e}"); - Vec::new() - } - }; - let valid_jobs = valid_jobs + claimed_jobs.len(); - let fetched_jobs = match fetch_redis_jobs( - &mut connection, - &sub_queue.name, - pod_uid, - max_jobs.saturating_sub(valid_jobs), - ) { - Ok(o) => o, - Err(e) => { - tracing::warn!("worker_loop_pull_jobs: failed to fetch jobs: {e}"); - Vec::new() - } - }; - let jobs: Vec<_> = fetched_jobs - .iter() - .chain(claimed_jobs.iter()) - .map(|it| { - (sub_queue.clone(), it.clone()) - .try_into() - .map_err(Error::JobParse) - }) - .collect::, Error>>()?; - let mut no_duplicated_jobs = HashMap::new(); - for job in &jobs { - no_duplicated_jobs.entry(&job.essential).or_insert(job); - } - Ok(Some( - no_duplicated_jobs - .into_iter() - .collect::>() - .into_iter() - .map(|it| it.1.clone()) - .collect(), - )) -} - -fn discover_sub_queue( - jobs: &[Job], - config: &Config, - redis_client: &Client, - pod_uid: &str, - sub_queues: &[SubQueue], - total_length: usize, -) -> Result, SubQueue)>, Error> { - for queue in sub_queues { - let Some(jobs) = pull_jobs(jobs, queue, config, redis_client, pod_uid, total_length)? - else { - continue; - }; - if jobs.is_empty() { - continue; - } - return Ok(Some((jobs, queue.clone()))); - } - Ok(None) -} - -fn valid_jobs(jobs: &[Job]) -> usize { - jobs.iter() - .filter(|it| !matches!(it.state, JobState::Cleaned)) - .count() -} +mod test; + +use std::collections::HashMap; + +use cuscuta_common::{ + data::Song, + db::{ + self, + job::{ + Job, JobFailure, JobFailureResuming, JobFailureType, JobState, SubQueue, + scan_sub_queue, + track::{JobTrackQueueStatus, JobTrackTag}, + }, + log::WorkerEventType, + }, +}; +use redis::{ + Client, TypedCommands, + streams::{StreamAutoClaimOptions, StreamId, StreamReadOptions}, +}; + +use crate::{ + data::Config, + worker::{Error, update_job_track_info}, + worker_write_event, +}; + +// 好吧我承认这里写的有点脏了 +pub async fn scan_sub_queue_and_pull_job( + redis_client: &Client, + current_jobs: &mut Vec, + cursor: &mut usize, + config: &Config, + song_list: &[Song], + worker_id: &str, +) -> Result, Error> { + let current_sub_queue = current_jobs.first().map(|it| it.sub_queue.clone()); + let song_list_len = song_list.len(); + let (new_jobs, current_segments) = if let Some(s) = current_sub_queue { + ( + pull_jobs( + current_jobs, + &s, + config, + redis_client, + worker_id, + song_list_len, + )?, + s, + ) + } else { + let sub_queues = scan_sub_queue(redis_client).map_err(|e| match e { + db::redis::Error::Redis(redis_error) => Error::Redis(redis_error), + db::redis::Error::BadData(e) => Error::BadState { + message: format!("bad data: {e}"), + }, + })?; + let Some((jobs, sub_queue)) = discover_sub_queue_for_jobs( + current_jobs, + config, + redis_client, + worker_id, + &sub_queues, + song_list_len, + )? + else { + return Ok(None); + }; + *cursor = sub_queue.segment.start; + (Some(jobs), sub_queue) + }; + if let Some(new_jobs) = new_jobs { + for it in new_jobs { + tracing::info!("worker_loop: pulled job: {it:?}"); + worker_write_event!(WorkerEventType::Trace, format!("pulled: {it:?}")); + update_job_track_info( + redis_client, + &it, + |it| { + it.status = JobTrackQueueStatus::Pending; + }, + || JobTrackTag { + status: JobTrackQueueStatus::Queueing, + job_ids: vec![it.job_id.clone()], + queue: it.sub_queue.clone(), + job_essential: it.essential.clone(), + failures: vec![JobFailure::new( + JobFailureType::TargetKeyNotFound("pulling job".to_owned()), + JobFailureResuming::NoOp, + )], + }, + )?; + current_jobs.push(it); + } + } + Ok(Some(current_segments)) +} + +#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] +fn claim_redis_jobs( + connection: &mut redis::Connection, + key: &str, + consumer: &str, + min_idle_time_secs: f64, + target_counts: usize, +) -> Result, Error> { + if target_counts == 0 { + return Ok(Vec::new()); + } + Ok(connection + .xautoclaim_options( + key, + "default_group", + consumer, + (min_idle_time_secs * 1000.0) as i64, + "0-0", + StreamAutoClaimOptions::default().count(target_counts), + ) + .map_err(Error::Redis)? + .claimed) +} + +fn fetch_redis_jobs( + connection: &mut redis::Connection, + key: &str, + consumer: &str, + target_counts: usize, +) -> Result, Error> { + if target_counts == 0 { + return Ok(Vec::new()); + } + Ok(connection + .xread_options( + &[&key], + &[">"], + &StreamReadOptions::default() + .group("default_group", consumer) + .count(target_counts), + ) + .map_err(Error::Redis)? + .map_or(Vec::new(), |it| { + it.keys.into_iter().next().map_or(Vec::new(), |it| it.ids) + })) +} + +#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] +fn pull_jobs( + jobs: &[Job], + sub_queue: &SubQueue, + config: &Config, + redis_client: &Client, + pod_uid: &str, + total_length: usize, +) -> Result>, Error> { + // TODO: 添加无GROUP找不到的错误处理（跳过） + let valid_jobs = valid_jobs(jobs); + let mut connection = redis_client.get_connection().map_err(Error::Redis)?; + let max_jobs = config.worker_max_jobs.try_into().expect("wait... what?"); + if valid_jobs >= 1 { + let _ = connection.expire(sub_queue.name.clone(), config.redis_stream_refresh_ttl); + } + if valid_jobs >= max_jobs { + return Ok(Option::None); + } + let divisions = jobs + .first() + .map_or(1, |it| { + (total_length as f64 / it.sub_queue.segment.len() as f64) as i32 + }) + .max(1); + let min_idle_time = + (config.worker_job_max_work_time_secs as f64 / f64::from(divisions)).round(); + let claimed_jobs = match claim_redis_jobs( + &mut connection, + &sub_queue.name, + pod_uid, + min_idle_time, + max_jobs.saturating_sub(valid_jobs), + ) { + Ok(o) => o, + Err(e) => { + tracing::warn!("worker_loop_pull_jobs: failed to claim jobs: {e}"); + Vec::new() + } + }; + let valid_jobs = valid_jobs + claimed_jobs.len(); + let fetched_jobs = match fetch_redis_jobs( + &mut connection, + &sub_queue.name, + pod_uid, + max_jobs.saturating_sub(valid_jobs), + ) { + Ok(o) => o, + Err(e) => { + tracing::warn!("worker_loop_pull_jobs: failed to fetch jobs: {e}"); + Vec::new() + } + }; + let jobs: Vec<_> = fetched_jobs + .iter() + .chain(claimed_jobs.iter()) + .map(|it| { + (sub_queue.clone(), it.clone()) + .try_into() + .map_err(Error::JobParse) + }) + .collect::, Error>>()?; + let mut no_duplicated_jobs = HashMap::new(); + for job in &jobs { + no_duplicated_jobs.entry(&job.essential).or_insert(job); + } + Ok(Some( + no_duplicated_jobs + .into_iter() + .collect::>() + .into_iter() + .map(|it| it.1.clone()) + .collect(), + )) +} + +fn discover_sub_queue_for_jobs( + jobs: &[Job], + config: &Config, + redis_client: &Client, + pod_uid: &str, + sub_queues: &[SubQueue], + total_length: usize, +) -> Result, SubQueue)>, Error> { + for queue in sub_queues { + let Some(jobs) = pull_jobs(jobs, queue, config, redis_client, pod_uid, total_length)? + else { + continue; + }; + if jobs.is_empty() { + continue; + } + return Ok(Some((jobs, queue.clone()))); + } + Ok(None) +} + +fn valid_jobs(jobs: &[Job]) -> usize { + jobs.iter() + .filter(|it| !matches!(it.state, JobState::Cleaned)) + .count() +} diff --git a/cuscuta-worker/src/worker/pull/test.rs b/cuscuta-worker/src/worker/pull/test.rs new file mode 100644 index 0000000..76558b3 --- /dev/null +++ b/cuscuta-worker/src/worker/pull/test.rs @@ -0,0 +1,159 @@ +#![cfg(test)] + +use cuscuta_common::{ + data::Song, + db::{ + job::{Job, JobEssential, SubQueue}, + redis::{job_sub_queue_redis_key, sub_queue_postfix}, + }, +}; +use cuscuta_test::{ + mock::SimpleMockable, + mocks::mock::job_state, + redis::{ClientWrap, fake_enqueue}, + redis_client_image, +}; + +use crate::{ + data::Config, + worker::pull::{scan_sub_queue_and_pull_job, valid_jobs}, +}; + +#[tokio::test] +#[allow(clippy::cast_possible_truncation)] +async fn pull_job_test() { + // 算了还是写一点注释吧我怕以后看不懂了( •̀ ω •́ )✧ + let hash = "0000000"; + let friend_code = "123456789"; + let job_base_timestamp = 1_784_475_024; + let queue_timestamp = "1784475024"; + let worker_id = "mock_id"; + let mut current_jobs = Vec::new(); + let mut cursor = 0; + let config = Config::mock(); + let ClientWrap(client, _container) = redis_client_image!(); + let first_jobs_count = config.worker_max_jobs + 2; + + // 入队测试 + for v in 0..first_jobs_count { + fake_enqueue( + &client, + hash, + friend_code, + &(job_base_timestamp + v).to_string(), + queue_timestamp, + ) + .expect("failed to enqueue fake jobs"); + } + + // 空本地队列首次pull测试 + let sub_queue = scan_sub_queue_and_pull_job( + &client, + &mut current_jobs, + &mut cursor, + &config, + &Vec::::mock(), + worker_id, + ) + .await + .expect("failed to scan sub queue or pull jobs") + .expect("expected `Some(SubQueue)`"); + assert_eq!( + sub_queue, + SubQueue::try_from( + job_sub_queue_redis_key(&sub_queue_postfix(hash, queue_timestamp, 0, 10)).as_str() + ) + .expect("failed to parse sub queue, this should not happen") + ); + assert_eq!(current_jobs.len(), config.worker_max_jobs as usize); + let last_timestamp = (job_base_timestamp + config.worker_max_jobs - 1).to_string(); + let last_job = current_jobs + .iter() + .find(|it| it.essential.timestamp == last_timestamp.as_str()) + .expect("expected `Some(Job)`"); + assert_eq!( + last_job.essential, + JobEssential::new(friend_code.to_string(), last_timestamp, 0, 10, 0) + ); + assert_eq!(last_job.sub_queue, sub_queue); + + // 已有任务再次pull测试 + let append_timestamp = (job_base_timestamp + 100).to_string(); + current_jobs.truncate(1); + fake_enqueue( + &client, + hash, + friend_code, + &append_timestamp, + queue_timestamp, + ) + .expect("failed to enqueue fake jobs"); + let sub_queue = scan_sub_queue_and_pull_job( + &client, + &mut current_jobs, + &mut cursor, + &config, + &Vec::::mock(), + worker_id, + ) + .await + .expect("failed to scan sub queue or pull jobs") + .expect("expected `Some(SubQueue)`"); + assert_eq!( + sub_queue, + SubQueue::try_from( + job_sub_queue_redis_key(&sub_queue_postfix(hash, queue_timestamp, 0, 10)).as_str() + ) + .expect("failed to parse sub queue, this should not happen") + ); + assert_eq!( + current_jobs.len(), + (first_jobs_count - (config.worker_max_jobs - 1) + 1) as usize + ); + let append_job = current_jobs + .iter() + .find(|it| it.essential.timestamp == append_timestamp.as_str()) + .expect("expected `Some(Job)`"); + assert_eq!( + append_job.essential, + JobEssential::new(friend_code.to_string(), append_timestamp, 0, 10, 0) + ); + assert_eq!(append_job.sub_queue, sub_queue); +} + +#[test] +fn valid_jobs_count_test() { + assert_eq!( + valid_jobs(&[ + Job { + state: job_state::mock_cleaned(), + ..Job::mock() + }, + Job { + state: job_state::mock_cleaned(), + ..Job::mock() + }, + Job { + state: job_state::mock_cleaned(), + ..Job::mock() + }, + Job { + state: job_state::mock_failed(), + ..Job::mock() + }, + Job { + state: job_state::mock_finished(), + ..Job::mock() + }, + Job { + state: job_state::mock_pending(), + ..Job::mock() + }, + Job { + state: job_state::mock_pulled(), + ..Job::mock() + }, + ]), + 4 + ); +} diff --git a/helm/templates/worker-deployment.yaml b/helm/templates/worker-deployment.yaml index 50cf9fe..b512b88 100644 --- a/helm/templates/worker-deployment.yaml +++ b/helm/templates/worker-deployment.yaml @@ -79,6 +79,8 @@ spec: value: {{ .Values.worker.accountLeaseTimeRefreshGapSecs | quote }} - name: WORKER_JOB_MAX_WORK_TIME_SECS value: {{ .Values.worker.jobMaxWorkTimeSecs | quote }} + - name: WORKER_EMPTY_FRIENDS_DELAY_TIME_SECS + value: {{ .Values.worker.emptyFriendsDelayTimeSecs | quote }} - name: REDIS_STREAM_REFRESH_TTL value: {{ .Values.redisStreamRefreshTtl | quote }} - name: ETA_ENABLE diff --git a/helm/values.yaml b/helm/values.yaml index cbca28e..54852f9 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -124,6 +124,7 @@ worker: accountLeaseTimeSecs: 120 accountLeaseTimeRefreshGapSecs: 30 jobMaxWorkTimeSecs: 1200 + emptyFriendsDelayTimeSecs: 10 # -- cuscuta-mock (mock API server for testing; disable in production) mock: