From 82ac7336982533d76eecff765e6d782a023f11c9 Mon Sep 17 00:00:00 2001 From: FarnaHerry Date: Tue, 4 Aug 2026 21:31:45 +0800 Subject: [PATCH 1/4] fix: remove .xlings.json pinning mcpp 0.0.87 (breaks out-of-the-box build) --- .xlings.json | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .xlings.json diff --git a/.xlings.json b/.xlings.json deleted file mode 100644 index 81146db..0000000 --- a/.xlings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "workspace": { - "mcpp": "0.0.87" - } -} From f302873d77473a516e489e1b0ddc92920253e4c2 Mon Sep 17 00:00:00 2001 From: FarnaHerry Date: Tue, 4 Aug 2026 21:32:08 +0800 Subject: [PATCH 2/4] feat(http): segmented parallel download via HTTP Range; bump 0.3.0 aria2-style multi-connection downloads. download_to_file_parallel() probes with Range: bytes=0-0, then splits the file into segments fetched concurrently into a pre-allocated file by a worker pool pulling from a shared segment index: - maxConnectionsPerFile caps concurrent workers (-x) - maxSegments sets the split count (-s); 0 ties it to the connection count, preserving the pre-feature behavior - minSegmentBytes sets the smallest split worth making (--min-split-size) - interrupted segments resume mid-range on retry (2 retries each) - falls back to the sequential path when the server ignores Range, the resource is empty (416), or the file is too small to split - 206 responses are validated (Content-Range start/total, no chunked framing) so a misbehaving server cannot corrupt the output - progress callbacks stay monotonic across worker threads All existing API is unchanged; default config (maxConnectionsPerFile=1) behaves exactly like before. --- mcpp.lock | 2 +- mcpp.toml | 8 +- src/http.cppm | 569 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 576 insertions(+), 3 deletions(-) diff --git a/mcpp.lock b/mcpp.lock index 07de937..37d7543 100644 --- a/mcpp.lock +++ b/mcpp.lock @@ -5,5 +5,5 @@ version = 2 namespace = "compat" version = "3.6.1" source = "index+compat@3.6.1" -hash = "fnv1a:83dea67ac1379be3" +hash = "fnv1a:bb700e69e973ccd0" diff --git a/mcpp.toml b/mcpp.toml index ec01e93..089bfb2 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,8 +1,12 @@ +# Local fork of mcpplibs/tinyhttps 0.2.9 with segmented parallel downloads +# (HTTP Range multi-connection, aria2-style). All existing API is unchanged; +# the new surface is HttpClientConfig.maxConnectionsPerFile and +# HttpClient::download_to_file_parallel(). [package] namespace = "mcpplibs" name = "tinyhttps" -version = "0.2.9" -description = "Minimal C++23 HTTP/HTTPS client with SSE streaming support" +version = "0.3.0" +description = "Minimal C++23 HTTP/HTTPS client (fork: + Range segmented parallel download)" license = "Apache-2.0" repo = "https://github.com/mcpplibs/tinyhttps" diff --git a/src/http.cppm b/src/http.cppm index 67d700e..79cd41a 100644 --- a/src/http.cppm +++ b/src/http.cppm @@ -39,6 +39,10 @@ export struct HttpClientConfig { bool verifySsl { true }; bool keepAlive { true }; int maxRedirects { 10 }; // 0 = don't follow redirects + // Parallel download controls (aria2-style): + int maxConnectionsPerFile { 1 }; // concurrency cap: how many segment workers run at once (1 = sequential) + int maxSegments { 0 }; // max split count (-s). 0 = tie split count to maxConnectionsPerFile (legacy) + std::int64_t minSegmentBytes { 1 << 20 }; // minimum bytes per segment (--min-split-size) }; // Progress callback for streaming downloads: (totalBytes, downloadedBytes) @@ -269,6 +273,56 @@ static bool iequals(std::string_view a, std::string_view b) { return true; } +// Parse the numeric status code from a status line ("HTTP/1.1 206 Partial Content"). +// Returns 0 on a malformed line. +static int parse_status_code(std::string_view statusLine) { + auto sp = statusLine.find(' '); + if (sp == std::string_view::npos) return 0; + int code = 0; + for (char c : statusLine.substr(sp + 1)) { + if (c < '0' || c > '9') break; + code = code * 10 + (c - '0'); + } + return code; +} + +// Parse the total size from a Content-Range header, e.g. "bytes 0-0/10485760". +// Returns nullopt when the total is absent or unknown ("bytes 0-0/*"). +static std::optional parse_content_range_total(std::string_view value) { + auto slash = value.rfind('/'); + if (slash == std::string_view::npos) return std::nullopt; + std::string_view totalStr = value.substr(slash + 1); + if (totalStr == "*") return std::nullopt; + std::int64_t total = 0; + for (char c : totalStr) { + if (c < '0' || c > '9') return std::nullopt; + if (total > (std::numeric_limits::max() - (c - '0')) / 10) { + return std::nullopt; // overflow + } + total = total * 10 + (c - '0'); + } + return total; +} + +// Parse the range start from a Content-Range header, e.g. "bytes 1024-2047/4096" +// yields 1024. Returns nullopt for non-bytes units or a malformed value. +static std::optional parse_content_range_start(std::string_view value) { + auto space = value.find(' '); + if (space == std::string_view::npos) return std::nullopt; + if (!iequals(value.substr(0, space), "bytes")) return std::nullopt; + auto dash = value.find('-', space + 1); + if (dash == std::string_view::npos) return std::nullopt; + std::int64_t start = 0; + for (char c : value.substr(space + 1, dash - space - 1)) { + if (c < '0' || c > '9') return std::nullopt; + if (start > (std::numeric_limits::max() - (c - '0')) / 10) { + return std::nullopt; // overflow + } + start = start * 10 + (c - '0'); + } + return start; +} + export class HttpClient { public: // Thread-safety: HttpClient owns a mutable connection pool and is not synchronized. @@ -813,6 +867,23 @@ public: std::move(isCancelled), 0); } + // Parallel segmented download using HTTP Range (aria2-style). Probes the + // server for Range support, then fetches non-overlapping segments into a + // pre-allocated file: split count is maxSegments (or ceil(size / + // minSegmentBytes)), and concurrency is capped by maxConnectionsPerFile. + // Falls back to download_to_file() when the server ignores Range or the + // file is too small to shard. Same progress/cancel semantics as + // download_to_file(). + DownloadToFileResult download_to_file_parallel( + const std::string& url, + const std::filesystem::path& destFile, + DownloadProgressFn onProgress = nullptr, + std::function isCancelled = nullptr) + { + return download_to_file_parallel_impl(url, destFile, std::move(onProgress), + std::move(isCancelled), 0); + } + HttpClientConfig& config() { return config_; } const HttpClientConfig& config() const { return config_; } @@ -1114,6 +1185,504 @@ private: return result; } + // Establish a fresh TLS connection to `parsed` (proxy-aware). Leaves `sock` + // invalid on failure. Used by parallel segment workers — each worker gets + // its own connection and never touches the shared pool_. + bool connect_fresh(const ParsedUrl& parsed, TlsSocket& sock) { + bool connected = false; + if (config_.proxy.has_value()) { + auto proxyConf = parse_proxy_url(config_.proxy.value()); + auto tunnel = proxy_connect(proxyConf.host, proxyConf.port, + parsed.host, parsed.port, + config_.connectTimeoutMs); + if (tunnel.is_valid()) { + connected = sock.connect_over(std::move(tunnel), + parsed.host.c_str(), + config_.verifySsl); + } + } else { + connected = sock.connect(parsed.host.c_str(), parsed.port, + config_.connectTimeoutMs, config_.verifySsl); + } + return connected; + } + + DownloadToFileResult download_to_file_parallel_impl( + const std::string& url, + const std::filesystem::path& destFile, + DownloadProgressFn onProgress, + std::function isCancelled, + int redirectCount) + { + DownloadToFileResult result; + + // Parallelism disabled — identical behavior to the sequential path. + if (config_.maxConnectionsPerFile <= 1) { + return download_to_file_impl(url, destFile, std::move(onProgress), + std::move(isCancelled), redirectCount); + } + + auto parsed = parse_url(url); + if (parsed.scheme != "https") { + result.error = "Only HTTPS is supported"; + return result; + } + + // Probe: can the server honor Range? + TlsSocket probeSock; + if (!connect_fresh(parsed, probeSock)) { + result.error = "Connection failed"; + return result; + } + + std::string probeReq = "GET " + parsed.path + " HTTP/1.1\r\nHost: " + parsed.host; + if (parsed.port != 443) probeReq += ":" + std::to_string(parsed.port); + probeReq += "\r\nUser-Agent: tinyhttps/1.0\r\nAccept: */*\r\nRange: bytes=0-0\r\nConnection: close\r\n\r\n"; + + if (!write_all(probeSock, probeReq)) { + result.error = "Write failed"; + return result; + } + + std::string statusLine = read_line(probeSock, config_.readTimeoutMs); + if (statusLine.empty()) { + result.error = "No response"; + return result; + } + int statusCode = parse_status_code(statusLine); + + std::string location; + std::string contentRange; + std::string etag; + std::string lastModified; + bool chunked = false; + std::int64_t contentLength = -1; + while (true) { + std::string line = read_line(probeSock, config_.readTimeoutMs); + if (line.empty()) break; + auto colon = line.find(':'); + if (colon == std::string::npos) continue; + std::string key = line.substr(0, colon); + std::string_view val = std::string_view(line).substr(colon + 1); + while (!val.empty() && val[0] == ' ') val = val.substr(1); + std::string valStr(val); + if (iequals(key, "Location")) location = valStr; + if (iequals(key, "Content-Range")) contentRange = valStr; + if (iequals(key, "ETag")) etag = valStr; + if (iequals(key, "Last-Modified")) lastModified = valStr; + if (iequals(key, "Transfer-Encoding") && iequals(valStr, "chunked")) chunked = true; + if (iequals(key, "Content-Length")) { + contentLength = 0; + for (char c : valStr) { + if (c >= '0' && c <= '9') contentLength = contentLength * 10 + (c - '0'); + } + } + } + + // Follow redirects up front so every segment worker targets the final + // URL directly instead of following the chain itself. + if (statusCode >= 300 && statusCode < 400 && + !location.empty() && redirectCount < config_.maxRedirects) { + probeSock.close(); + if (location.starts_with("/")) { + location = parsed.scheme + "://" + parsed.host + + (parsed.port != 443 ? ":" + std::to_string(parsed.port) : "") + + location; + } + return download_to_file_parallel_impl(location, destFile, + std::move(onProgress), + std::move(isCancelled), + redirectCount + 1); + } + + // 200 — server ignored Range; the probe response body is the whole + // file, so write it out directly (no second round-trip). + if (statusCode == 200) { + std::error_code ec; + std::filesystem::create_directories(destFile.parent_path(), ec); + std::ofstream ofs(destFile, std::ios::binary); + if (!ofs) { + result.error = "Cannot open file: " + destFile.string(); + return result; + } + std::int64_t written = 0; + auto cancelled = [&]() -> bool { + if (isCancelled && isCancelled()) { + result.error = "cancelled"; + result.bytesWritten = written; + return true; + } + return false; + }; + if (chunked) { + while (true) { + if (cancelled()) return result; + std::string sizeLine = read_line(probeSock, config_.readTimeoutMs); + auto semi = sizeLine.find(';'); + if (semi != std::string::npos) sizeLine = sizeLine.substr(0, semi); + while (!sizeLine.empty() && (sizeLine.back() == ' ' || sizeLine.back() == '\t')) + sizeLine.pop_back(); + int chunkSize = parse_hex(sizeLine); + if (chunkSize == 0) break; + char buf[8192]; + int remaining = chunkSize; + while (remaining > 0) { + if (cancelled()) return result; + int toRead = remaining > static_cast(sizeof(buf)) + ? static_cast(sizeof(buf)) : remaining; + if (!read_exact(probeSock, buf, toRead, config_.readTimeoutMs)) { + result.error = "Read error during fallback download"; + result.bytesWritten = written; + return result; + } + ofs.write(buf, toRead); + written += toRead; + remaining -= toRead; + if (onProgress) onProgress(0, written); + } + read_line(probeSock, config_.readTimeoutMs); // trailing CRLF + } + } else if (contentLength >= 0) { + char buf[8192]; + std::int64_t remaining = contentLength; + while (remaining > 0) { + if (cancelled()) return result; + int toRead = remaining > static_cast(sizeof(buf)) + ? static_cast(sizeof(buf)) + : static_cast(remaining); + if (!read_exact(probeSock, buf, toRead, config_.readTimeoutMs)) { + result.error = "Read error during fallback download"; + result.bytesWritten = written; + return result; + } + ofs.write(buf, toRead); + written += toRead; + remaining -= toRead; + if (onProgress) onProgress(contentLength, written); + } + } else { + char buf[8192]; + while (true) { + if (cancelled()) return result; + if (!probeSock.wait_readable(config_.readTimeoutMs)) break; + int ret = probeSock.read(buf, sizeof(buf)); + if (ret <= 0) break; + ofs.write(buf, ret); + written += ret; + if (onProgress) onProgress(0, written); + } + } + ofs.close(); + result.statusCode = 200; + result.finalUrl = url; + result.etag = std::move(etag); + result.lastModified = std::move(lastModified); + if (contentLength >= 0) result.expectedBytes = contentLength; + result.bytesWritten = written; + return result; + } + + probeSock.close(); + + // 416 — resource is empty; write a zero-byte file and succeed. + if (statusCode == 416) { + std::error_code ec; + std::filesystem::create_directories(destFile.parent_path(), ec); + { + std::ofstream ofs(destFile, std::ios::binary); + if (!ofs) { + result.error = "Cannot create file: " + destFile.string(); + return result; + } + } + result.statusCode = 200; + result.finalUrl = url; + result.expectedBytes = 0; + result.bytesWritten = 0; + return result; + } + + if (statusCode != 206) { + result.error = "HTTP " + std::to_string(statusCode); + return result; + } + + auto totalOpt = parse_content_range_total(contentRange); + if (!totalOpt || *totalOpt == 0) { + // Total unknown or empty — sequential path. + return download_to_file_impl(url, destFile, std::move(onProgress), + std::move(isCancelled), redirectCount); + } + std::int64_t totalBytes = *totalOpt; + + // Segment. Split into ceil(total / minSegmentBytes) pieces, capped by + // maxSegments when set; with maxSegments == 0 (legacy default) the + // split count is tied to the connection count. + constexpr int MAX_SEGMENTS = 2048; // hard cap against pathological configs + std::int64_t minSeg = config_.minSegmentBytes > 0 + ? config_.minSegmentBytes : (1 << 20); + int nSegs = static_cast((totalBytes + minSeg - 1) / minSeg); + if (config_.maxSegments > 0) { + nSegs = std::min(nSegs, config_.maxSegments); + } else { + nSegs = std::min(nSegs, config_.maxConnectionsPerFile); + } + nSegs = std::min(nSegs, MAX_SEGMENTS); + if (nSegs <= 1) { + // Too small to split — sequential path. + return download_to_file_impl(url, destFile, std::move(onProgress), + std::move(isCancelled), redirectCount); + } + const int nWorkers = std::min(config_.maxConnectionsPerFile, nSegs); + + // Pre-allocate the file so each segment can write into place. + std::error_code ec; + std::filesystem::create_directories(destFile.parent_path(), ec); + std::ofstream create(destFile, std::ios::binary); + if (!create) { + result.error = "Cannot create file: " + destFile.string(); + return result; + } + create.close(); + std::error_code resizeEc; + std::filesystem::resize_file(destFile, totalBytes, resizeEc); + if (resizeEc) { + result.error = "Cannot resize file: " + destFile.string(); + return result; + } + + // Non-overlapping segment boundaries. + std::vector> segments; + segments.reserve(nSegs); + std::int64_t segSize = (totalBytes + nSegs - 1) / nSegs; + for (int i = 0; i < nSegs; ++i) { + std::int64_t s = i * segSize; + std::int64_t e = std::min(s + segSize - 1, totalBytes - 1); + if (s > e) break; + segments.emplace_back(s, e); + } + + // Shared worker state. + std::atomic globalWritten{0}; + std::atomic userCancelled{false}; + std::atomic anyFailed{false}; + std::atomic nextSegment{0}; + std::mutex stateMutex; // serializes onProgress + firstError + std::string firstError; + std::int64_t reportedMax = 0; // guarded by stateMutex: keeps onProgress monotonic + + auto fail = [&](std::string msg) { + std::lock_guard lock(stateMutex); + if (firstError.empty()) firstError = std::move(msg); + anyFailed.store(true); + }; + + // Workers pull segments from a shared index until exhausted, so + // maxSegments can exceed maxConnectionsPerFile (aria2 -s vs -x). + auto worker = [&]() { + // Open once per worker; Windows CRT sharing allows concurrent + // in-place writes at disjoint offsets. + std::fstream file(destFile, std::ios::binary | std::ios::in | std::ios::out); + if (!file) { + fail("Cannot open file: " + destFile.string()); + return; + } + + constexpr int MAX_RETRIES = 2; + const int totalSegs = static_cast(segments.size()); + + for (;;) { + if (userCancelled.load() || anyFailed.load()) return; + const int idx = nextSegment.fetch_add(1); + if (idx >= totalSegs) return; // queue exhausted + + const std::int64_t segStart = segments[idx].first; + const std::int64_t segEnd = segments[idx].second; + const std::int64_t segLen = segEnd - segStart + 1; + std::int64_t segWritten = 0; + + for (int attempt = 0; attempt <= MAX_RETRIES; ++attempt) { + if (userCancelled.load() || anyFailed.load()) return; + + TlsSocket sock; + if (!connect_fresh(parsed, sock)) { + if (attempt == MAX_RETRIES) { + fail("Connection failed for segment " + + std::to_string(segStart) + "-" + std::to_string(segEnd)); + return; + } + continue; + } + + // Request only the not-yet-downloaded remainder (natural resume). + std::int64_t chunkStart = segStart + segWritten; + std::string req = "GET " + parsed.path + " HTTP/1.1\r\nHost: " + parsed.host; + if (parsed.port != 443) req += ":" + std::to_string(parsed.port); + req += "\r\nUser-Agent: tinyhttps/1.0\r\nAccept: */*\r\n" + "Range: bytes=" + std::to_string(chunkStart) + "-" + + std::to_string(segEnd) + + "\r\nConnection: close\r\n\r\n"; + + if (!write_all(sock, req)) { + sock.close(); + if (attempt == MAX_RETRIES) { + fail("Write failed for segment " + std::to_string(segStart) + + "-" + std::to_string(segEnd)); + return; + } + continue; + } + + std::string statusLine = read_line(sock, config_.readTimeoutMs); + int statusCode = parse_status_code(statusLine); + if (statusCode != 206) { + // Retry; a persistent non-206 is a segment failure. + sock.close(); + if (attempt == MAX_RETRIES) { + fail("Unexpected status " + std::to_string(statusCode) + + " for segment " + std::to_string(segStart) + "-" + + std::to_string(segEnd)); + return; + } + continue; + } + + // Read response headers, keeping Content-Range for validation. + bool segChunked = false; + std::string segContentRange; + while (true) { + std::string line = read_line(sock, config_.readTimeoutMs); + if (line.empty()) break; + auto colon = line.find(':'); + if (colon == std::string::npos) continue; + std::string key = line.substr(0, colon); + std::string_view val = std::string_view(line).substr(colon + 1); + while (!val.empty() && val[0] == ' ') val = val.substr(1); + std::string valStr(val); + if (iequals(key, "Transfer-Encoding") && iequals(valStr, "chunked")) + segChunked = true; + if (iequals(key, "Content-Range")) + segContentRange = valStr; + } + + // A chunked 206 would write raw framing bytes into the + // pre-allocated file, and a mismatched Content-Range would + // write the wrong bytes — neither is retryable, so fail. + bool rangeOk = !segChunked; + if (rangeOk && !segContentRange.empty()) { + auto rangeStart = parse_content_range_start(segContentRange); + rangeOk = rangeStart && *rangeStart == chunkStart && + parse_content_range_total(segContentRange) == totalBytes; + } + if (!rangeOk) { + sock.close(); + fail(segChunked + ? "Chunked 206 response for segment " + + std::to_string(segStart) + "-" + std::to_string(segEnd) + : "Mismatched Content-Range for segment " + + std::to_string(segStart) + "-" + std::to_string(segEnd)); + return; + } + + file.seekp(segStart + segWritten); + + bool ok = true; + std::int64_t remaining = segLen - segWritten; + char buf[8192]; + while (remaining > 0) { + if (userCancelled.load() || anyFailed.load()) { + ok = false; + break; + } + if (isCancelled && isCancelled()) { + userCancelled.store(true); + ok = false; + break; + } + int toRead = remaining > static_cast(sizeof(buf)) + ? static_cast(sizeof(buf)) : static_cast(remaining); + if (!read_exact(sock, buf, toRead, config_.readTimeoutMs)) { + ok = false; + break; // connection dropped — retry the remaining range + } + file.write(buf, toRead); + if (!file) { + ok = false; + break; + } + segWritten += toRead; + remaining -= toRead; + std::int64_t g = globalWritten.fetch_add(toRead) + toRead; + std::lock_guard lock(stateMutex); + if (onProgress && g > reportedMax) { + reportedMax = g; + onProgress(totalBytes, g); + } + } + + sock.close(); + if (!ok) { + if (userCancelled.load() || anyFailed.load()) return; + if (attempt == MAX_RETRIES) { + fail("Read error for segment " + std::to_string(segStart) + + "-" + std::to_string(segEnd)); + return; + } + continue; // retry the remainder of this segment + } + break; // this segment done — grab the next one + } + } + }; + + // Exceptions escaping a thread call std::terminate — surface them as + // download failures instead (e.g. a throwing onProgress callback). + auto workerGuarded = [&]() { + try { + worker(); + } catch (const std::exception& e) { + fail(std::string("Worker exception: ") + e.what()); + } catch (...) { + fail("Worker exception"); + } + }; + + std::vector threads; + threads.reserve(nWorkers); + for (int i = 0; i < nWorkers; ++i) { + threads.emplace_back(workerGuarded); + } + for (auto& t : threads) t.join(); + + if (userCancelled.load()) { + result.statusCode = 0; + result.error = "cancelled"; + result.bytesWritten = globalWritten.load(); + return result; + } + if (anyFailed.load()) { + std::string err; + { + std::lock_guard lock(stateMutex); + err = firstError; + } + std::error_code rmEc; + std::filesystem::remove(destFile, rmEc); + result.statusCode = 0; + result.error = err.empty() ? "Download failed" : err; + result.bytesWritten = globalWritten.load(); + return result; + } + + result.statusCode = 206; + result.finalUrl = url; + result.etag = std::move(etag); + result.lastModified = std::move(lastModified); + result.expectedBytes = totalBytes; + result.bytesWritten = totalBytes; + return result; + } + HttpClientConfig config_; std::map pool_; }; From 8e689093f28066c04594fb27b008f49816f90248 Mon Sep 17 00:00:00 2001 From: FarnaHerry Date: Tue, 4 Aug 2026 21:32:31 +0800 Subject: [PATCH 3/4] test(download): cover parallel downloads; migrate endpoints to httpbingo.org New ParallelDownloadTest suite (7 cases) against httpbingo.org, which honors Range on /range/N and serves deterministic content: - segmented result byte-identical to sequential download - maxSegments decoupled from connection count - monotonic parallel progress - fallback when the server ignores Range (200) - small-file fallback to the sequential path - redirect resolution before segmenting - cancellation aborts all workers Also migrate the existing httpbin.org tests to httpbingo.org (same API, more reliable); httpbin.org frequently serves 503s. --- .gitignore | 1 + tests/test_download.cpp | 198 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 190 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 1a3902c..787b291 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ build/ .xmake/ # mcpp build artefacts target/ +compile_commands.json diff --git a/tests/test_download.cpp b/tests/test_download.cpp index c862fbe..a7ad1e6 100644 --- a/tests/test_download.cpp +++ b/tests/test_download.cpp @@ -37,7 +37,7 @@ TEST(DownloadResultContract, CarriesTransferAndResponseMetadata) { } // Test download_to_file against a real HTTPS endpoint. -// Uses httpbin.org which returns known-size responses. +// Uses httpbingo.org (a maintained httpbin work-alike) which returns known-size responses. class DownloadToFileTest : public ::testing::Test { protected: @@ -67,7 +67,7 @@ TEST_F(DownloadToFileTest, BasicDownloadWithProgress) { int callCount = 0; auto result = client.download_to_file( - "https://httpbin.org/bytes/100", + "https://httpbingo.org/bytes/100", dest, [&](std::int64_t total, std::int64_t downloaded) { lastTotal = total; @@ -97,7 +97,7 @@ TEST_F(DownloadToFileTest, ProgressIncrementsMonotonically) { std::vector downloadedValues; auto result = client.download_to_file( - "https://httpbin.org/bytes/51200", + "https://httpbingo.org/bytes/51200", dest, [&](std::int64_t total, std::int64_t downloaded) { (void)total; @@ -128,9 +128,9 @@ TEST_F(DownloadToFileTest, FollowsRedirects) { auto dest = tmpDir / "redirected.bin"; - // httpbin /redirect-to redirects to the given URL + // httpbingo /redirect-to redirects to the given URL auto result = client.download_to_file( - "https://httpbin.org/redirect-to?url=https%3A%2F%2Fhttpbin.org%2Fbytes%2F50", + "https://httpbingo.org/redirect-to?url=https%3A%2F%2Fhttpbingo.org%2Fbytes%2F50", dest ); @@ -149,7 +149,7 @@ TEST_F(DownloadToFileTest, NoProgressCallbackStillWorks) { auto dest = tmpDir / "no_progress.bin"; auto result = client.download_to_file( - "https://httpbin.org/bytes/200", + "https://httpbingo.org/bytes/200", dest ); @@ -167,7 +167,7 @@ TEST_F(DownloadToFileTest, Http404ReturnsError) { auto dest = tmpDir / "not_found.bin"; auto result = client.download_to_file( - "https://httpbin.org/status/404", + "https://httpbingo.org/status/404", dest ); @@ -186,7 +186,7 @@ TEST_F(DownloadToFileTest, TotalBytesKnownForContentLength) { std::int64_t reportedTotal = -1; auto result = client.download_to_file( - "https://httpbin.org/bytes/1024", + "https://httpbingo.org/bytes/1024", dest, [&](std::int64_t total, [[maybe_unused]] std::int64_t downloaded) { if (reportedTotal < 0) reportedTotal = total; @@ -194,6 +194,186 @@ TEST_F(DownloadToFileTest, TotalBytesKnownForContentLength) { ); ASSERT_TRUE(result.ok()) << "Error: " << result.error; - // httpbin /bytes/N returns Content-Length: N + // httpbingo /bytes/N returns Content-Length: N EXPECT_EQ(reportedTotal, 1024); } + +// Test download_to_file_parallel against endpoints with known Range behavior. +// httpbingo.org /range/N honors Range (206) with deterministic content; +// /bytes/N ignores Range (200), which exercises the fallback path. + +class ParallelDownloadTest : public ::testing::Test { +protected: + std::filesystem::path tmpDir; + + void SetUp() override { + https::Socket::platform_init(); + tmpDir = std::filesystem::temp_directory_path() / "tinyhttps_parallel_test"; + std::filesystem::create_directories(tmpDir); + } + void TearDown() override { + std::error_code ec; + std::filesystem::remove_all(tmpDir, ec); + } +}; + +TEST_F(ParallelDownloadTest, SegmentedResultMatchesSequential) { + const std::string url = "https://httpbingo.org/range/65536"; + + https::HttpClient seqClient({}); + auto seqDest = tmpDir / "seq.bin"; + auto seq = seqClient.download_to_file(url, seqDest); + ASSERT_TRUE(seq.ok()) << "Sequential error: " << seq.error; + + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 4; + cfg.minSegmentBytes = 1024; // small so test files actually split + https::HttpClient parClient(cfg); + + auto parDest = tmpDir / "par.bin"; + auto par = parClient.download_to_file_parallel(url, parDest); + ASSERT_TRUE(par.ok()) << "Parallel error: " << par.error; + + EXPECT_EQ(par.bytesWritten, 65536); + ASSERT_TRUE(par.expectedBytes.has_value()); + EXPECT_EQ(*par.expectedBytes, 65536); + EXPECT_EQ(std::filesystem::file_size(parDest), 65536u); + + std::ifstream seqFile(seqDest, std::ios::binary); + std::ifstream parFile(parDest, std::ios::binary); + std::string seqContent{ std::istreambuf_iterator(seqFile), + std::istreambuf_iterator() }; + std::string parContent{ std::istreambuf_iterator(parFile), + std::istreambuf_iterator() }; + EXPECT_EQ(parContent, seqContent) + << "Segmented download content differs from sequential"; +} + +TEST_F(ParallelDownloadTest, MoreSegmentsThanConnections) { + // maxSegments decoupled from connection count (aria2 -s): 16 segments + // pulled by only 2 workers. + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 2; + cfg.maxSegments = 16; + cfg.minSegmentBytes = 1024; + https::HttpClient client(cfg); + + auto dest = tmpDir / "decoupled.bin"; + auto result = client.download_to_file_parallel( + "https://httpbingo.org/range/32768", dest); + + ASSERT_TRUE(result.ok()) << "Error: " << result.error; + EXPECT_EQ(result.bytesWritten, 32768); + EXPECT_EQ(std::filesystem::file_size(dest), 32768u); +} + +TEST_F(ParallelDownloadTest, ProgressIsMonotonic) { + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 4; + cfg.minSegmentBytes = 1024; + https::HttpClient client(cfg); + + auto dest = tmpDir / "monotonic.bin"; + std::vector values; + std::int64_t reportedTotal = -1; + + auto result = client.download_to_file_parallel( + "https://httpbingo.org/range/131072", + dest, + [&](std::int64_t total, std::int64_t downloaded) { + reportedTotal = total; + values.push_back(downloaded); + } + ); + + ASSERT_TRUE(result.ok()) << "Error: " << result.error; + EXPECT_EQ(reportedTotal, 131072); + ASSERT_FALSE(values.empty()); + EXPECT_EQ(values.back(), 131072); + for (std::size_t i = 1; i < values.size(); ++i) { + EXPECT_GT(values[i], values[i - 1]) + << "Parallel progress not monotonic at index " << i; + } +} + +TEST_F(ParallelDownloadTest, FallsBackWhenServerIgnoresRange) { + // /bytes/N does not honor Range — probe gets 200 and the body is + // streamed out over the single probe connection. + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 4; + cfg.minSegmentBytes = 1024; + https::HttpClient client(cfg); + + auto dest = tmpDir / "fallback.bin"; + auto result = client.download_to_file_parallel( + "https://httpbingo.org/bytes/4096", dest); + + ASSERT_TRUE(result.ok()) << "Error: " << result.error; + EXPECT_EQ(result.statusCode, 200); + EXPECT_EQ(result.bytesWritten, 4096); + EXPECT_EQ(std::filesystem::file_size(dest), 4096u); +} + +TEST_F(ParallelDownloadTest, SmallFileFallsBackToSequential) { + // File smaller than minSegmentBytes — not worth splitting. + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 4; + cfg.minSegmentBytes = 1 << 20; + https::HttpClient client(cfg); + + auto dest = tmpDir / "small.bin"; + auto result = client.download_to_file_parallel( + "https://httpbingo.org/range/2048", dest); + + ASSERT_TRUE(result.ok()) << "Error: " << result.error; + EXPECT_EQ(result.statusCode, 200); + EXPECT_EQ(result.bytesWritten, 2048); +} + +TEST_F(ParallelDownloadTest, FollowsRedirectBeforeSegmenting) { + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 4; + cfg.minSegmentBytes = 1024; + https::HttpClient client(cfg); + + auto dest = tmpDir / "redirected.bin"; + auto result = client.download_to_file_parallel( + "https://httpbingo.org/redirect-to?url=https%3A%2F%2Fhttpbingo.org%2Frange%2F8192", + dest); + + ASSERT_TRUE(result.ok()) << "Error: " << result.error; + EXPECT_EQ(result.bytesWritten, 8192); + EXPECT_EQ(result.finalUrl, "https://httpbingo.org/range/8192"); +} + +TEST_F(ParallelDownloadTest, CancellationAbortsWorkers) { + https::HttpClientConfig cfg; + cfg.connectTimeoutMs = 15000; + cfg.readTimeoutMs = 30000; + cfg.maxConnectionsPerFile = 4; + cfg.minSegmentBytes = 1024; + https::HttpClient client(cfg); + + auto dest = tmpDir / "cancelled.bin"; + auto result = client.download_to_file_parallel( + "https://httpbingo.org/range/524288", // httpbingo /range caps at 512KiB + dest, + nullptr, + [] { return true; } // cancel immediately + ); + + EXPECT_FALSE(result.ok()); + EXPECT_EQ(result.error, "cancelled"); +} From eb0fc468363ed14a776dd69b715b7c76ddd71e8f Mon Sep 17 00:00:00 2001 From: FarnaHerry Date: Tue, 4 Aug 2026 21:32:31 +0800 Subject: [PATCH 4/4] docs(readme): document segmented parallel download API --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index a4d0481..fcafe4c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Minimal C++23 HTTP/HTTPS client library with SSE (Server-Sent Events) streaming - HTTP/HTTPS client with connection pooling (keep-alive) - SSE (Server-Sent Events) streaming - Proxy support (HTTP CONNECT) +- Segmented parallel downloads via HTTP Range (aria2-style) - C++23 modules ## Usage @@ -28,6 +29,41 @@ auto resp = client.send(mcpplibs::tinyhttps::HttpRequest::post( )); ``` +## Parallel downloads + +`download_to_file_parallel()` probes the server with `Range: bytes=0-0`; when +the server answers 206 the file is split into segments fetched concurrently +into a pre-allocated file. It falls back to a plain sequential download when +the server ignores Range (200) or the file is too small to split. + +```cpp +import mcpplibs.tinyhttps; +namespace https = mcpplibs::tinyhttps; + +https::HttpClientConfig cfg; +cfg.maxConnectionsPerFile = 8; // concurrent segment workers +cfg.maxSegments = 16; // aria2 -s: split count (0 = tie to connections) +cfg.minSegmentBytes = 4 << 20; // aria2 --min-split-size: 4 MiB + +https::HttpClient client(cfg); +auto result = client.download_to_file_parallel( + "https://example.com/big.iso", + "big.iso", + [](std::int64_t total, std::int64_t done) { + // monotonic progress; total is 0 when unknown + }, + [] { return false; } // return true to cancel +); +if (result.ok()) { /* result.bytesWritten, result.expectedBytes, ... */ } +``` + +Behavior notes: + +- Segment boundaries never overlap; interrupted segments resume mid-range on + retry (up to 2 retries per segment). +- Progress callbacks are serialized and monotonically increasing. +- The pre-allocated target file is removed if the download fails. + ## 使用 mcpp 构建 ### 添加依赖