From e5474968d45243853a67c6cadb23dd70eabade92 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Mon, 24 Aug 2026 19:04:10 -0300 Subject: [PATCH 1/7] [WIP] Implement a metrics Prometheus endpoint Signed-off-by: Juan Cruz Viotti --- docs/api.md | 40 ++ enterprise/e2e/html/compose.yml | 16 + enterprise/e2e/html/hurl/metrics.all.hurl | 79 ++++ .../e2e/html/hurl/prometheus.compose.hurl | 130 +++++++ enterprise/e2e/html/prometheus.yml | 13 + .../sourcemeta/one/enterprise_server.h | 1 + .../one/enterprise_server_action_metrics_v1.h | 349 ++++++++++++++++++ src/actions/CMakeLists.txt | 1 + src/actions/action_metrics_v1.h | 84 +++++ src/actions/actions.cc | 1 + src/actions/include/sourcemeta/one/actions.h | 11 + .../include/sourcemeta/one/http_helpers.h | 22 +- .../include/sourcemeta/one/http_request.h | 35 +- src/index/endpoints.h | 1 + src/index/generators.h | 5 + src/router/CMakeLists.txt | 4 +- src/router/include/sourcemeta/one/router.h | 6 + .../include/sourcemeta/one/router_metrics.h | 101 +++++ src/router/metrics.cc | 92 +++++ src/router/router.cc | 3 + src/server/server.cc | 13 + test/e2e/common.mk | 13 +- test/e2e/html/hurl/metrics.community.hurl | 81 ++++ 23 files changed, 1096 insertions(+), 5 deletions(-) create mode 100644 enterprise/e2e/html/hurl/metrics.all.hurl create mode 100644 enterprise/e2e/html/hurl/prometheus.compose.hurl create mode 100644 enterprise/e2e/html/prometheus.yml create mode 100644 enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h create mode 100644 src/actions/action_metrics_v1.h create mode 100644 src/router/include/sourcemeta/one/router_metrics.h create mode 100644 src/router/metrics.cc create mode 100644 test/e2e/html/hurl/metrics.community.hurl diff --git a/docs/api.md b/docs/api.md index b823fd002..72db674a9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -51,6 +51,46 @@ GET /self/v1/health The HTTP method is not `GET` or `HEAD`. +### Metrics + +!!! success "Enterprise" + + This endpoint is only available in the [Enterprise](commercial.md) + edition. Learn more about [commercial licensing](commercial.md). + +*This endpoint reports instance telemetry in the [Prometheus exposition +format](https://prometheus.io/docs/instrumenting/exposition_formats/).* + +``` +GET /self/v1/metrics +``` + +Any Prometheus-compatible scraper reads this without configuration beyond the +path. Request behaviour follows the +[RED](https://grafana.com/blog/2018/08/02/the-red-method-how-to-instrument-your-services/) +method, meaning rate, errors and duration, reported per action and status code. +Alongside it are the standard `process_*` metrics and an instance information +metric. The response documents itself through its `# HELP` and `# TYPE` lines. +This endpoint is not exempt from [authentication](#authentication), so a policy +covering its path gates it like any other route. + +Metric and label names are part of the public interface, as alerting rules and +dashboards are written against them. New metrics may appear in a minor release, +while renaming or removing one, changing its type or unit, or changing +histogram bucket boundaries are breaking changes. + +=== "200" + + The metrics in the Prometheus exposition format. + +=== "403" + + The instance is running the Community edition. + +=== "405" + + The HTTP method is not `GET` or `HEAD`. + ### List *This endpoint lists the contents of a directory at the specified `{path}` diff --git a/enterprise/e2e/html/compose.yml b/enterprise/e2e/html/compose.yml index 714999012..cd8628040 100644 --- a/enterprise/e2e/html/compose.yml +++ b/enterprise/e2e/html/compose.yml @@ -9,3 +9,19 @@ services: - SOURCEMETA_ONE_PORT=8001 ports: - "${PORT}:8001" + + # A real scraper rather than a reader of our own making, so what this + # instance publishes is judged by the software it claims to speak to + prometheus: + image: prom/prometheus:v3.1.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=1h + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9099:9090" + depends_on: + sandbox: + condition: service_started diff --git a/enterprise/e2e/html/hurl/metrics.all.hurl b/enterprise/e2e/html/hurl/metrics.all.hurl new file mode 100644 index 000000000..4d7966030 --- /dev/null +++ b/enterprise/e2e/html/hurl/metrics.all.hurl @@ -0,0 +1,79 @@ +# A scrape reports on the requests that came before it, so this one gives it +# something to have counted +GET {{base}}/self/v1/health +HTTP 200 + +GET {{base}}/self/v1/metrics +HTTP 200 +Cache-Control: no-store +Content-Type: text/plain; version=0.0.4; charset=utf-8 +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ +body matches /# TYPE sourcemeta_one_build_info gauge\nsourcemeta_one_build_info\{version="[^"]+",edition="enterprise"\} 1\n/ +body matches /# TYPE process_start_time_seconds gauge\nprocess_start_time_seconds [0-9.]+\n/ +body matches /# TYPE process_cpu_seconds_total counter\nprocess_cpu_seconds_total [0-9.e-]+\n/ +body matches /# TYPE process_resident_memory_bytes gauge\nprocess_resident_memory_bytes [0-9]+\n/ +body matches /# TYPE process_virtual_memory_bytes gauge\nprocess_virtual_memory_bytes [0-9]+\n/ +body matches /# TYPE process_open_fds gauge\nprocess_open_fds [0-9]+\n/ +body matches /# TYPE process_max_fds gauge\nprocess_max_fds [0-9]+\n/ +body matches /# TYPE sourcemeta_one_http_requests_in_flight gauge\nsourcemeta_one_http_requests_in_flight [0-9]+\n/ +body matches /# TYPE sourcemeta_one_metrics_dropped_total counter\nsourcemeta_one_metrics_dropped_total 0\n/ +body matches /# TYPE sourcemeta_one_http_requests_total counter\n/ +body matches /sourcemeta_one_http_requests_total\{action="health_check_v1",code="200"\} [0-9]+\n/ +body matches /# TYPE sourcemeta_one_http_request_duration_seconds histogram\n/ +body matches /sourcemeta_one_http_request_duration_seconds_bucket\{action="health_check_v1",le="0.0001"\} [0-9]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_bucket\{action="health_check_v1",le="1.0"\} [0-9]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_bucket\{action="health_check_v1",le="\+Inf"\} [0-9]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_sum\{action="health_check_v1"\} [0-9.e-]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_count\{action="health_check_v1"\} [0-9]+\n/ +body not matches /le="1e-04"/ + +# The endpoint counts itself like anything else, one scrape behind +GET {{base}}/self/v1/metrics +HTTP 200 +[Asserts] +body matches /sourcemeta_one_http_requests_total\{action="metrics_v1",code="200"\} [0-9]+\n/ + +POST {{base}}/self/v1/metrics +HTTP 405 +Cache-Control: no-store +Content-Type: application/problem+json +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Allow: GET, HEAD, OPTIONS +Link: ; rel="describedby" +[Captures] +last_response: body +schema_path: header "Link" regex "<([^>]+)>" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +jsonpath "$.status" == 405 +jsonpath "$.type" == "urn:sourcemeta:one:method-not-allowed" +jsonpath "$.title" == "Method Not Allowed" + +POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}} +``` +{{last_response}} +``` +HTTP 200 +Cache-Control: no-store +Link: ; rel="describedby" +[Asserts] +jsonpath "$.valid" == true + +OPTIONS {{base}}/self/v1/metrics +HTTP 204 +Cache-Control: no-store +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Access-Control-Allow-Methods: GET, HEAD, OPTIONS +Access-Control-Allow-Headers: Accept, Accept-Encoding +Access-Control-Max-Age: 3600 +Allow: GET, HEAD, OPTIONS diff --git a/enterprise/e2e/html/hurl/prometheus.compose.hurl b/enterprise/e2e/html/hurl/prometheus.compose.hurl new file mode 100644 index 000000000..b89009640 --- /dev/null +++ b/enterprise/e2e/html/hurl/prometheus.compose.hurl @@ -0,0 +1,130 @@ +# Something for a scrape to have counted, so that the assertions below read a +# series that exists rather than one that happens not to yet +GET {{base}}/self/v1/health +HTTP 200 + +GET {{base}}/self/v1/api/list +HTTP 200 + +GET {{base}}/self/v1/metrics +HTTP 200 +Content-Type: text/plain; version=0.0.4; charset=utf-8 + +# Whether a scrape succeeded is the scraper's own verdict on what it read, and +# it is zero for a target it could not reach or could not parse. Nothing else +# here proves as much in a single number +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: up{job="sourcemeta-one"} +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.resultType" == "vector" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "1" + +# The scraper reports on its own targets, which says the endpoint answered +# rather than merely that something was listening +GET http://localhost:9099/api/v1/targets +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +state: active +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.activeTargets" count == 1 +jsonpath "$.data.activeTargets[0].health" == "up" +jsonpath "$.data.activeTargets[0].lastError" == "" +jsonpath "$.data.activeTargets[0].scrapePool" == "sourcemeta-one" +jsonpath "$.data.activeTargets[0].labels.job" == "sourcemeta-one" + +# An info metric survives the round trip with its labels intact, which is the +# whole reason it is shaped as a gauge whose value says nothing +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_build_info +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].metric.edition" == "enterprise" +jsonpath "$.data.result[0].metric.__name__" == "sourcemeta_one_build_info" +jsonpath "$.data.result[0].value[1]" == "1" + +# A counter carrying both of its labels, which is what makes the series +# addressable by the questions an operator actually asks +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_http_requests_total{action="health_check_v1",code="200"} +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].metric.action" == "health_check_v1" +jsonpath "$.data.result[0].metric.code" == "200" + +# Asking for a percentile is what proves the buckets are cumulative, ordered, +# and closed by an infinite bound. A malformed histogram answers nothing here +# even though it parses perfectly well as text +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: histogram_quantile(0.95, sum by (le) (rate(sourcemeta_one_http_request_duration_seconds_bucket[1m]))) +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" exists + +# The count a histogram carries and the number of requests counted separately +# are two ways of saying the same thing, and they disagree if either is wrong +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sum(sourcemeta_one_http_request_duration_seconds_count) - sum(sourcemeta_one_http_requests_total) +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "0" + +# Nothing was lost on the way to being counted +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_metrics_dropped_total +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "0" + +# The process metrics use names shared across every language's client library, +# so a dashboard written against any of them finds these +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: process_resident_memory_bytes > 0 and process_max_fds > 0 and process_start_time_seconds > 0 +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 diff --git a/enterprise/e2e/html/prometheus.yml b/enterprise/e2e/html/prometheus.yml new file mode 100644 index 000000000..6a3b493d1 --- /dev/null +++ b/enterprise/e2e/html/prometheus.yml @@ -0,0 +1,13 @@ +# A scrape interval this short is unreasonable for a real deployment and +# exactly right here, where the point is for a series to exist before the +# assertions that read it run +global: + scrape_interval: 1s + evaluation_interval: 1s + +scrape_configs: + - job_name: sourcemeta-one + metrics_path: /self/v1/metrics + static_configs: + - targets: + - "sandbox:8001" diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server.h b/enterprise/server/include/sourcemeta/one/enterprise_server.h index b135ad888..3ddc1e60f 100644 --- a/enterprise/server/include/sourcemeta/one/enterprise_server.h +++ b/enterprise/server/include/sourcemeta/one/enterprise_server.h @@ -8,5 +8,6 @@ #include #include #include +#include #endif diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h new file mode 100644 index 000000000..54005a958 --- /dev/null +++ b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h @@ -0,0 +1,349 @@ +#ifndef SOURCEMETA_ONE_ENTERPRISE_SERVER_ACTION_METRICS_V1_H +#define SOURCEMETA_ONE_ENTERPRISE_SERVER_ACTION_METRICS_V1_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include // std::ranges::transform +#include // std::array +#include // std::uint64_t +#include // std::filesystem::path +#include // std::format +#include // std::span +#include // std::string +#include // std::string_view +#include // std::vector + +#if defined(__APPLE__) +#include // proc_pidinfo, PROC_PIDLISTFDS, PROC_PIDLISTFD_SIZE +#include // mach_task_self, task_info, MACH_TASK_BASIC_INFO +#include // getrusage, getrlimit, RUSAGE_SELF, RLIMIT_NOFILE +#include // getpid +#elif defined(__linux__) +#include // std::ifstream +#include // std::istringstream +#include // getrlimit, RLIMIT_NOFILE +#include // std::error_code +#include // sysconf, _SC_CLK_TCK, _SC_PAGESIZE +#endif + +class ActionMetrics_v1 : public sourcemeta::one::RouterAction { +public: + static constexpr std::string_view DESCRIPTION{ + "Report instance telemetry in the Prometheus exposition format"}; + static constexpr bool READ_ONLY{true}; + static constexpr bool DESTRUCTIVE{false}; + static constexpr bool IDEMPOTENT{true}; + static constexpr bool OPEN_WORLD{false}; + + // How a boundary is spelled is part of the name of the series it bounds, so + // it is decided here rather than by whatever a formatter does with the same + // number on a given platform + static constexpr std::array + BOUNDARIES{{"0.0001", "0.00025", "0.0005", "0.001", "0.0025", "0.005", + "0.01", "0.05", "0.25", "1.0"}}; + + ActionMetrics_v1( + const std::filesystem::path &base, + const sourcemeta::core::URITemplateRouterView &router, + const sourcemeta::core::URITemplateRouter::Identifier identifier, + sourcemeta::one::Router &dispatcher) + : sourcemeta::one::RouterAction{base, router.base_url(), dispatcher} { + router.arguments( + identifier, [this](const auto &key, const auto &value) -> void { + if (key == "errorSchema") { + this->error_schema_ = std::get(value); + } + }); + + // A label value never changes for the life of an instance, so it is + // spelled once here rather than on every scrape + this->actions_.reserve(sourcemeta::one::ACTION_NAMES.size()); + for (const auto &name : sourcemeta::one::ACTION_NAMES) { + std::string lowered{name}; + std::ranges::transform(lowered, lowered.begin(), + [](const char character) -> char { + return sourcemeta::core::to_lowercase(character); + }); + this->actions_.push_back(std::move(lowered)); + } + } + + auto rest(const std::span, + const sourcemeta::one::Authentication::Caller &, + sourcemeta::one::HTTPRequest &request, + sourcemeta::one::HTTPResponse &response) -> void override { + if (request.method() == "options") { + sourcemeta::one::cors_preflight(request, response, "GET, HEAD, OPTIONS", + "Accept, Accept-Encoding"); + return; + } + + if (request.method() != "get" && request.method() != "head") { + sourcemeta::one::json_error( + request, response, sourcemeta::core::HTTP_STATUS_METHOD_NOT_ALLOWED, + "urn:sourcemeta:one:method-not-allowed", + "This HTTP method is invalid for this URL", this->error_schema_, "*", + "GET, HEAD, OPTIONS"); + return; + } + + const auto payload{this->serialize()}; + response.write_status(sourcemeta::core::HTTP_STATUS_OK); + // The exposition format names its own revision in the media type, which is + // how a scraper knows what it is reading without asking + response.write_header("Content-Type", + "text/plain; version=0.0.4; charset=utf-8"); + // A scrape stands for the moment it was taken, so an answer kept and handed + // to the next one would report a past that never comes back + response.write_header("Cache-Control", + sourcemeta::one::cache_control_no_store()); + sourcemeta::one::send_response(sourcemeta::core::HTTP_STATUS_OK, request, + response, payload, + sourcemeta::one::Encoding::Identity); + } + + auto mcp(const sourcemeta::core::MCPProtocolVersion, + const sourcemeta::core::JSON &id, const sourcemeta::core::JSON &, + const sourcemeta::one::Authentication::Caller &) + -> sourcemeta::core::JSON override { + return sourcemeta::core::jsonrpc_make_error_method_not_found(id); + } + +private: + // What a process says about itself, which the platform answers rather than + // this program keeping count of. Anything a platform cannot cheaply say is + // left out rather than guessed at + struct Process { + double cpu_seconds{0}; + std::uint64_t resident_bytes{0}; + std::uint64_t virtual_bytes{0}; + std::uint64_t open_descriptors{0}; + std::uint64_t maximum_descriptors{0}; + }; + + [[nodiscard]] static auto descriptor_limit() -> std::uint64_t { + rlimit limit{}; + if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { + return 0; + } + + return static_cast(limit.rlim_cur); + } + +#if defined(__APPLE__) + + [[nodiscard]] static auto read_process() -> Process { + Process sample; + + rusage usage{}; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + sample.cpu_seconds = + static_cast(usage.ru_utime.tv_sec) + + static_cast(usage.ru_utime.tv_usec) / 1000000.0 + + static_cast(usage.ru_stime.tv_sec) + + static_cast(usage.ru_stime.tv_usec) / 1000000.0; + } + + mach_task_basic_info info{}; + mach_msg_type_number_t count{MACH_TASK_BASIC_INFO_COUNT}; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, + reinterpret_cast(&info), + &count) == KERN_SUCCESS) { + sample.resident_bytes = info.resident_size; + sample.virtual_bytes = info.virtual_size; + } + + const auto descriptors{ + proc_pidinfo(getpid(), PROC_PIDLISTFDS, 0, nullptr, 0)}; + if (descriptors > 0) { + sample.open_descriptors = + static_cast(descriptors) / PROC_PIDLISTFD_SIZE; + } + + sample.maximum_descriptors = descriptor_limit(); + return sample; + } + +#elif defined(__linux__) + + // The second field is a command name in parentheses that may itself contain + // spaces, so what follows it is found from the last parenthesis rather than + // by counting separators from the beginning + [[nodiscard]] static auto read_process() -> Process { + Process sample; + + std::ifstream stream{"/proc/self/stat"}; + if (stream.is_open()) { + std::string line; + std::getline(stream, line); + const auto comm{line.rfind(')')}; + if (comm != std::string::npos) { + std::istringstream fields{line.substr(comm + 1)}; + std::vector tokens; + std::string token; + while (fields >> token) { + tokens.push_back(token); + } + + const auto ticks{static_cast(sysconf(_SC_CLK_TCK))}; + if (tokens.size() > 21 && ticks > 0) { + sample.cpu_seconds = + (std::stod(tokens.at(11)) + std::stod(tokens.at(12))) / ticks; + sample.virtual_bytes = std::stoull(tokens.at(20)); + sample.resident_bytes = + std::stoull(tokens.at(21)) * + static_cast(sysconf(_SC_PAGESIZE)); + } + } + } + + std::error_code error; + const std::filesystem::directory_iterator descriptors{"/proc/self/fd", + error}; + if (!error) { + for (const auto &entry : descriptors) { + static_cast(entry); + sample.open_descriptors += 1; + } + } + + sample.maximum_descriptors = descriptor_limit(); + return sample; + } + +#else + + [[nodiscard]] static auto read_process() -> Process { return {}; } + +#endif + + static auto family(std::string &output, const std::string_view name, + const std::string_view help, const std::string_view type) + -> void { + output += + std::format("# HELP {} {}\n# TYPE {} {}\n", name, help, name, type); + } + + [[nodiscard]] auto serialize() const -> std::string { + const auto &metrics{this->dispatcher().metrics()}; + const auto state{metrics.snapshot()}; + const auto process{read_process()}; + + std::string edition{sourcemeta::one::edition()}; + std::ranges::transform(edition, edition.begin(), + [](const char character) -> char { + return sourcemeta::core::to_lowercase(character); + }); + + std::string output; + output.reserve(8192); + + family(output, "sourcemeta_one_build_info", "Build information", "gauge"); + output += std::format( + "sourcemeta_one_build_info{{version=\"{}\",edition=\"{}\"}} 1\n\n", + sourcemeta::one::version(), edition); + + family(output, "process_start_time_seconds", + "Start time of the process since the Unix epoch", "gauge"); + output += + std::format("process_start_time_seconds {}\n\n", metrics.started()); + + family(output, "process_cpu_seconds_total", + "Total user and system CPU time spent", "counter"); + output += + std::format("process_cpu_seconds_total {}\n\n", process.cpu_seconds); + + family(output, "process_resident_memory_bytes", "Resident memory size", + "gauge"); + output += std::format("process_resident_memory_bytes {}\n\n", + process.resident_bytes); + + family(output, "process_virtual_memory_bytes", "Virtual memory size", + "gauge"); + output += std::format("process_virtual_memory_bytes {}\n\n", + process.virtual_bytes); + + family(output, "process_open_fds", "Number of open file descriptors", + "gauge"); + output += std::format("process_open_fds {}\n\n", process.open_descriptors); + + family(output, "process_max_fds", "Maximum number of open file descriptors", + "gauge"); + output += + std::format("process_max_fds {}\n\n", process.maximum_descriptors); + + family(output, "sourcemeta_one_http_requests_in_flight", + "Requests currently being served", "gauge"); + output += std::format("sourcemeta_one_http_requests_in_flight {}\n\n", + state.in_flight); + + family(output, "sourcemeta_one_metrics_dropped_total", + "Observations that could not be recorded", "counter"); + output += std::format("sourcemeta_one_metrics_dropped_total {}\n\n", + state.dropped); + + family(output, "sourcemeta_one_http_requests_total", + "Total HTTP requests handled", "counter"); + for (const auto &[key, count] : state.requests) { + output += std::format( + "sourcemeta_one_http_requests_total{{action=\"{}\"," + "code=\"{}\"}} {}\n", + this->actions_.at(sourcemeta::one::RouterMetrics::action_of(key)), + sourcemeta::one::RouterMetrics::status_of(key), count); + } + + output += "\n"; + family(output, "sourcemeta_one_http_request_duration_seconds", + "Request duration", "histogram"); + for (std::size_t action = 0; action < state.buckets.size(); action++) { + std::uint64_t cumulative{0}; + for (const auto count : state.buckets[action]) { + cumulative += count; + } + + if (cumulative == 0) { + continue; + } + + cumulative = 0; + const auto &name{this->actions_.at(action)}; + for (std::size_t bucket = 0; + bucket < sourcemeta::one::RouterMetrics::BUCKET_COUNT; bucket++) { + cumulative += state.buckets[action][bucket]; + output += std::format("sourcemeta_one_http_request_duration_seconds_" + "bucket{{action=\"{}\",le=\"{}\"}} {}\n", + name, BOUNDARIES.at(bucket), cumulative); + } + + cumulative += + state.buckets[action][sourcemeta::one::RouterMetrics::BUCKET_COUNT]; + output += std::format("sourcemeta_one_http_request_duration_seconds_" + "bucket{{action=\"{}\",le=\"+Inf\"}} {}\n", + name, cumulative); + output += std::format("sourcemeta_one_http_request_duration_seconds_sum{{" + "action=\"{}\"}} {}\n", + name, state.sums[action]); + output += + std::format("sourcemeta_one_http_request_duration_seconds_count{{" + "action=\"{}\"}} {}\n", + name, cumulative); + } + + return output; + } + + std::string_view error_schema_; + std::vector actions_; +}; + +#endif diff --git a/src/actions/CMakeLists.txt b/src/actions/CMakeLists.txt index 8fdbd5eca..4c1c56724 100644 --- a/src/actions/CMakeLists.txt +++ b/src/actions/CMakeLists.txt @@ -4,6 +4,7 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT one NAME actions action_default_v1.h action_dependency_tree_v1.h action_health_check_v1.h + action_metrics_v1.h action_jsonschema_evaluate_v1.h action_jsonschema_rdf_v1.h action_jsonschema_serve_v1.h diff --git a/src/actions/action_metrics_v1.h b/src/actions/action_metrics_v1.h new file mode 100644 index 000000000..0a8e67478 --- /dev/null +++ b/src/actions/action_metrics_v1.h @@ -0,0 +1,84 @@ +#ifndef SOURCEMETA_ONE_ACTIONS_METRICS_V1_H +#define SOURCEMETA_ONE_ACTIONS_METRICS_V1_H + +#if defined(SOURCEMETA_ONE_ENTERPRISE) + +#include + +#else + +#include +#include +#include +#include + +#include +#include + +#include // std::filesystem::path +#include // std::span +#include // std::string_view + +class ActionMetrics_v1 : public sourcemeta::one::RouterAction { +public: + static constexpr std::string_view DESCRIPTION{ + "Report instance telemetry in the Prometheus exposition format"}; + static constexpr bool READ_ONLY{true}; + static constexpr bool DESTRUCTIVE{false}; + static constexpr bool IDEMPOTENT{true}; + static constexpr bool OPEN_WORLD{false}; + + ActionMetrics_v1( + const std::filesystem::path &base, + const sourcemeta::core::URITemplateRouterView &router, + const sourcemeta::core::URITemplateRouter::Identifier identifier, + sourcemeta::one::Router &dispatcher) + : sourcemeta::one::RouterAction{base, router.base_url(), dispatcher} { + router.arguments( + identifier, [this](const auto &key, const auto &value) -> void { + if (key == "errorSchema") { + this->error_schema_ = std::get(value); + } + }); + } + + auto rest(const std::span, + const sourcemeta::one::Authentication::Caller &, + sourcemeta::one::HTTPRequest &request, + sourcemeta::one::HTTPResponse &response) -> void override { + if (request.method() == "options") { + sourcemeta::one::cors_preflight(request, response, "GET, HEAD, OPTIONS", + "Accept, Accept-Encoding"); + return; + } + + if (request.method() != "get" && request.method() != "head") { + sourcemeta::one::json_error( + request, response, sourcemeta::core::HTTP_STATUS_METHOD_NOT_ALLOWED, + "urn:sourcemeta:one:method-not-allowed", + "This HTTP method is invalid for this URL", this->error_schema_, "*", + "GET, HEAD, OPTIONS"); + return; + } + + sourcemeta::one::json_error( + request, response, sourcemeta::core::HTTP_STATUS_FORBIDDEN, + "urn:sourcemeta:one:enterprise-required", + "This feature is only available in the Enterprise edition", + this->error_schema_, "*"); + } + + auto mcp(const sourcemeta::core::MCPProtocolVersion, + const sourcemeta::core::JSON &id, const sourcemeta::core::JSON &, + const sourcemeta::one::Authentication::Caller &) + -> sourcemeta::core::JSON override { + return sourcemeta::core::jsonrpc_make_error_method_not_found(id); + } + +private: + std::string_view error_schema_; +}; + +#endif + +#endif diff --git a/src/actions/actions.cc b/src/actions/actions.cc index 9dc89c11a..b46274894 100644 --- a/src/actions/actions.cc +++ b/src/actions/actions.cc @@ -23,6 +23,7 @@ #include "action_list_directory_v1.h" #include "action_mcp_prm_v1.h" #include "action_mcp_v1.h" +#include "action_metrics_v1.h" #include "action_not_found_v1.h" #include "action_schema_search_v1.h" #include "action_serve_explorer_artifact_v1.h" diff --git a/src/actions/include/sourcemeta/one/actions.h b/src/actions/include/sourcemeta/one/actions.h index 250a3aefd..5a03ab9ff 100644 --- a/src/actions/include/sourcemeta/one/actions.h +++ b/src/actions/include/sourcemeta/one/actions.h @@ -12,6 +12,7 @@ namespace sourcemeta::one { #define SOURCEMETA_ONE_FOR_EACH_ACTION(X) \ X(DEFAULT_V1, ActionDefault_v1) \ X(HEALTH_CHECK_V1, ActionHealthCheck_v1) \ + X(METRICS_V1, ActionMetrics_v1) \ X(NOT_FOUND_V1, ActionNotFound_v1) \ X(SCHEMA_ARTIFACT_V1, ActionServeSchemaArtifact_v1) \ X(EXPLORER_ARTIFACT_V1, ActionServeExplorerArtifact_v1) \ @@ -45,6 +46,16 @@ enum : std::uint8_t { #undef SOURCEMETA_ONE_DEFINE_ACTION_TYPE +#define SOURCEMETA_ONE_DEFINE_ACTION_NAME(Name, Class) std::string_view{#Name}, + +// What an action is called wherever one has to be named to somebody outside +// this program, taken from the same list the handlers themselves come from so +// that neither can name an action the other does not have +inline constexpr std::array ACTION_NAMES{ + {SOURCEMETA_ONE_FOR_EACH_ACTION(SOURCEMETA_ONE_DEFINE_ACTION_NAME)}}; + +#undef SOURCEMETA_ONE_DEFINE_ACTION_NAME + extern const std::array CONSTRUCTORS; diff --git a/src/http/include/sourcemeta/one/http_helpers.h b/src/http/include/sourcemeta/one/http_helpers.h index 466047069..6a6896a5f 100644 --- a/src/http/include/sourcemeta/one/http_helpers.h +++ b/src/http/include/sourcemeta/one/http_helpers.h @@ -14,9 +14,11 @@ #include // std::ranges::equal #include // std::array #include // assert -#include // std::chrono::system_clock +#include // std::chrono::system_clock, std::chrono::steady_clock #include // std::size_t +#include // std::uint8_t, std::uint16_t #include // std::format +#include // std::function #include // std::mutex, std::scoped_lock #include // std::optional #include // std::print @@ -193,6 +195,22 @@ inline auto request_body_too_large(const HTTPRequest &request) -> bool { return declared.has_value() && declared.value() > MAX_REQUEST_BODY_BYTES; } +// What is said about a request once it has been answered. Whoever stands the +// server up decides whether anything listens and what it does with what it +// hears, so this module never learns what an answer is counted for +using ResponseObserver = std::function; + +inline ResponseObserver RESPONSE_OBSERVER{}; + +inline auto observe_response(const sourcemeta::core::HTTPStatus &status, + const HTTPRequest &request) -> void { + if (RESPONSE_OBSERVER) [[unlikely]] { + RESPONSE_OBSERVER(request.observed_as(), status.code, + request.observed_from()); + } +} + // Answering can be the last thing that happens on a connection, and a request // read asynchronously is held by the very handler that goes away with it. So // what is said about a request is taken while it is certainly still there, @@ -204,6 +222,7 @@ inline auto send_response(const sourcemeta::core::HTTPStatus &status, std::format("{} {} {}", status.wire, request.method(), request.path())}; response.send_without_content(); HTTP_LOG(line); + observe_response(status, request); } inline auto send_response( @@ -217,6 +236,7 @@ inline auto send_response( response.send(request, message, current_encoding, precomputed_compressed_size); HTTP_LOG(line); + observe_response(status, request); } // RFC 9110 §9.3.7: OPTIONS responses describe communication options diff --git a/src/http/include/sourcemeta/one/http_request.h b/src/http/include/sourcemeta/one/http_request.h index 5ec99dac1..ff4ddff49 100644 --- a/src/http/include/sourcemeta/one/http_request.h +++ b/src/http/include/sourcemeta/one/http_request.h @@ -7,10 +7,12 @@ #include #include -#include // std::chrono::system_clock +#include // std::chrono::system_clock, std::chrono::steady_clock #include // std::invocable #include // std::size_t +#include // std::uint8_t #include // std::exception_ptr, std::current_exception +#include // std::numeric_limits #include // std::shared_ptr, std::make_shared #include // std::optional #include // std::string @@ -25,6 +27,11 @@ namespace sourcemeta::one { inline constexpr std::size_t MAX_REQUEST_BODY_BYTES{ static_cast(4) * 1024 * 1024}; +// A request answered before any handler was chosen, which is every refusal +// that happens ahead of routing +inline constexpr std::uint8_t OBSERVATION_UNROUTED{ + std::numeric_limits::max()}; + class HTTPRequest { public: // Primary constructor from raw uWebSockets pointers @@ -53,6 +60,28 @@ class HTTPRequest { : sourcemeta::one::Encoding::Identity; } + // When this request reached a handler and which handler it reached, kept + // so that whoever answers can say both without asking the router again. The + // action is a value no handler owns until one is chosen, so a request + // answered before routing carries none + auto observe_from(const std::chrono::steady_clock::time_point value) noexcept + -> void { + this->observed_from_ = value; + } + + [[nodiscard]] auto observed_from() const noexcept + -> std::chrono::steady_clock::time_point { + return this->observed_from_; + } + + auto observe_as(const std::uint8_t value) noexcept -> void { + this->observed_as_ = value; + } + + [[nodiscard]] auto observed_as() const noexcept -> std::uint8_t { + return this->observed_as_; + } + [[nodiscard]] auto method() const noexcept -> std::string_view { return this->request_ ? this->request_->getMethod() : this->method_; } @@ -150,6 +179,8 @@ class HTTPRequest { auto snapshot = std::make_shared( std::string{this->method()}, std::string{this->path()}, this->response_encoding_, raw_response); + snapshot->observe_from(this->observed_from_); + snapshot->observe_as(this->observed_as_); auto buffer = std::make_shared(); auto completed = std::make_shared(false); @@ -204,6 +235,8 @@ class HTTPRequest { bool satisfiable_encoding_{true}; sourcemeta::one::Encoding response_encoding_{ sourcemeta::one::Encoding::Identity}; + std::chrono::steady_clock::time_point observed_from_{}; + std::uint8_t observed_as_{OBSERVATION_UNROUTED}; }; } // namespace sourcemeta::one diff --git a/src/index/endpoints.h b/src/index/endpoints.h index 03252d0d7..21181e03c 100644 --- a/src/index/endpoints.h +++ b/src/index/endpoints.h @@ -34,6 +34,7 @@ inline constexpr std::string_view ENDPOINT_SCHEMA_TRACE{ inline constexpr std::string_view ENDPOINT_SCHEMA_SEARCH{ "/self/v1/api/schemas/search"}; inline constexpr std::string_view ENDPOINT_HEALTH{"/self/v1/health"}; +inline constexpr std::string_view ENDPOINT_METRICS{"/self/v1/metrics"}; inline constexpr std::string_view ENDPOINT_AUTH_LOGOUT{"/self/v1/auth/logout"}; inline constexpr std::string_view ENDPOINT_AUTH_LOGIN_PAGE{ "/self/v1/auth/login"}; diff --git a/src/index/generators.h b/src/index/generators.h index 28f309a74..68321eeb9 100644 --- a/src/index/generators.h +++ b/src/index/generators.h @@ -900,6 +900,11 @@ struct GENERATE_URITEMPLATE_ROUTES { next_id++, sourcemeta::one::ACTION_TYPE_HEALTH_CHECK_V1, health_check_arguments); + const sourcemeta::core::URITemplateRouter::Argument metrics_arguments[] = + {{"errorSchema", std::string_view{error_schema}}}; + router.add(sourcemeta::one::ENDPOINT_METRICS, "server_metrics", next_id++, + sourcemeta::one::ACTION_TYPE_METRICS_V1, metrics_arguments); + const sourcemeta::core::URITemplateRouter::Argument auth_logout_arguments[] = { {"errorSchema", std::string_view{error_schema}}}; diff --git a/src/router/CMakeLists.txt b/src/router/CMakeLists.txt index c37b00583..d341fff5c 100644 --- a/src/router/CMakeLists.txt +++ b/src/router/CMakeLists.txt @@ -1,6 +1,6 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT one NAME router - PRIVATE_HEADERS lru.h - SOURCES artifact.cc router.cc evaluate.cc) + PRIVATE_HEADERS lru.h metrics.h + SOURCES artifact.cc router.cc evaluate.cc metrics.cc) target_link_libraries(sourcemeta_one_router PUBLIC sourcemeta::one::authentication) target_link_libraries(sourcemeta_one_router PUBLIC sourcemeta::one::http) diff --git a/src/router/include/sourcemeta/one/router.h b/src/router/include/sourcemeta/one/router.h index 05dff6da2..263f2c972 100644 --- a/src/router/include/sourcemeta/one/router.h +++ b/src/router/include/sourcemeta/one/router.h @@ -11,6 +11,7 @@ #include #include #include +#include #include // std::size_t #include // std::uint8_t @@ -397,6 +398,10 @@ class Router { return this->authentication_; } + [[nodiscard]] auto metrics() const noexcept -> RouterMetrics & { + return this->metrics_; + } + private: static constexpr std::size_t TEMPLATE_CACHE_CAPACITY{50}; @@ -417,6 +422,7 @@ class Router { RouterLRU template_cache_{ TEMPLATE_CACHE_CAPACITY}; Authentication authentication_; + mutable RouterMetrics metrics_; }; } // namespace sourcemeta::one diff --git a/src/router/include/sourcemeta/one/router_metrics.h b/src/router/include/sourcemeta/one/router_metrics.h new file mode 100644 index 000000000..028d489ea --- /dev/null +++ b/src/router/include/sourcemeta/one/router_metrics.h @@ -0,0 +1,101 @@ +#ifndef SOURCEMETA_ONE_ROUTER_METRICS_H +#define SOURCEMETA_ONE_ROUTER_METRICS_H + +#include // std::array +#include // std::atomic +#include // std::size_t +#include // std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t +#include // std::map +#include // std::mutex +#include // std::vector + +namespace sourcemeta::one { + +// What a request cost, kept as numbers and nothing else. What any of it is +// called, and whether an instance says any of it out loud, is decided by +// whoever reads this rather than here +class RouterMetrics { +public: + static constexpr std::size_t BUCKET_COUNT{10}; + + // An order of magnitude finer at the low end than the usual boundaries, + // since answering out of a metapack takes well under a millisecond and the + // defaults would put nearly every request in the first bucket + static constexpr std::array BUCKETS{ + {0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.05, 0.25, 1.0}}; + + // Everything counted so far, taken in one pass so that what is read cannot + // disagree with itself. Requests are ordered, since two readings of an + // unchanged instance should say the same thing in the same order + struct Snapshot { + std::uint64_t in_flight{0}; + std::uint64_t dropped{0}; + std::map requests; + std::vector> buckets; + std::vector sums; + }; + + RouterMetrics(); + + // To avoid mistakes + RouterMetrics(const RouterMetrics &) = delete; + RouterMetrics(RouterMetrics &&) = delete; + auto operator=(const RouterMetrics &) -> RouterMetrics & = delete; + auto operator=(RouterMetrics &&) -> RouterMetrics & = delete; + + // How many handlers this instance has, which is every index that may ever be + // counted against + auto resize(std::size_t actions) -> void; + + // A request has reached a handler, which is what the in-flight count is the + // difference between + auto enter() noexcept -> void; + + // A request has been answered, which is both what is counted and what brings + // the in-flight count back down + auto observe(std::uint8_t action, std::uint16_t status, + double seconds) noexcept -> void; + + [[nodiscard]] auto snapshot() const -> Snapshot; + + // When this instance began, as seconds since the Unix epoch + [[nodiscard]] auto started() const noexcept -> double { + return this->started_; + } + + // How a request and what it was answered with are held together, since a + // series is named by both and neither alone identifies one + [[nodiscard]] static constexpr auto + action_of(const std::uint32_t key) noexcept -> std::size_t { + return static_cast(key >> 16U); + } + + [[nodiscard]] static constexpr auto + status_of(const std::uint32_t key) noexcept -> std::uint16_t { + return static_cast(key & 0xFFFFU); + } + +private: + // One of these per hardware thread, so that answering a request touches a + // line no other thread is writing to. Reading them is a scrape, which + // happens once every several seconds and can afford to visit each in turn + struct Shard { + mutable std::mutex mutex; + std::map requests; + std::vector> buckets; + std::vector sums; + }; + + [[nodiscard]] auto shard() const noexcept -> Shard &; + + std::size_t actions_{0}; + mutable std::vector shards_; + std::atomic entered_{0}; + std::atomic answered_{0}; + std::atomic dropped_{0}; + double started_; +}; + +} // namespace sourcemeta::one + +#endif diff --git a/src/router/metrics.cc b/src/router/metrics.cc new file mode 100644 index 000000000..272d8d84f --- /dev/null +++ b/src/router/metrics.cc @@ -0,0 +1,92 @@ +#include + +#include // std::ranges::lower_bound, std::max +#include // std::chrono::system_clock, std::chrono::duration +#include // std::hash +#include // std::distance +#include // std::scoped_lock +#include // std::thread, std::this_thread + +namespace sourcemeta::one { + +RouterMetrics::RouterMetrics() + : shards_{std::max(std::thread::hardware_concurrency(), 1)}, + started_{std::chrono::duration{ + std::chrono::system_clock::now().time_since_epoch()} + .count()} {} + +auto RouterMetrics::resize(const std::size_t actions) -> void { + this->actions_ = actions; + for (auto &shard : this->shards_) { + const std::scoped_lock guard{shard.mutex}; + shard.buckets.assign(actions, {}); + shard.sums.assign(actions, 0.0); + } +} + +auto RouterMetrics::shard() const noexcept -> Shard & { + const auto index{std::hash{}(std::this_thread::get_id()) % + this->shards_.size()}; + return this->shards_[index]; +} + +auto RouterMetrics::enter() noexcept -> void { + this->entered_.fetch_add(1, std::memory_order_relaxed); +} + +auto RouterMetrics::observe(const std::uint8_t action, + const std::uint16_t status, + const double seconds) noexcept -> void { + this->answered_.fetch_add(1, std::memory_order_relaxed); + if (action >= this->actions_) [[unlikely]] { + return; + } + + const auto boundary{std::ranges::lower_bound(BUCKETS, seconds)}; + const auto bucket{ + static_cast(std::distance(BUCKETS.begin(), boundary))}; + + // Nothing said about a request may change how it was answered, and the + // answer has already gone out by the time this runs + try { + auto &target{this->shard()}; + const std::scoped_lock guard{target.mutex}; + const auto key{(static_cast(action) << 16U) | status}; + target.requests[key] += 1; + target.buckets[action][bucket] += 1; + target.sums[action] += seconds; + } catch (...) { + // A gap somebody can see is worth more than one they cannot, so what could + // not be recorded is counted and said in the answer itself + this->dropped_.fetch_add(1, std::memory_order_relaxed); + } +} + +auto RouterMetrics::snapshot() const -> Snapshot { + Snapshot result; + result.dropped = this->dropped_.load(std::memory_order_relaxed); + const auto entered{this->entered_.load(std::memory_order_relaxed)}; + const auto answered{this->answered_.load(std::memory_order_relaxed)}; + result.in_flight = entered > answered ? entered - answered : 0; + result.buckets.assign(this->actions_, {}); + result.sums.assign(this->actions_, 0.0); + + for (const auto &shard : this->shards_) { + const std::scoped_lock guard{shard.mutex}; + for (const auto &[key, count] : shard.requests) { + result.requests[key] += count; + } + + for (std::size_t action = 0; action < shard.buckets.size(); action++) { + for (std::size_t bucket = 0; bucket <= BUCKET_COUNT; bucket++) { + result.buckets[action][bucket] += shard.buckets[action][bucket]; + } + + result.sums[action] += shard.sums[action]; + } + } + + return result; +} + +} // namespace sourcemeta::one diff --git a/src/router/router.cc b/src/router/router.cc index 88f0c4082..fcf3c510f 100644 --- a/src/router/router.cc +++ b/src/router/router.cc @@ -4,6 +4,7 @@ #include #include // std::chrono::seconds +#include // std::uint8_t #include // std::make_unique #include // std::call_once #include // std::optional, std::nullopt @@ -119,6 +120,8 @@ auto Router::dispatch( const std::span matches, sourcemeta::one::HTTPRequest &request, sourcemeta::one::HTTPResponse &response) -> void { + request.observe_as(static_cast(context)); + this->metrics_.enter(); auto *instance{this->action(identifier, context)}; if (instance == nullptr) [[unlikely]] { this->error(request, response, diff --git a/src/server/server.cc b/src/server/server.cc index a8d561bc2..5ee65abb9 100644 --- a/src/server/server.cc +++ b/src/server/server.cc @@ -24,6 +24,7 @@ static auto dispatch(sourcemeta::one::Router &actions, sourcemeta::one::HTTPRequest &request, sourcemeta::one::HTTPResponse &response) noexcept -> void { try { + request.observe_from(std::chrono::steady_clock::now()); request.negotiate(); if (request.satisfiable_encoding()) { thread_local std::array< @@ -112,6 +113,18 @@ auto main(int argc, char *argv[]) noexcept -> int { sourcemeta::one::Router actions{base, router, sourcemeta::one::CONSTRUCTORS}; +#if defined(SOURCEMETA_ONE_ENTERPRISE) + actions.metrics().resize(sourcemeta::one::ACTION_TYPE_COUNT); + sourcemeta::one::RESPONSE_OBSERVER = + [&actions](const std::uint8_t action, const std::uint16_t status, + const std::chrono::steady_clock::time_point from) -> void { + actions.metrics().observe( + action, status, + std::chrono::duration{std::chrono::steady_clock::now() - from} + .count()); + }; +#endif + const sourcemeta::one::HTTPServer server{ port, [&actions, &router](sourcemeta::one::HTTPRequest &request, diff --git a/test/e2e/common.mk b/test/e2e/common.mk index cc1522d17..dadebcc07 100644 --- a/test/e2e/common.mk +++ b/test/e2e/common.mk @@ -16,7 +16,7 @@ export EDITION all: $(MAKE) down $(MAKE) up - $(MAKE) test-hurl test-playwright; \ + $(MAKE) test-hurl test-hurl-compose test-playwright; \ status=$$?; $(MAKE) down; exit $$status .PHONY: up @@ -32,6 +32,17 @@ test-hurl: $(wildcard hurl/*.all.hurl) \ $(wildcard hurl/*.$(EDITION).hurl) +# Assertions that read a service standing beside the registry, which only +# exists here. The native runner never invokes this target, so a sandbox can +# carry both kinds of test without the second kind failing where there is +# nothing beside it +.PHONY: test-hurl-compose +test-hurl-compose: +ifneq ($(wildcard hurl/*.compose.hurl),) + $(HURL) $(HURL_FLAGS) --repeat 10 --test --variable base=$(BASE):$(PORT) --variable port=$(PORT) \ + $(wildcard hurl/*.compose.hurl) +endif + .PHONY: test-playwright test-playwright: ifneq ($(wildcard playwright/),) diff --git a/test/e2e/html/hurl/metrics.community.hurl b/test/e2e/html/hurl/metrics.community.hurl new file mode 100644 index 000000000..6956443ee --- /dev/null +++ b/test/e2e/html/hurl/metrics.community.hurl @@ -0,0 +1,81 @@ +GET {{base}}/self/v1/metrics +HTTP 403 +Cache-Control: no-store +Content-Type: application/problem+json +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Link: ; rel="describedby" +[Captures] +last_response: body +schema_path: header "Link" regex "<([^>]+)>" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ +jsonpath "$.status" == 403 +jsonpath "$.type" == "urn:sourcemeta:one:enterprise-required" +jsonpath "$.title" == "Forbidden" +jsonpath "$.detail" == "This feature is only available in the Enterprise edition" + +POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}} +``` +{{last_response}} +``` +HTTP 200 +Cache-Control: no-store +Link: ; rel="describedby" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +jsonpath "$.valid" == true + +POST {{base}}/self/v1/metrics +HTTP 405 +Cache-Control: no-store +Content-Type: application/problem+json +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Allow: GET, HEAD, OPTIONS +Link: ; rel="describedby" +[Captures] +last_response: body +schema_path: header "Link" regex "<([^>]+)>" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ +jsonpath "$.status" == 405 +jsonpath "$.type" == "urn:sourcemeta:one:method-not-allowed" +jsonpath "$.title" == "Method Not Allowed" + +POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}} +``` +{{last_response}} +``` +HTTP 200 +Cache-Control: no-store +Link: ; rel="describedby" +[Asserts] +jsonpath "$.valid" == true + +OPTIONS {{base}}/self/v1/metrics +HTTP 204 +Cache-Control: no-store +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Access-Control-Allow-Methods: GET, HEAD, OPTIONS +Access-Control-Allow-Headers: Accept, Accept-Encoding +Access-Control-Max-Age: 3600 +Allow: GET, HEAD, OPTIONS +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ From 989ce99098ebdbd7369f8973074c067348146191 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 09:49:59 -0300 Subject: [PATCH 2/7] Simpler Signed-off-by: Juan Cruz Viotti --- docs/api.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/api.md b/docs/api.md index 72db674a9..df8d30ace 100644 --- a/docs/api.md +++ b/docs/api.md @@ -69,16 +69,9 @@ Any Prometheus-compatible scraper reads this without configuration beyond the path. Request behaviour follows the [RED](https://grafana.com/blog/2018/08/02/the-red-method-how-to-instrument-your-services/) method, meaning rate, errors and duration, reported per action and status code. -Alongside it are the standard `process_*` metrics and an instance information -metric. The response documents itself through its `# HELP` and `# TYPE` lines. This endpoint is not exempt from [authentication](#authentication), so a policy covering its path gates it like any other route. -Metric and label names are part of the public interface, as alerting rules and -dashboards are written against them. New metrics may appear in a minor release, -while renaming or removing one, changing its type or unit, or changing -histogram bucket boundaries are breaking changes. - === "200" The metrics in the Prometheus exposition format. From c5dcab28561f58c0486fa04c856c2f05d90784bb Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 10:07:13 -0300 Subject: [PATCH 3/7] Simpler Signed-off-by: Juan Cruz Viotti --- enterprise/e2e/auth/compose.yml | 18 ++ enterprise/e2e/auth/environment | 1 + enterprise/e2e/auth/hurl/metrics.all.hurl | 248 ++++++++++++++++++ enterprise/e2e/auth/one.json | 7 + enterprise/e2e/{html => auth}/prometheus.yml | 5 + enterprise/e2e/html/compose.yml | 16 -- .../e2e/html/hurl/prometheus.compose.hurl | 130 --------- test/e2e/common.mk | 13 +- 8 files changed, 280 insertions(+), 158 deletions(-) create mode 100644 enterprise/e2e/auth/hurl/metrics.all.hurl rename enterprise/e2e/{html => auth}/prometheus.yml (67%) delete mode 100644 enterprise/e2e/html/hurl/prometheus.compose.hurl diff --git a/enterprise/e2e/auth/compose.yml b/enterprise/e2e/auth/compose.yml index d5f21755a..925712ae6 100644 --- a/enterprise/e2e/auth/compose.yml +++ b/enterprise/e2e/auth/compose.yml @@ -45,3 +45,21 @@ services: condition: service_started ports: - "${PORT}:8001" + + # A real scraper rather than a reader of our own making, so what this + # instance publishes is judged by the software it claims to speak to. It + # scrapes a gated path, so it also proves a policy admits a scraper holding + # the right credential and nothing else + prometheus: + image: prom/prometheus:v3.1.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=1h + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9099:9090" + depends_on: + sandbox: + condition: service_started diff --git a/enterprise/e2e/auth/environment b/enterprise/e2e/auth/environment index 2cd8feba7..f6ac4b9a4 100644 --- a/enterprise/e2e/auth/environment +++ b/enterprise/e2e/auth/environment @@ -2,6 +2,7 @@ ONE_E2E_KEY_PRIMARY=primary-secret-key ONE_E2E_KEY_SECONDARY=secondary-secret-key ONE_E2E_KEY_INTERNAL=internal-secret-key ONE_E2E_KEY_SURFACE=surface-secret-key +ONE_E2E_KEY_METRICS=metrics-secret-key ONE_E2E_KEY_MIXED_PLAIN=mixed-plain-key ONE_E2E_KEY_MIXED_HASHED=2508e2ba355946ba6065f01eb183424aaea10ddaf7bbff90e38f3b702e89e23d ONE_E2E_KEY_BLANK= diff --git a/enterprise/e2e/auth/hurl/metrics.all.hurl b/enterprise/e2e/auth/hurl/metrics.all.hurl new file mode 100644 index 000000000..96eddb905 --- /dev/null +++ b/enterprise/e2e/auth/hurl/metrics.all.hurl @@ -0,0 +1,248 @@ +# A policy on /self/v1/metrics gates the scrape surface at dispatch: denied +# without a credential, with the canonical 401 shape +GET {{base}}/self/v1/metrics +HTTP 401 +Cache-Control: no-store +Content-Type: application/problem+json +WWW-Authenticate: Bearer realm="registry" +Link: ; rel="describedby" +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag, WWW-Authenticate +[Captures] +denied_body: body +denied_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:authentication-required" +jsonpath "$.title" == "Unauthorized" +jsonpath "$.status" == 401 +jsonpath "$.detail" == "This resource requires authentication" + +# The problem document validates against the error schema named by its Link +POST {{base}}/self/v1/api/schemas/evaluate{{denied_schema}} +``` +{{denied_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# A key that opens another surface does not open this one +GET {{base}}/self/v1/metrics +Authorization: Bearer surface-secret-key +HTTP 401 +Cache-Control: no-store +Content-Type: application/problem+json +WWW-Authenticate: Bearer realm="registry" +Link: ; rel="describedby" +[Captures] +crossover_body: body +crossover_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:authentication-required" +jsonpath "$.status" == 401 + +POST {{base}}/self/v1/api/schemas/evaluate{{crossover_schema}} +``` +{{crossover_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# The metrics key admits, and the exposition answers +GET {{base}}/self/v1/metrics +Authorization: Bearer metrics-secret-key +HTTP 200 +Cache-Control: no-store +Content-Type: text/plain; version=0.0.4; charset=utf-8 +[Asserts] +header "Vary" not exists +header "Link" not exists +body matches /# TYPE sourcemeta_one_build_info gauge\nsourcemeta_one_build_info\{version="[^"]+",edition="enterprise"\} 1\n/ +body matches /# TYPE process_start_time_seconds gauge\nprocess_start_time_seconds [0-9.]+\n/ +body matches /# TYPE process_resident_memory_bytes gauge\nprocess_resident_memory_bytes [0-9]+\n/ +body matches /# TYPE process_open_fds gauge\nprocess_open_fds [0-9]+\n/ +body matches /# TYPE sourcemeta_one_http_requests_in_flight gauge\n/ +body matches /# TYPE sourcemeta_one_metrics_dropped_total counter\nsourcemeta_one_metrics_dropped_total 0\n/ +body matches /# TYPE sourcemeta_one_http_requests_total counter\n/ +body matches /sourcemeta_one_http_requests_total\{action="metrics_v1",code="401"\} [0-9]+\n/ +body not matches /le="1e-04"/ + +# A CORS preflight carries no credentials and is never gated +OPTIONS {{base}}/self/v1/metrics +Origin: http://example.com +Access-Control-Request-Method: GET +HTTP 204 +Cache-Control: no-store +Access-Control-Allow-Origin: * +Access-Control-Allow-Methods: GET, HEAD, OPTIONS +Access-Control-Allow-Headers: Accept, Accept-Encoding +Access-Control-Max-Age: 3600 +Allow: GET, HEAD, OPTIONS + +# The gate precedes method validation: an unsupported method denies without the +# key rather than revealing the method is wrong +POST {{base}}/self/v1/metrics +HTTP 401 +Cache-Control: no-store +Content-Type: application/problem+json +WWW-Authenticate: Bearer realm="registry" +Link: ; rel="describedby" +[Captures] +method_denied_body: body +method_denied_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:authentication-required" +jsonpath "$.status" == 401 + +POST {{base}}/self/v1/api/schemas/evaluate{{method_denied_schema}} +``` +{{method_denied_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# With the key the surface admits and the method check then answers 405 +POST {{base}}/self/v1/metrics +Authorization: Bearer metrics-secret-key +HTTP 405 +Cache-Control: no-store +Content-Type: application/problem+json +Allow: GET, HEAD, OPTIONS +Link: ; rel="describedby" +Access-Control-Allow-Origin: * +[Captures] +method_allowed_body: body +method_allowed_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:method-not-allowed" +jsonpath "$.title" == "Method Not Allowed" +jsonpath "$.status" == 405 + +POST {{base}}/self/v1/api/schemas/evaluate{{method_allowed_schema}} +``` +{{method_allowed_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# Whether a scrape succeeded is the scraper's own verdict on what it read, and +# it is zero for a target it could not reach, could not authenticate against, +# or could not parse. Nothing else here proves as much in a single number +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: up{job="sourcemeta-one"} +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.resultType" == "vector" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "1" + +# The scraper reports on its own target, which says the gate admitted it rather +# than merely that something was listening +GET http://localhost:9099/api/v1/targets +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +state: active +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.activeTargets" count == 1 +jsonpath "$.data.activeTargets[0].health" == "up" +jsonpath "$.data.activeTargets[0].lastError" == "" +jsonpath "$.data.activeTargets[0].scrapePool" == "sourcemeta-one" + +# An info metric survives the round trip with its labels intact, which is the +# whole reason it is shaped as a gauge whose value says nothing +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_build_info +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].metric.edition" == "enterprise" +jsonpath "$.data.result[0].metric.__name__" == "sourcemeta_one_build_info" +jsonpath "$.data.result[0].value[1]" == "1" + +# A counter carrying both of its labels, which is what makes the series +# addressable by the questions an operator actually asks. The refusals above +# are counted like anything else +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_http_requests_total{action="metrics_v1",code="401"} +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].metric.action" == "metrics_v1" +jsonpath "$.data.result[0].metric.code" == "401" + +# Asking for a percentile is what proves the buckets are cumulative, ordered, +# and closed by an infinite bound. A malformed histogram answers nothing here +# even though it parses perfectly well as text +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: histogram_quantile(0.95, sum by (le) (rate(sourcemeta_one_http_request_duration_seconds_bucket[1m]))) +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" exists + +# The count a histogram carries and the number of requests counted separately +# are two ways of saying the same thing, and they disagree if either is wrong +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sum(sourcemeta_one_http_request_duration_seconds_count) - sum(sourcemeta_one_http_requests_total) +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "0" + +# Nothing was lost on the way to being counted +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_metrics_dropped_total +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "0" + +# The process metrics use names shared across every language's client library, +# so a dashboard written against any of them finds these +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: process_resident_memory_bytes > 0 and process_max_fds > 0 and process_start_time_seconds > 0 +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 diff --git a/enterprise/e2e/auth/one.json b/enterprise/e2e/auth/one.json index 2764a9a6b..d9ed07e54 100644 --- a/enterprise/e2e/auth/one.json +++ b/enterprise/e2e/auth/one.json @@ -29,6 +29,13 @@ "paths": [ "/self/v1/health" ], "keys": [ { "environmentVariable": "ONE_E2E_KEY_SURFACE" } ] }, + { + "type": "apiKey", + "algorithm": "identity", + "name": "metrics", + "paths": [ "/self/v1/metrics" ], + "keys": [ { "environmentVariable": "ONE_E2E_KEY_METRICS" } ] + }, { "type": "apiKey", "algorithm": "identity", diff --git a/enterprise/e2e/html/prometheus.yml b/enterprise/e2e/auth/prometheus.yml similarity index 67% rename from enterprise/e2e/html/prometheus.yml rename to enterprise/e2e/auth/prometheus.yml index 6a3b493d1..7f42147b8 100644 --- a/enterprise/e2e/html/prometheus.yml +++ b/enterprise/e2e/auth/prometheus.yml @@ -8,6 +8,11 @@ global: scrape_configs: - job_name: sourcemeta-one metrics_path: /self/v1/metrics + # The path is governed by a policy, so the scraper presents a credential + # like any other caller + authorization: + type: Bearer + credentials: metrics-secret-key static_configs: - targets: - "sandbox:8001" diff --git a/enterprise/e2e/html/compose.yml b/enterprise/e2e/html/compose.yml index cd8628040..714999012 100644 --- a/enterprise/e2e/html/compose.yml +++ b/enterprise/e2e/html/compose.yml @@ -9,19 +9,3 @@ services: - SOURCEMETA_ONE_PORT=8001 ports: - "${PORT}:8001" - - # A real scraper rather than a reader of our own making, so what this - # instance publishes is judged by the software it claims to speak to - prometheus: - image: prom/prometheus:v3.1.0 - command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=1h - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - ports: - - "9099:9090" - depends_on: - sandbox: - condition: service_started diff --git a/enterprise/e2e/html/hurl/prometheus.compose.hurl b/enterprise/e2e/html/hurl/prometheus.compose.hurl deleted file mode 100644 index b89009640..000000000 --- a/enterprise/e2e/html/hurl/prometheus.compose.hurl +++ /dev/null @@ -1,130 +0,0 @@ -# Something for a scrape to have counted, so that the assertions below read a -# series that exists rather than one that happens not to yet -GET {{base}}/self/v1/health -HTTP 200 - -GET {{base}}/self/v1/api/list -HTTP 200 - -GET {{base}}/self/v1/metrics -HTTP 200 -Content-Type: text/plain; version=0.0.4; charset=utf-8 - -# Whether a scrape succeeded is the scraper's own verdict on what it read, and -# it is zero for a target it could not reach or could not parse. Nothing else -# here proves as much in a single number -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: up{job="sourcemeta-one"} -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.resultType" == "vector" -jsonpath "$.data.result" count == 1 -jsonpath "$.data.result[0].value[1]" == "1" - -# The scraper reports on its own targets, which says the endpoint answered -# rather than merely that something was listening -GET http://localhost:9099/api/v1/targets -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -state: active -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.activeTargets" count == 1 -jsonpath "$.data.activeTargets[0].health" == "up" -jsonpath "$.data.activeTargets[0].lastError" == "" -jsonpath "$.data.activeTargets[0].scrapePool" == "sourcemeta-one" -jsonpath "$.data.activeTargets[0].labels.job" == "sourcemeta-one" - -# An info metric survives the round trip with its labels intact, which is the -# whole reason it is shaped as a gauge whose value says nothing -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: sourcemeta_one_build_info -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.result" count == 1 -jsonpath "$.data.result[0].metric.edition" == "enterprise" -jsonpath "$.data.result[0].metric.__name__" == "sourcemeta_one_build_info" -jsonpath "$.data.result[0].value[1]" == "1" - -# A counter carrying both of its labels, which is what makes the series -# addressable by the questions an operator actually asks -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: sourcemeta_one_http_requests_total{action="health_check_v1",code="200"} -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.result" count == 1 -jsonpath "$.data.result[0].metric.action" == "health_check_v1" -jsonpath "$.data.result[0].metric.code" == "200" - -# Asking for a percentile is what proves the buckets are cumulative, ordered, -# and closed by an infinite bound. A malformed histogram answers nothing here -# even though it parses perfectly well as text -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: histogram_quantile(0.95, sum by (le) (rate(sourcemeta_one_http_request_duration_seconds_bucket[1m]))) -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.result" count == 1 -jsonpath "$.data.result[0].value[1]" exists - -# The count a histogram carries and the number of requests counted separately -# are two ways of saying the same thing, and they disagree if either is wrong -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: sum(sourcemeta_one_http_request_duration_seconds_count) - sum(sourcemeta_one_http_requests_total) -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.result" count == 1 -jsonpath "$.data.result[0].value[1]" == "0" - -# Nothing was lost on the way to being counted -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: sourcemeta_one_metrics_dropped_total -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.result" count == 1 -jsonpath "$.data.result[0].value[1]" == "0" - -# The process metrics use names shared across every language's client library, -# so a dashboard written against any of them finds these -GET http://localhost:9099/api/v1/query -[Options] -retry: 30 -retry-interval: 1000 -[QueryStringParams] -query: process_resident_memory_bytes > 0 and process_max_fds > 0 and process_start_time_seconds > 0 -HTTP 200 -[Asserts] -jsonpath "$.status" == "success" -jsonpath "$.data.result" count == 1 diff --git a/test/e2e/common.mk b/test/e2e/common.mk index dadebcc07..cc1522d17 100644 --- a/test/e2e/common.mk +++ b/test/e2e/common.mk @@ -16,7 +16,7 @@ export EDITION all: $(MAKE) down $(MAKE) up - $(MAKE) test-hurl test-hurl-compose test-playwright; \ + $(MAKE) test-hurl test-playwright; \ status=$$?; $(MAKE) down; exit $$status .PHONY: up @@ -32,17 +32,6 @@ test-hurl: $(wildcard hurl/*.all.hurl) \ $(wildcard hurl/*.$(EDITION).hurl) -# Assertions that read a service standing beside the registry, which only -# exists here. The native runner never invokes this target, so a sandbox can -# carry both kinds of test without the second kind failing where there is -# nothing beside it -.PHONY: test-hurl-compose -test-hurl-compose: -ifneq ($(wildcard hurl/*.compose.hurl),) - $(HURL) $(HURL_FLAGS) --repeat 10 --test --variable base=$(BASE):$(PORT) --variable port=$(PORT) \ - $(wildcard hurl/*.compose.hurl) -endif - .PHONY: test-playwright test-playwright: ifneq ($(wildcard playwright/),) From 1e790458762ee585d7bf1a001658d987c5bb47ad Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 10:23:41 -0300 Subject: [PATCH 4/7] Simpler Signed-off-by: Juan Cruz Viotti --- .../one/enterprise_server_action_metrics_v1.h | 26 ++----- src/actions/actions.cc | 4 +- src/actions/include/sourcemeta/one/actions.h | 78 +++++++++++-------- 3 files changed, 55 insertions(+), 53 deletions(-) diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h index 54005a958..10dfb86eb 100644 --- a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h +++ b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h @@ -64,18 +64,6 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { this->error_schema_ = std::get(value); } }); - - // A label value never changes for the life of an instance, so it is - // spelled once here rather than on every scrape - this->actions_.reserve(sourcemeta::one::ACTION_NAMES.size()); - for (const auto &name : sourcemeta::one::ACTION_NAMES) { - std::string lowered{name}; - std::ranges::transform(lowered, lowered.begin(), - [](const char character) -> char { - return sourcemeta::core::to_lowercase(character); - }); - this->actions_.push_back(std::move(lowered)); - } } auto rest(const std::span, @@ -295,11 +283,12 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { family(output, "sourcemeta_one_http_requests_total", "Total HTTP requests handled", "counter"); for (const auto &[key, count] : state.requests) { - output += std::format( - "sourcemeta_one_http_requests_total{{action=\"{}\"," - "code=\"{}\"}} {}\n", - this->actions_.at(sourcemeta::one::RouterMetrics::action_of(key)), - sourcemeta::one::RouterMetrics::status_of(key), count); + output += + std::format("sourcemeta_one_http_requests_total{{action=\"{}\"," + "code=\"{}\"}} {}\n", + sourcemeta::one::ACTION_NAMES.at( + sourcemeta::one::RouterMetrics::action_of(key)), + sourcemeta::one::RouterMetrics::status_of(key), count); } output += "\n"; @@ -316,7 +305,7 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { } cumulative = 0; - const auto &name{this->actions_.at(action)}; + const auto &name{sourcemeta::one::ACTION_NAMES.at(action)}; for (std::size_t bucket = 0; bucket < sourcemeta::one::RouterMetrics::BUCKET_COUNT; bucket++) { cumulative += state.buckets[action][bucket]; @@ -343,7 +332,6 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { } std::string_view error_schema_; - std::vector actions_; }; #endif diff --git a/src/actions/actions.cc b/src/actions/actions.cc index b46274894..7596fe6f2 100644 --- a/src/actions/actions.cc +++ b/src/actions/actions.cc @@ -40,7 +40,7 @@ struct ActionMetadata { bool open_world; }; -#define SOURCEMETA_ONE_DEFINE_METADATA(Name, Class) \ +#define SOURCEMETA_ONE_DEFINE_METADATA(Name, Class, Label) \ ActionMetadata{Class::DESCRIPTION, Class::READ_ONLY, Class::DESTRUCTIVE, \ Class::IDEMPOTENT, Class::OPEN_WORLD}, @@ -53,7 +53,7 @@ const std::array METADATA{ namespace sourcemeta::one { -#define SOURCEMETA_ONE_MAKE_CONSTRUCTOR_ENTRY(Name, Class) \ +#define SOURCEMETA_ONE_MAKE_CONSTRUCTOR_ENTRY(Name, Class, Label) \ table[ACTION_TYPE_##Name] = &make_router_action; const std::array CONSTRUCTORS{ diff --git a/src/actions/include/sourcemeta/one/actions.h b/src/actions/include/sourcemeta/one/actions.h index 5a03ab9ff..2db66e18f 100644 --- a/src/actions/include/sourcemeta/one/actions.h +++ b/src/actions/include/sourcemeta/one/actions.h @@ -9,35 +9,49 @@ namespace sourcemeta::one { +// The third column is what an action is called to anybody outside this +// program, which is a name this project promises rather than one it happens to +// use. It is written down rather than derived from the first, so that renaming +// a handler is an internal matter and renaming what an operator sees is a +// deliberate act #define SOURCEMETA_ONE_FOR_EACH_ACTION(X) \ - X(DEFAULT_V1, ActionDefault_v1) \ - X(HEALTH_CHECK_V1, ActionHealthCheck_v1) \ - X(METRICS_V1, ActionMetrics_v1) \ - X(NOT_FOUND_V1, ActionNotFound_v1) \ - X(SCHEMA_ARTIFACT_V1, ActionServeSchemaArtifact_v1) \ - X(EXPLORER_ARTIFACT_V1, ActionServeExplorerArtifact_v1) \ - X(GET_SCHEMA_HEALTH_V1, ActionGetSchemaHealth_v1) \ - X(GET_SCHEMA_LOCATIONS_V1, ActionGetSchemaLocations_v1) \ - X(GET_SCHEMA_POSITIONS_V1, ActionGetSchemaPositions_v1) \ - X(GET_SCHEMA_STATS_V1, ActionGetSchemaStats_v1) \ - X(GET_SCHEMA_METADATA_V1, ActionGetSchemaMetadata_v1) \ - X(LIST_DIRECTORY_V1, ActionListDirectory_v1) \ - X(DEPENDENCY_TREE_V1, ActionDependencyTree_v1) \ - X(GET_SCHEMA_DEPENDENCIES_V1, ActionGetSchemaDependencies_v1) \ - X(GET_SCHEMA_DEPENDENTS_V1, ActionGetSchemaDependents_v1) \ - X(JSONSCHEMA_EVALUATE_V1, ActionJSONSchemaEvaluate_v1) \ - X(JSONSCHEMA_RDF_V1, ActionJSONSchemaRDF_v1) \ - X(JSONSCHEMA_TRACE_V1, ActionJSONSchemaTrace_v1) \ - X(SCHEMA_SEARCH_V1, ActionSchemaSearch_v1) \ - X(SERVE_STATIC_V1, ActionServeStatic_v1) \ - X(MCP_V1, ActionMCP_v1) \ - X(AUTH_LOGOUT_V1, ActionAuthLogout_v1) \ - X(AUTH_LOGIN_V1, ActionAuthLogin_v1) \ - X(AUTH_LOGIN_PAGE_V1, ActionAuthLoginPage_v1) \ - X(AUTH_CALLBACK_V1, ActionAuthCallback_v1) \ - X(MCP_PROTECTED_RESOURCE_METADATA_V1, ActionMCPProtectedResourceMetadata_v1) - -#define SOURCEMETA_ONE_DEFINE_ACTION_TYPE(Name, Class) ACTION_TYPE_##Name, + X(DEFAULT_V1, ActionDefault_v1, "default_v1") \ + X(HEALTH_CHECK_V1, ActionHealthCheck_v1, "health_check_v1") \ + X(METRICS_V1, ActionMetrics_v1, "metrics_v1") \ + X(NOT_FOUND_V1, ActionNotFound_v1, "not_found_v1") \ + X(SCHEMA_ARTIFACT_V1, ActionServeSchemaArtifact_v1, "schema_artifact_v1") \ + X(EXPLORER_ARTIFACT_V1, ActionServeExplorerArtifact_v1, \ + "explorer_artifact_v1") \ + X(GET_SCHEMA_HEALTH_V1, ActionGetSchemaHealth_v1, "get_schema_health_v1") \ + X(GET_SCHEMA_LOCATIONS_V1, ActionGetSchemaLocations_v1, \ + "get_schema_locations_v1") \ + X(GET_SCHEMA_POSITIONS_V1, ActionGetSchemaPositions_v1, \ + "get_schema_positions_v1") \ + X(GET_SCHEMA_STATS_V1, ActionGetSchemaStats_v1, "get_schema_stats_v1") \ + X(GET_SCHEMA_METADATA_V1, ActionGetSchemaMetadata_v1, \ + "get_schema_metadata_v1") \ + X(LIST_DIRECTORY_V1, ActionListDirectory_v1, "list_directory_v1") \ + X(DEPENDENCY_TREE_V1, ActionDependencyTree_v1, "dependency_tree_v1") \ + X(GET_SCHEMA_DEPENDENCIES_V1, ActionGetSchemaDependencies_v1, \ + "get_schema_dependencies_v1") \ + X(GET_SCHEMA_DEPENDENTS_V1, ActionGetSchemaDependents_v1, \ + "get_schema_dependents_v1") \ + X(JSONSCHEMA_EVALUATE_V1, ActionJSONSchemaEvaluate_v1, \ + "jsonschema_evaluate_v1") \ + X(JSONSCHEMA_RDF_V1, ActionJSONSchemaRDF_v1, "jsonschema_rdf_v1") \ + X(JSONSCHEMA_TRACE_V1, ActionJSONSchemaTrace_v1, "jsonschema_trace_v1") \ + X(SCHEMA_SEARCH_V1, ActionSchemaSearch_v1, "schema_search_v1") \ + X(SERVE_STATIC_V1, ActionServeStatic_v1, "serve_static_v1") \ + X(MCP_V1, ActionMCP_v1, "mcp_v1") \ + X(AUTH_LOGOUT_V1, ActionAuthLogout_v1, "auth_logout_v1") \ + X(AUTH_LOGIN_V1, ActionAuthLogin_v1, "auth_login_v1") \ + X(AUTH_LOGIN_PAGE_V1, ActionAuthLoginPage_v1, "auth_login_page_v1") \ + X(AUTH_CALLBACK_V1, ActionAuthCallback_v1, "auth_callback_v1") \ + X(MCP_PROTECTED_RESOURCE_METADATA_V1, ActionMCPProtectedResourceMetadata_v1, \ + "mcp_protected_resource_metadata_v1") + +#define SOURCEMETA_ONE_DEFINE_ACTION_TYPE(Name, Class, Label) \ + ACTION_TYPE_##Name, enum : std::uint8_t { SOURCEMETA_ONE_FOR_EACH_ACTION(SOURCEMETA_ONE_DEFINE_ACTION_TYPE) @@ -46,11 +60,11 @@ enum : std::uint8_t { #undef SOURCEMETA_ONE_DEFINE_ACTION_TYPE -#define SOURCEMETA_ONE_DEFINE_ACTION_NAME(Name, Class) std::string_view{#Name}, +#define SOURCEMETA_ONE_DEFINE_ACTION_NAME(Name, Class, Label) \ + std::string_view{Label}, -// What an action is called wherever one has to be named to somebody outside -// this program, taken from the same list the handlers themselves come from so -// that neither can name an action the other does not have +// Indexed by the same values the enum above defines, since both come from the +// one list and neither can name an action the other does not have inline constexpr std::array ACTION_NAMES{ {SOURCEMETA_ONE_FOR_EACH_ACTION(SOURCEMETA_ONE_DEFINE_ACTION_NAME)}}; From 180634d2c599ccc88dd265d3505829dbb5715441 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 10:45:07 -0300 Subject: [PATCH 5/7] Simpler Signed-off-by: Juan Cruz Viotti --- .../one/enterprise_server_action_metrics_v1.h | 18 +- src/http/CMakeLists.txt | 2 +- src/http/include/sourcemeta/one/http.h | 1 + .../include/sourcemeta/one/http_helpers.h | 20 +- .../include/sourcemeta/one/http_metrics.h | 182 ++++++++++++++++++ .../include/sourcemeta/one/http_request.h | 54 +++--- src/router/CMakeLists.txt | 4 +- src/router/include/sourcemeta/one/router.h | 5 +- .../include/sourcemeta/one/router_metrics.h | 101 ---------- src/router/metrics.cc | 92 --------- src/router/router.cc | 15 +- src/server/server.cc | 13 -- 12 files changed, 237 insertions(+), 270 deletions(-) create mode 100644 src/http/include/sourcemeta/one/http_metrics.h delete mode 100644 src/router/include/sourcemeta/one/router_metrics.h delete mode 100644 src/router/metrics.cc diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h index 10dfb86eb..3abe4bbd8 100644 --- a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h +++ b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h @@ -48,7 +48,7 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { // it is decided here rather than by whatever a formatter does with the same // number on a given platform static constexpr std::array + sourcemeta::one::HTTPMetrics::BUCKET_COUNT> BOUNDARIES{{"0.0001", "0.00025", "0.0005", "0.001", "0.0025", "0.005", "0.01", "0.05", "0.25", "1.0"}}; @@ -282,13 +282,11 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { family(output, "sourcemeta_one_http_requests_total", "Total HTTP requests handled", "counter"); - for (const auto &[key, count] : state.requests) { - output += - std::format("sourcemeta_one_http_requests_total{{action=\"{}\"," - "code=\"{}\"}} {}\n", - sourcemeta::one::ACTION_NAMES.at( - sourcemeta::one::RouterMetrics::action_of(key)), - sourcemeta::one::RouterMetrics::status_of(key), count); + for (const auto &entry : state.requests) { + output += std::format("sourcemeta_one_http_requests_total{{action=\"{}\"," + "code=\"{}\"}} {}\n", + sourcemeta::one::ACTION_NAMES.at(entry.action), + entry.status, entry.count); } output += "\n"; @@ -307,7 +305,7 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { cumulative = 0; const auto &name{sourcemeta::one::ACTION_NAMES.at(action)}; for (std::size_t bucket = 0; - bucket < sourcemeta::one::RouterMetrics::BUCKET_COUNT; bucket++) { + bucket < sourcemeta::one::HTTPMetrics::BUCKET_COUNT; bucket++) { cumulative += state.buckets[action][bucket]; output += std::format("sourcemeta_one_http_request_duration_seconds_" "bucket{{action=\"{}\",le=\"{}\"}} {}\n", @@ -315,7 +313,7 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { } cumulative += - state.buckets[action][sourcemeta::one::RouterMetrics::BUCKET_COUNT]; + state.buckets[action][sourcemeta::one::HTTPMetrics::BUCKET_COUNT]; output += std::format("sourcemeta_one_http_request_duration_seconds_" "bucket{{action=\"{}\",le=\"+Inf\"}} {}\n", name, cumulative); diff --git a/src/http/CMakeLists.txt b/src/http/CMakeLists.txt index d192fc153..2581eb7ea 100644 --- a/src/http/CMakeLists.txt +++ b/src/http/CMakeLists.txt @@ -1,5 +1,5 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT one NAME http - PRIVATE_HEADERS uwebsockets.h request.h response.h helpers.h server.h) + PRIVATE_HEADERS uwebsockets.h request.h response.h metrics.h helpers.h server.h) target_link_libraries(sourcemeta_one_http INTERFACE sourcemeta::core::json) target_link_libraries(sourcemeta_one_http INTERFACE sourcemeta::core::time) diff --git a/src/http/include/sourcemeta/one/http.h b/src/http/include/sourcemeta/one/http.h index 87a288500..40ba14eb4 100644 --- a/src/http/include/sourcemeta/one/http.h +++ b/src/http/include/sourcemeta/one/http.h @@ -2,6 +2,7 @@ #define SOURCEMETA_ONE_HTTP_H #include +#include #include #include #include diff --git a/src/http/include/sourcemeta/one/http_helpers.h b/src/http/include/sourcemeta/one/http_helpers.h index 6a6896a5f..eb15224d2 100644 --- a/src/http/include/sourcemeta/one/http_helpers.h +++ b/src/http/include/sourcemeta/one/http_helpers.h @@ -195,22 +195,6 @@ inline auto request_body_too_large(const HTTPRequest &request) -> bool { return declared.has_value() && declared.value() > MAX_REQUEST_BODY_BYTES; } -// What is said about a request once it has been answered. Whoever stands the -// server up decides whether anything listens and what it does with what it -// hears, so this module never learns what an answer is counted for -using ResponseObserver = std::function; - -inline ResponseObserver RESPONSE_OBSERVER{}; - -inline auto observe_response(const sourcemeta::core::HTTPStatus &status, - const HTTPRequest &request) -> void { - if (RESPONSE_OBSERVER) [[unlikely]] { - RESPONSE_OBSERVER(request.observed_as(), status.code, - request.observed_from()); - } -} - // Answering can be the last thing that happens on a connection, and a request // read asynchronously is held by the very handler that goes away with it. So // what is said about a request is taken while it is certainly still there, @@ -222,7 +206,7 @@ inline auto send_response(const sourcemeta::core::HTTPStatus &status, std::format("{} {} {}", status.wire, request.method(), request.path())}; response.send_without_content(); HTTP_LOG(line); - observe_response(status, request); + request.observation().record(status.code); } inline auto send_response( @@ -236,7 +220,7 @@ inline auto send_response( response.send(request, message, current_encoding, precomputed_compressed_size); HTTP_LOG(line); - observe_response(status, request); + request.observation().record(status.code); } // RFC 9110 §9.3.7: OPTIONS responses describe communication options diff --git a/src/http/include/sourcemeta/one/http_metrics.h b/src/http/include/sourcemeta/one/http_metrics.h new file mode 100644 index 000000000..3e6f0345f --- /dev/null +++ b/src/http/include/sourcemeta/one/http_metrics.h @@ -0,0 +1,182 @@ +#ifndef SOURCEMETA_ONE_HTTP_METRICS_H +#define SOURCEMETA_ONE_HTTP_METRICS_H + +#include // std::ranges::lower_bound, std::max +#include // std::array +#include // std::atomic +#include // std::chrono::system_clock, std::chrono::duration +#include // std::size_t +#include // std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t +#include // std::distance +#include // std::map +#include // std::mutex, std::scoped_lock +#include // std::thread +#include // std::vector + +namespace sourcemeta::one { + +// What answering a request cost, kept as numbers and nothing else. What any of +// it is called, and whether an instance says any of it out loud, is decided by +// whoever reads this rather than here +class HTTPMetrics { +public: + static constexpr std::size_t BUCKET_COUNT{10}; + + // An order of magnitude finer at the low end than the usual boundaries, + // since answering out of a metapack takes well under a millisecond and the + // defaults would put nearly every request in the first bucket + static constexpr std::array BUCKETS{ + {0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.05, 0.25, 1.0}}; + + // How many requests were answered a given way, with what they were answered + // about and what they were answered with kept apart, since holding them + // together is this class's own business + struct Entry { + std::uint8_t action{0}; + std::uint16_t status{0}; + std::uint64_t count{0}; + }; + + // Everything counted so far, taken in one pass so that what is read cannot + // disagree with itself. Entries are ordered, since two readings of an + // unchanged instance should say the same thing in the same order + struct Snapshot { + std::uint64_t in_flight{0}; + std::uint64_t dropped{0}; + std::vector requests; + std::vector> buckets; + std::vector sums; + }; + + explicit HTTPMetrics(const std::size_t actions) + : actions_{actions}, + shards_{std::max(std::thread::hardware_concurrency(), 1)}, + started_{std::chrono::duration{ + std::chrono::system_clock::now().time_since_epoch()} + .count()} { + for (auto &shard : this->shards_) { + shard.buckets.assign(actions, {}); + shard.sums.assign(actions, 0.0); + } + } + + // To avoid mistakes + HTTPMetrics(const HTTPMetrics &) = delete; + HTTPMetrics(HTTPMetrics &&) = delete; + auto operator=(const HTTPMetrics &) -> HTTPMetrics & = delete; + auto operator=(HTTPMetrics &&) -> HTTPMetrics & = delete; + + // A request has reached a handler, which is what the in-flight count is the + // difference between + auto enter() noexcept -> void { + this->entered_.fetch_add(1, std::memory_order_relaxed); + } + + // A request has been answered, which is both what is counted and what brings + // the in-flight count back down + auto observe(const std::uint8_t action, const std::uint16_t status, + const double seconds) noexcept -> void { + this->answered_.fetch_add(1, std::memory_order_relaxed); + if (action >= this->actions_) [[unlikely]] { + return; + } + + const auto boundary{std::ranges::lower_bound(BUCKETS, seconds)}; + const auto bucket{ + static_cast(std::distance(BUCKETS.begin(), boundary))}; + + // Nothing said about a request may change how it was answered, and the + // answer has already gone out by the time this runs + try { + auto &target{this->shard()}; + const std::scoped_lock guard{target.mutex}; + target.requests[key(action, status)] += 1; + target.buckets[action][bucket] += 1; + target.sums[action] += seconds; + } catch (...) { + // A gap somebody can see is worth more than one they cannot, so what + // could not be recorded is counted and said in the answer itself + this->dropped_.fetch_add(1, std::memory_order_relaxed); + } + } + + [[nodiscard]] auto snapshot() const -> Snapshot { + Snapshot result; + result.dropped = this->dropped_.load(std::memory_order_relaxed); + const auto entered{this->entered_.load(std::memory_order_relaxed)}; + const auto answered{this->answered_.load(std::memory_order_relaxed)}; + result.in_flight = entered > answered ? entered - answered : 0; + result.buckets.assign(this->actions_, {}); + result.sums.assign(this->actions_, 0.0); + + std::map totals; + for (const auto &shard : this->shards_) { + const std::scoped_lock guard{shard.mutex}; + for (const auto &[entry, count] : shard.requests) { + totals[entry] += count; + } + + for (std::size_t action = 0; action < shard.buckets.size(); action++) { + for (std::size_t bucket = 0; bucket <= BUCKET_COUNT; bucket++) { + result.buckets[action][bucket] += shard.buckets[action][bucket]; + } + + result.sums[action] += shard.sums[action]; + } + } + + result.requests.reserve(totals.size()); + for (const auto &[entry, count] : totals) { + result.requests.push_back( + {.action = static_cast(entry >> 16U), + .status = static_cast(entry & 0xFFFFU), + .count = count}); + } + + return result; + } + + // When this instance began, as seconds since the Unix epoch + [[nodiscard]] auto started() const noexcept -> double { + return this->started_; + } + +private: + // A series is named by what was asked and what was answered together, so + // neither alone identifies one + [[nodiscard]] static constexpr auto key(const std::uint8_t action, + const std::uint16_t status) noexcept + -> std::uint32_t { + return (static_cast(action) << 16U) | status; + } + + // One of these per hardware thread, so that answering a request touches a + // line no other thread is writing to. Reading them is a scrape, which + // happens once every several seconds and can afford to visit each in turn + struct Shard { + mutable std::mutex mutex; + std::map requests; + std::vector> buckets; + std::vector sums; + }; + + // A thread takes the next place in line the first time it answers anything, + // and keeps it, so which line to touch costs nothing to work out afterwards + [[nodiscard]] auto shard() const noexcept -> Shard & { + static std::atomic next{0}; + thread_local const std::size_t assigned{ + next.fetch_add(1, std::memory_order_relaxed)}; + return this->shards_[assigned % this->shards_.size()]; + } + + std::size_t actions_; + mutable std::vector shards_; + std::atomic entered_{0}; + std::atomic answered_{0}; + std::atomic dropped_{0}; + double started_; +}; + +} // namespace sourcemeta::one + +#endif diff --git a/src/http/include/sourcemeta/one/http_request.h b/src/http/include/sourcemeta/one/http_request.h index ff4ddff49..027cf3d48 100644 --- a/src/http/include/sourcemeta/one/http_request.h +++ b/src/http/include/sourcemeta/one/http_request.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -12,7 +13,6 @@ #include // std::size_t #include // std::uint8_t #include // std::exception_ptr, std::current_exception -#include // std::numeric_limits #include // std::shared_ptr, std::make_shared #include // std::optional #include // std::string @@ -27,10 +27,26 @@ namespace sourcemeta::one { inline constexpr std::size_t MAX_REQUEST_BODY_BYTES{ static_cast(4) * 1024 * 1024}; -// A request answered before any handler was chosen, which is every refusal -// that happens ahead of routing -inline constexpr std::uint8_t OBSERVATION_UNROUTED{ - std::numeric_limits::max()}; +// What is remembered about a request so that whoever answers it can say what +// it cost without asking the router again. Nothing is counted until a handler +// is chosen, so a request refused ahead of routing carries no destination and +// is never counted +struct Observation { + std::chrono::steady_clock::time_point started{}; + HTTPMetrics *metrics{nullptr}; + std::uint8_t action{0}; + + auto record(const std::uint16_t status) const -> void { + if (this->metrics == nullptr) { + return; + } + + this->metrics->observe(this->action, status, + std::chrono::duration{ + std::chrono::steady_clock::now() - this->started} + .count()); + } +}; class HTTPRequest { public: @@ -60,26 +76,12 @@ class HTTPRequest { : sourcemeta::one::Encoding::Identity; } - // When this request reached a handler and which handler it reached, kept - // so that whoever answers can say both without asking the router again. The - // action is a value no handler owns until one is chosen, so a request - // answered before routing carries none - auto observe_from(const std::chrono::steady_clock::time_point value) noexcept - -> void { - this->observed_from_ = value; - } - - [[nodiscard]] auto observed_from() const noexcept - -> std::chrono::steady_clock::time_point { - return this->observed_from_; - } - - auto observe_as(const std::uint8_t value) noexcept -> void { - this->observed_as_ = value; + [[nodiscard]] auto observation() noexcept -> Observation & { + return this->observation_; } - [[nodiscard]] auto observed_as() const noexcept -> std::uint8_t { - return this->observed_as_; + [[nodiscard]] auto observation() const noexcept -> const Observation & { + return this->observation_; } [[nodiscard]] auto method() const noexcept -> std::string_view { @@ -179,8 +181,7 @@ class HTTPRequest { auto snapshot = std::make_shared( std::string{this->method()}, std::string{this->path()}, this->response_encoding_, raw_response); - snapshot->observe_from(this->observed_from_); - snapshot->observe_as(this->observed_as_); + snapshot->observation_ = this->observation_; auto buffer = std::make_shared(); auto completed = std::make_shared(false); @@ -235,8 +236,7 @@ class HTTPRequest { bool satisfiable_encoding_{true}; sourcemeta::one::Encoding response_encoding_{ sourcemeta::one::Encoding::Identity}; - std::chrono::steady_clock::time_point observed_from_{}; - std::uint8_t observed_as_{OBSERVATION_UNROUTED}; + Observation observation_{}; }; } // namespace sourcemeta::one diff --git a/src/router/CMakeLists.txt b/src/router/CMakeLists.txt index d341fff5c..c37b00583 100644 --- a/src/router/CMakeLists.txt +++ b/src/router/CMakeLists.txt @@ -1,6 +1,6 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT one NAME router - PRIVATE_HEADERS lru.h metrics.h - SOURCES artifact.cc router.cc evaluate.cc metrics.cc) + PRIVATE_HEADERS lru.h + SOURCES artifact.cc router.cc evaluate.cc) target_link_libraries(sourcemeta_one_router PUBLIC sourcemeta::one::authentication) target_link_libraries(sourcemeta_one_router PUBLIC sourcemeta::one::http) diff --git a/src/router/include/sourcemeta/one/router.h b/src/router/include/sourcemeta/one/router.h index 263f2c972..bb1e473b2 100644 --- a/src/router/include/sourcemeta/one/router.h +++ b/src/router/include/sourcemeta/one/router.h @@ -11,7 +11,6 @@ #include #include #include -#include #include // std::size_t #include // std::uint8_t @@ -398,7 +397,7 @@ class Router { return this->authentication_; } - [[nodiscard]] auto metrics() const noexcept -> RouterMetrics & { + [[nodiscard]] auto metrics() const noexcept -> HTTPMetrics & { return this->metrics_; } @@ -422,7 +421,7 @@ class Router { RouterLRU template_cache_{ TEMPLATE_CACHE_CAPACITY}; Authentication authentication_; - mutable RouterMetrics metrics_; + mutable HTTPMetrics metrics_; }; } // namespace sourcemeta::one diff --git a/src/router/include/sourcemeta/one/router_metrics.h b/src/router/include/sourcemeta/one/router_metrics.h deleted file mode 100644 index 028d489ea..000000000 --- a/src/router/include/sourcemeta/one/router_metrics.h +++ /dev/null @@ -1,101 +0,0 @@ -#ifndef SOURCEMETA_ONE_ROUTER_METRICS_H -#define SOURCEMETA_ONE_ROUTER_METRICS_H - -#include // std::array -#include // std::atomic -#include // std::size_t -#include // std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t -#include // std::map -#include // std::mutex -#include // std::vector - -namespace sourcemeta::one { - -// What a request cost, kept as numbers and nothing else. What any of it is -// called, and whether an instance says any of it out loud, is decided by -// whoever reads this rather than here -class RouterMetrics { -public: - static constexpr std::size_t BUCKET_COUNT{10}; - - // An order of magnitude finer at the low end than the usual boundaries, - // since answering out of a metapack takes well under a millisecond and the - // defaults would put nearly every request in the first bucket - static constexpr std::array BUCKETS{ - {0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.05, 0.25, 1.0}}; - - // Everything counted so far, taken in one pass so that what is read cannot - // disagree with itself. Requests are ordered, since two readings of an - // unchanged instance should say the same thing in the same order - struct Snapshot { - std::uint64_t in_flight{0}; - std::uint64_t dropped{0}; - std::map requests; - std::vector> buckets; - std::vector sums; - }; - - RouterMetrics(); - - // To avoid mistakes - RouterMetrics(const RouterMetrics &) = delete; - RouterMetrics(RouterMetrics &&) = delete; - auto operator=(const RouterMetrics &) -> RouterMetrics & = delete; - auto operator=(RouterMetrics &&) -> RouterMetrics & = delete; - - // How many handlers this instance has, which is every index that may ever be - // counted against - auto resize(std::size_t actions) -> void; - - // A request has reached a handler, which is what the in-flight count is the - // difference between - auto enter() noexcept -> void; - - // A request has been answered, which is both what is counted and what brings - // the in-flight count back down - auto observe(std::uint8_t action, std::uint16_t status, - double seconds) noexcept -> void; - - [[nodiscard]] auto snapshot() const -> Snapshot; - - // When this instance began, as seconds since the Unix epoch - [[nodiscard]] auto started() const noexcept -> double { - return this->started_; - } - - // How a request and what it was answered with are held together, since a - // series is named by both and neither alone identifies one - [[nodiscard]] static constexpr auto - action_of(const std::uint32_t key) noexcept -> std::size_t { - return static_cast(key >> 16U); - } - - [[nodiscard]] static constexpr auto - status_of(const std::uint32_t key) noexcept -> std::uint16_t { - return static_cast(key & 0xFFFFU); - } - -private: - // One of these per hardware thread, so that answering a request touches a - // line no other thread is writing to. Reading them is a scrape, which - // happens once every several seconds and can afford to visit each in turn - struct Shard { - mutable std::mutex mutex; - std::map requests; - std::vector> buckets; - std::vector sums; - }; - - [[nodiscard]] auto shard() const noexcept -> Shard &; - - std::size_t actions_{0}; - mutable std::vector shards_; - std::atomic entered_{0}; - std::atomic answered_{0}; - std::atomic dropped_{0}; - double started_; -}; - -} // namespace sourcemeta::one - -#endif diff --git a/src/router/metrics.cc b/src/router/metrics.cc deleted file mode 100644 index 272d8d84f..000000000 --- a/src/router/metrics.cc +++ /dev/null @@ -1,92 +0,0 @@ -#include - -#include // std::ranges::lower_bound, std::max -#include // std::chrono::system_clock, std::chrono::duration -#include // std::hash -#include // std::distance -#include // std::scoped_lock -#include // std::thread, std::this_thread - -namespace sourcemeta::one { - -RouterMetrics::RouterMetrics() - : shards_{std::max(std::thread::hardware_concurrency(), 1)}, - started_{std::chrono::duration{ - std::chrono::system_clock::now().time_since_epoch()} - .count()} {} - -auto RouterMetrics::resize(const std::size_t actions) -> void { - this->actions_ = actions; - for (auto &shard : this->shards_) { - const std::scoped_lock guard{shard.mutex}; - shard.buckets.assign(actions, {}); - shard.sums.assign(actions, 0.0); - } -} - -auto RouterMetrics::shard() const noexcept -> Shard & { - const auto index{std::hash{}(std::this_thread::get_id()) % - this->shards_.size()}; - return this->shards_[index]; -} - -auto RouterMetrics::enter() noexcept -> void { - this->entered_.fetch_add(1, std::memory_order_relaxed); -} - -auto RouterMetrics::observe(const std::uint8_t action, - const std::uint16_t status, - const double seconds) noexcept -> void { - this->answered_.fetch_add(1, std::memory_order_relaxed); - if (action >= this->actions_) [[unlikely]] { - return; - } - - const auto boundary{std::ranges::lower_bound(BUCKETS, seconds)}; - const auto bucket{ - static_cast(std::distance(BUCKETS.begin(), boundary))}; - - // Nothing said about a request may change how it was answered, and the - // answer has already gone out by the time this runs - try { - auto &target{this->shard()}; - const std::scoped_lock guard{target.mutex}; - const auto key{(static_cast(action) << 16U) | status}; - target.requests[key] += 1; - target.buckets[action][bucket] += 1; - target.sums[action] += seconds; - } catch (...) { - // A gap somebody can see is worth more than one they cannot, so what could - // not be recorded is counted and said in the answer itself - this->dropped_.fetch_add(1, std::memory_order_relaxed); - } -} - -auto RouterMetrics::snapshot() const -> Snapshot { - Snapshot result; - result.dropped = this->dropped_.load(std::memory_order_relaxed); - const auto entered{this->entered_.load(std::memory_order_relaxed)}; - const auto answered{this->answered_.load(std::memory_order_relaxed)}; - result.in_flight = entered > answered ? entered - answered : 0; - result.buckets.assign(this->actions_, {}); - result.sums.assign(this->actions_, 0.0); - - for (const auto &shard : this->shards_) { - const std::scoped_lock guard{shard.mutex}; - for (const auto &[key, count] : shard.requests) { - result.requests[key] += count; - } - - for (std::size_t action = 0; action < shard.buckets.size(); action++) { - for (std::size_t bucket = 0; bucket <= BUCKET_COUNT; bucket++) { - result.buckets[action][bucket] += shard.buckets[action][bucket]; - } - - result.sums[action] += shard.sums[action]; - } - } - - return result; -} - -} // namespace sourcemeta::one diff --git a/src/router/router.cc b/src/router/router.cc index fcf3c510f..80e4db266 100644 --- a/src/router/router.cc +++ b/src/router/router.cc @@ -3,7 +3,7 @@ #include #include -#include // std::chrono::seconds +#include // std::chrono::seconds, std::chrono::steady_clock #include // std::uint8_t #include // std::make_unique #include // std::call_once @@ -70,7 +70,8 @@ Router::Router(const std::filesystem::path &base, slots_size_{router.size() + 1}, authentication_{ sourcemeta::one::Authentication::Table{base / "authentication.bin"}, - provider_fetcher()} { + provider_fetcher()}, + metrics_{constructors.size()} { router.arguments(0, [this](const auto &key, const auto &value) -> void { if (key == "errorSchema") { this->default_error_schema_ = std::get(value); @@ -120,8 +121,16 @@ auto Router::dispatch( const std::span matches, sourcemeta::one::HTTPRequest &request, sourcemeta::one::HTTPResponse &response) -> void { - request.observe_as(static_cast(context)); +#if defined(SOURCEMETA_ONE_ENTERPRISE) + // Everything an answer is counted against is known here and nowhere earlier, + // so it is taken here rather than gathered along the way + auto &observation{request.observation()}; + observation.started = std::chrono::steady_clock::now(); + observation.action = static_cast(context); + observation.metrics = &this->metrics_; this->metrics_.enter(); +#endif + auto *instance{this->action(identifier, context)}; if (instance == nullptr) [[unlikely]] { this->error(request, response, diff --git a/src/server/server.cc b/src/server/server.cc index 5ee65abb9..a8d561bc2 100644 --- a/src/server/server.cc +++ b/src/server/server.cc @@ -24,7 +24,6 @@ static auto dispatch(sourcemeta::one::Router &actions, sourcemeta::one::HTTPRequest &request, sourcemeta::one::HTTPResponse &response) noexcept -> void { try { - request.observe_from(std::chrono::steady_clock::now()); request.negotiate(); if (request.satisfiable_encoding()) { thread_local std::array< @@ -113,18 +112,6 @@ auto main(int argc, char *argv[]) noexcept -> int { sourcemeta::one::Router actions{base, router, sourcemeta::one::CONSTRUCTORS}; -#if defined(SOURCEMETA_ONE_ENTERPRISE) - actions.metrics().resize(sourcemeta::one::ACTION_TYPE_COUNT); - sourcemeta::one::RESPONSE_OBSERVER = - [&actions](const std::uint8_t action, const std::uint16_t status, - const std::chrono::steady_clock::time_point from) -> void { - actions.metrics().observe( - action, status, - std::chrono::duration{std::chrono::steady_clock::now() - from} - .count()); - }; -#endif - const sourcemeta::one::HTTPServer server{ port, [&actions, &router](sourcemeta::one::HTTPRequest &request, From 94ba8725ec7fd7e1c82b27d72d6e67c0024068e8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 11:10:46 -0300 Subject: [PATCH 6/7] Encapsulate Signed-off-by: Juan Cruz Viotti --- .../one/enterprise_server_action_metrics_v1.h | 4 +- .../include/sourcemeta/one/http_helpers.h | 1 - .../include/sourcemeta/one/http_metrics.h | 118 +++++++++++------- .../include/sourcemeta/one/http_request.h | 22 ++-- src/router/include/sourcemeta/one/router.h | 5 - src/router/router.cc | 19 ++- 6 files changed, 90 insertions(+), 79 deletions(-) diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h index 3abe4bbd8..2409d82ec 100644 --- a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h +++ b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h @@ -223,7 +223,7 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { } [[nodiscard]] auto serialize() const -> std::string { - const auto &metrics{this->dispatcher().metrics()}; + const auto &metrics{sourcemeta::one::http_metrics()}; const auto state{metrics.snapshot()}; const auto process{read_process()}; @@ -285,7 +285,7 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { for (const auto &entry : state.requests) { output += std::format("sourcemeta_one_http_requests_total{{action=\"{}\"," "code=\"{}\"}} {}\n", - sourcemeta::one::ACTION_NAMES.at(entry.action), + sourcemeta::one::ACTION_NAMES.at(entry.handler), entry.status, entry.count); } diff --git a/src/http/include/sourcemeta/one/http_helpers.h b/src/http/include/sourcemeta/one/http_helpers.h index eb15224d2..2934c0a77 100644 --- a/src/http/include/sourcemeta/one/http_helpers.h +++ b/src/http/include/sourcemeta/one/http_helpers.h @@ -18,7 +18,6 @@ #include // std::size_t #include // std::uint8_t, std::uint16_t #include // std::format -#include // std::function #include // std::mutex, std::scoped_lock #include // std::optional #include // std::print diff --git a/src/http/include/sourcemeta/one/http_metrics.h b/src/http/include/sourcemeta/one/http_metrics.h index 3e6f0345f..d26dc8679 100644 --- a/src/http/include/sourcemeta/one/http_metrics.h +++ b/src/http/include/sourcemeta/one/http_metrics.h @@ -1,7 +1,7 @@ #ifndef SOURCEMETA_ONE_HTTP_METRICS_H #define SOURCEMETA_ONE_HTTP_METRICS_H -#include // std::ranges::lower_bound, std::max +#include // std::ranges::lower_bound #include // std::array #include // std::atomic #include // std::chrono::system_clock, std::chrono::duration @@ -10,36 +10,36 @@ #include // std::distance #include // std::map #include // std::mutex, std::scoped_lock -#include // std::thread #include // std::vector namespace sourcemeta::one { -// What answering a request cost, kept as numbers and nothing else. What any of -// it is called, and whether an instance says any of it out loud, is decided by -// whoever reads this rather than here +// What serving requests cost, kept as numbers and nothing else. +// +// A request is counted against a handler, which is a number this class never +// interprets. What a handler is, what it is called, and whether any of this is +// ever said out loud are all decided elsewhere, which is what lets this stay +// true of any server rather than of this one class HTTPMetrics { public: static constexpr std::size_t BUCKET_COUNT{10}; // An order of magnitude finer at the low end than the usual boundaries, - // since answering out of a metapack takes well under a millisecond and the + // since answering out of a file takes well under a millisecond and the // defaults would put nearly every request in the first bucket static constexpr std::array BUCKETS{ {0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.05, 0.25, 1.0}}; - // How many requests were answered a given way, with what they were answered - // about and what they were answered with kept apart, since holding them - // together is this class's own business + // How many requests one handler answered one way struct Entry { - std::uint8_t action{0}; + std::uint8_t handler{0}; std::uint16_t status{0}; std::uint64_t count{0}; }; // Everything counted so far, taken in one pass so that what is read cannot // disagree with itself. Entries are ordered, since two readings of an - // unchanged instance should say the same thing in the same order + // unchanged server should say the same thing in the same order struct Snapshot { std::uint64_t in_flight{0}; std::uint64_t dropped{0}; @@ -48,17 +48,7 @@ class HTTPMetrics { std::vector sums; }; - explicit HTTPMetrics(const std::size_t actions) - : actions_{actions}, - shards_{std::max(std::thread::hardware_concurrency(), 1)}, - started_{std::chrono::duration{ - std::chrono::system_clock::now().time_since_epoch()} - .count()} { - for (auto &shard : this->shards_) { - shard.buckets.assign(actions, {}); - shard.sums.assign(actions, 0.0); - } - } + HTTPMetrics() = default; // To avoid mistakes HTTPMetrics(const HTTPMetrics &) = delete; @@ -66,18 +56,35 @@ class HTTPMetrics { auto operator=(const HTTPMetrics &) -> HTTPMetrics & = delete; auto operator=(HTTPMetrics &&) -> HTTPMetrics & = delete; - // A request has reached a handler, which is what the in-flight count is the - // difference between + // The server is coming up, with this many handlers it may ever count + // against, which whoever owns them is the only one to know. Nothing is + // counted before this is said + auto start(const std::size_t handlers) -> void { + this->handlers_ = handlers; + this->started_ = + std::chrono::duration{ + std::chrono::system_clock::now().time_since_epoch()} + .count(); + for (auto &shard : this->shards_) { + const std::scoped_lock guard{shard.mutex}; + shard.buckets.assign(handlers, {}); + shard.sums.assign(handlers, 0.0); + } + } + + // A request has arrived, which is what the in-flight count is the difference + // between auto enter() noexcept -> void { this->entered_.fetch_add(1, std::memory_order_relaxed); } // A request has been answered, which is both what is counted and what brings - // the in-flight count back down - auto observe(const std::uint8_t action, const std::uint16_t status, + // the in-flight count back down. A request answered before any handler was + // chosen carries none, and is counted as served without being attributed + auto observe(const std::uint8_t handler, const std::uint16_t status, const double seconds) noexcept -> void { this->answered_.fetch_add(1, std::memory_order_relaxed); - if (action >= this->actions_) [[unlikely]] { + if (handler >= this->handlers_) [[unlikely]] { return; } @@ -90,9 +97,9 @@ class HTTPMetrics { try { auto &target{this->shard()}; const std::scoped_lock guard{target.mutex}; - target.requests[key(action, status)] += 1; - target.buckets[action][bucket] += 1; - target.sums[action] += seconds; + target.requests[key(handler, status)] += 1; + target.buckets[handler][bucket] += 1; + target.sums[handler] += seconds; } catch (...) { // A gap somebody can see is worth more than one they cannot, so what // could not be recorded is counted and said in the answer itself @@ -106,8 +113,8 @@ class HTTPMetrics { const auto entered{this->entered_.load(std::memory_order_relaxed)}; const auto answered{this->answered_.load(std::memory_order_relaxed)}; result.in_flight = entered > answered ? entered - answered : 0; - result.buckets.assign(this->actions_, {}); - result.sums.assign(this->actions_, 0.0); + result.buckets.assign(this->handlers_, {}); + result.sums.assign(this->handlers_, 0.0); std::map totals; for (const auto &shard : this->shards_) { @@ -116,19 +123,19 @@ class HTTPMetrics { totals[entry] += count; } - for (std::size_t action = 0; action < shard.buckets.size(); action++) { + for (std::size_t handler = 0; handler < shard.buckets.size(); handler++) { for (std::size_t bucket = 0; bucket <= BUCKET_COUNT; bucket++) { - result.buckets[action][bucket] += shard.buckets[action][bucket]; + result.buckets[handler][bucket] += shard.buckets[handler][bucket]; } - result.sums[action] += shard.sums[action]; + result.sums[handler] += shard.sums[handler]; } } result.requests.reserve(totals.size()); for (const auto &[entry, count] : totals) { result.requests.push_back( - {.action = static_cast(entry >> 16U), + {.handler = static_cast(entry >> 16U), .status = static_cast(entry & 0xFFFFU), .count = count}); } @@ -136,23 +143,29 @@ class HTTPMetrics { return result; } - // When this instance began, as seconds since the Unix epoch + // When this server began, as seconds since the Unix epoch [[nodiscard]] auto started() const noexcept -> double { return this->started_; } private: - // A series is named by what was asked and what was answered together, so + // A series is named by what answered and what it answered together, so // neither alone identifies one - [[nodiscard]] static constexpr auto key(const std::uint8_t action, + [[nodiscard]] static constexpr auto key(const std::uint8_t handler, const std::uint16_t status) noexcept -> std::uint32_t { - return (static_cast(action) << 16U) | status; + return (static_cast(handler) << 16U) | status; } - // One of these per hardware thread, so that answering a request touches a - // line no other thread is writing to. Reading them is a scrape, which - // happens once every several seconds and can afford to visit each in turn + // Enough that threads rarely share one, and a fixed number rather than one + // per hardware thread so that building this allocates nothing and can + // therefore happen before anything else does. Threads beyond this many + // share, which the lock already accounts for + static constexpr std::size_t SHARD_COUNT{16}; + + // Answering a request touches a line no other thread is writing to. Reading + // them is a scrape, which happens once every several seconds and can afford + // to visit each in turn struct Shard { mutable std::mutex mutex; std::map requests; @@ -166,17 +179,28 @@ class HTTPMetrics { static std::atomic next{0}; thread_local const std::size_t assigned{ next.fetch_add(1, std::memory_order_relaxed)}; - return this->shards_[assigned % this->shards_.size()]; + return this->shards_[assigned % SHARD_COUNT]; } - std::size_t actions_; - mutable std::vector shards_; + std::size_t handlers_{0}; + mutable std::array shards_; std::atomic entered_{0}; std::atomic answered_{0}; std::atomic dropped_{0}; - double started_; + double started_{0}; }; +// What this process has served, which there is one of because there is one +// server. It lives here rather than on the server itself, since what a request +// is worth is read where a request is answered and that is not there. It is +// built before anything runs, so reaching it never costs a check +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +inline HTTPMetrics HTTP_METRICS; + +[[nodiscard]] inline auto http_metrics() noexcept -> HTTPMetrics & { + return HTTP_METRICS; +} + } // namespace sourcemeta::one #endif diff --git a/src/http/include/sourcemeta/one/http_request.h b/src/http/include/sourcemeta/one/http_request.h index 027cf3d48..b82284642 100644 --- a/src/http/include/sourcemeta/one/http_request.h +++ b/src/http/include/sourcemeta/one/http_request.h @@ -11,8 +11,9 @@ #include // std::chrono::system_clock, std::chrono::steady_clock #include // std::invocable #include // std::size_t -#include // std::uint8_t +#include // std::uint8_t, std::uint16_t #include // std::exception_ptr, std::current_exception +#include // std::numeric_limits #include // std::shared_ptr, std::make_shared #include // std::optional #include // std::string @@ -28,20 +29,14 @@ inline constexpr std::size_t MAX_REQUEST_BODY_BYTES{ static_cast(4) * 1024 * 1024}; // What is remembered about a request so that whoever answers it can say what -// it cost without asking the router again. Nothing is counted until a handler -// is chosen, so a request refused ahead of routing carries no destination and -// is never counted +// it cost. Which handler answered is a value nothing owns until one is chosen, +// so a request refused ahead of routing carries none and goes unattributed struct Observation { std::chrono::steady_clock::time_point started{}; - HTTPMetrics *metrics{nullptr}; - std::uint8_t action{0}; + std::uint8_t handler{std::numeric_limits::max()}; auto record(const std::uint16_t status) const -> void { - if (this->metrics == nullptr) { - return; - } - - this->metrics->observe(this->action, status, + http_metrics().observe(this->handler, status, std::chrono::duration{ std::chrono::steady_clock::now() - this->started} .count()); @@ -53,7 +48,10 @@ class HTTPRequest { // Primary constructor from raw uWebSockets pointers HTTPRequest(uWS::HttpRequest *request, uWS::HttpResponse *response) noexcept - : request_{request}, response_{response} {} + : request_{request}, response_{response} { + this->observation_.started = std::chrono::steady_clock::now(); + http_metrics().enter(); + } // Snapshot constructor for async contexts where uWS::HttpRequest is gone HTTPRequest(std::string method, std::string path, diff --git a/src/router/include/sourcemeta/one/router.h b/src/router/include/sourcemeta/one/router.h index bb1e473b2..05dff6da2 100644 --- a/src/router/include/sourcemeta/one/router.h +++ b/src/router/include/sourcemeta/one/router.h @@ -397,10 +397,6 @@ class Router { return this->authentication_; } - [[nodiscard]] auto metrics() const noexcept -> HTTPMetrics & { - return this->metrics_; - } - private: static constexpr std::size_t TEMPLATE_CACHE_CAPACITY{50}; @@ -421,7 +417,6 @@ class Router { RouterLRU template_cache_{ TEMPLATE_CACHE_CAPACITY}; Authentication authentication_; - mutable HTTPMetrics metrics_; }; } // namespace sourcemeta::one diff --git a/src/router/router.cc b/src/router/router.cc index 80e4db266..617ed29a4 100644 --- a/src/router/router.cc +++ b/src/router/router.cc @@ -3,7 +3,7 @@ #include #include -#include // std::chrono::seconds, std::chrono::steady_clock +#include // std::chrono::seconds #include // std::uint8_t #include // std::make_unique #include // std::call_once @@ -70,8 +70,9 @@ Router::Router(const std::filesystem::path &base, slots_size_{router.size() + 1}, authentication_{ sourcemeta::one::Authentication::Table{base / "authentication.bin"}, - provider_fetcher()}, - metrics_{constructors.size()} { + provider_fetcher()} { + // Only whoever holds the handler table knows how many there can be + sourcemeta::one::http_metrics().start(constructors.size()); router.arguments(0, [this](const auto &key, const auto &value) -> void { if (key == "errorSchema") { this->default_error_schema_ = std::get(value); @@ -121,15 +122,9 @@ auto Router::dispatch( const std::span matches, sourcemeta::one::HTTPRequest &request, sourcemeta::one::HTTPResponse &response) -> void { -#if defined(SOURCEMETA_ONE_ENTERPRISE) - // Everything an answer is counted against is known here and nowhere earlier, - // so it is taken here rather than gathered along the way - auto &observation{request.observation()}; - observation.started = std::chrono::steady_clock::now(); - observation.action = static_cast(context); - observation.metrics = &this->metrics_; - this->metrics_.enter(); -#endif + // Which handler answers is the one thing about a request that only routing + // knows, so it is the one thing said here + request.observation().handler = static_cast(context); auto *instance{this->action(identifier, context)}; if (instance == nullptr) [[unlikely]] { From 7e1de02a0d9ff3513af4db39cc902897b8102411 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 12:08:13 -0300 Subject: [PATCH 7/7] Fix Signed-off-by: Juan Cruz Viotti --- docs/api.md | 3 ++- .../one/enterprise_server_action_metrics_v1.h | 7 ++++- src/actions/include/sourcemeta/one/actions.h | 8 ++++-- .../include/sourcemeta/one/http_metrics.h | 7 +++++ .../include/sourcemeta/one/http_request.h | 26 +++++++++++++++++-- src/http/include/sourcemeta/one/http_server.h | 3 +++ 6 files changed, 48 insertions(+), 6 deletions(-) diff --git a/docs/api.md b/docs/api.md index df8d30ace..55163f1ec 100644 --- a/docs/api.md +++ b/docs/api.md @@ -68,7 +68,8 @@ GET /self/v1/metrics Any Prometheus-compatible scraper reads this without configuration beyond the path. Request behaviour follows the [RED](https://grafana.com/blog/2018/08/02/the-red-method-how-to-instrument-your-services/) -method, meaning rate, errors and duration, reported per action and status code. +method, with request and error counts reported per action and status code, and +duration reported per action. This endpoint is not exempt from [authentication](#authentication), so a policy covering its path gates it like any other route. diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h index 2409d82ec..77492b71c 100644 --- a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h +++ b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h @@ -195,14 +195,19 @@ class ActionMetrics_v1 : public sourcemeta::one::RouterAction { } } + // Reading the list takes a descriptor of its own, which the list then + // includes, so what is counted is one more than what was open std::error_code error; const std::filesystem::directory_iterator descriptors{"/proc/self/fd", error}; if (!error) { + std::uint64_t listed{0}; for (const auto &entry : descriptors) { static_cast(entry); - sample.open_descriptors += 1; + listed += 1; } + + sample.open_descriptors = listed > 0 ? listed - 1 : 0; } sample.maximum_descriptors = descriptor_limit(); diff --git a/src/actions/include/sourcemeta/one/actions.h b/src/actions/include/sourcemeta/one/actions.h index 2db66e18f..f6ad62852 100644 --- a/src/actions/include/sourcemeta/one/actions.h +++ b/src/actions/include/sourcemeta/one/actions.h @@ -9,6 +9,10 @@ namespace sourcemeta::one { +// New entries are appended rather than inserted, since the position of an +// entry is the identifier a built router records for it, and an identifier +// that moves points an already-built router at the wrong handler. +// // The third column is what an action is called to anybody outside this // program, which is a name this project promises rather than one it happens to // use. It is written down rather than derived from the first, so that renaming @@ -17,7 +21,6 @@ namespace sourcemeta::one { #define SOURCEMETA_ONE_FOR_EACH_ACTION(X) \ X(DEFAULT_V1, ActionDefault_v1, "default_v1") \ X(HEALTH_CHECK_V1, ActionHealthCheck_v1, "health_check_v1") \ - X(METRICS_V1, ActionMetrics_v1, "metrics_v1") \ X(NOT_FOUND_V1, ActionNotFound_v1, "not_found_v1") \ X(SCHEMA_ARTIFACT_V1, ActionServeSchemaArtifact_v1, "schema_artifact_v1") \ X(EXPLORER_ARTIFACT_V1, ActionServeExplorerArtifact_v1, \ @@ -48,7 +51,8 @@ namespace sourcemeta::one { X(AUTH_LOGIN_PAGE_V1, ActionAuthLoginPage_v1, "auth_login_page_v1") \ X(AUTH_CALLBACK_V1, ActionAuthCallback_v1, "auth_callback_v1") \ X(MCP_PROTECTED_RESOURCE_METADATA_V1, ActionMCPProtectedResourceMetadata_v1, \ - "mcp_protected_resource_metadata_v1") + "mcp_protected_resource_metadata_v1") \ + X(METRICS_V1, ActionMetrics_v1, "metrics_v1") #define SOURCEMETA_ONE_DEFINE_ACTION_TYPE(Name, Class, Label) \ ACTION_TYPE_##Name, diff --git a/src/http/include/sourcemeta/one/http_metrics.h b/src/http/include/sourcemeta/one/http_metrics.h index d26dc8679..c1a6226d8 100644 --- a/src/http/include/sourcemeta/one/http_metrics.h +++ b/src/http/include/sourcemeta/one/http_metrics.h @@ -78,6 +78,13 @@ class HTTPMetrics { this->entered_.fetch_add(1, std::memory_order_relaxed); } + // A request has gone without ever being answered, which a caller that hung + // up mid-upload does. Nothing is counted against a handler, since nothing + // was served, but the in-flight count comes back down all the same + auto abandon() noexcept -> void { + this->answered_.fetch_add(1, std::memory_order_relaxed); + } + // A request has been answered, which is both what is counted and what brings // the in-flight count back down. A request answered before any handler was // chosen carries none, and is counted as served without being attributed diff --git a/src/http/include/sourcemeta/one/http_request.h b/src/http/include/sourcemeta/one/http_request.h index b82284642..d7f7ba9a2 100644 --- a/src/http/include/sourcemeta/one/http_request.h +++ b/src/http/include/sourcemeta/one/http_request.h @@ -34,13 +34,33 @@ inline constexpr std::size_t MAX_REQUEST_BODY_BYTES{ struct Observation { std::chrono::steady_clock::time_point started{}; std::uint8_t handler{std::numeric_limits::max()}; + // A request arrives once and leaves once, but there is more than one way for + // it to leave and more than one place that notices. Settling here rather + // than at each of them is what keeps the in-flight count honest + mutable bool settled{false}; auto record(const std::uint16_t status) const -> void { + if (this->settled) { + return; + } + + this->settled = true; http_metrics().observe(this->handler, status, std::chrono::duration{ std::chrono::steady_clock::now() - this->started} .count()); } + + // Nothing was served, so nothing is counted against a handler, but the + // in-flight count comes back down all the same + auto abandon() const -> void { + if (this->settled) { + return; + } + + this->settled = true; + http_metrics().abandon(); + } }; class HTTPRequest { @@ -183,8 +203,10 @@ class HTTPRequest { auto buffer = std::make_shared(); auto completed = std::make_shared(false); - raw_response->onAborted( - [completed]() mutable -> void { *completed = true; }); + raw_response->onAborted([completed, snapshot]() mutable -> void { + *completed = true; + snapshot->observation_.abandon(); + }); raw_response->onData( // NOLINTNEXTLINE(bugprone-exception-escape) diff --git a/src/http/include/sourcemeta/one/http_server.h b/src/http/include/sourcemeta/one/http_server.h index 55b544ba5..3ad7b394d 100644 --- a/src/http/include/sourcemeta/one/http_server.h +++ b/src/http/include/sourcemeta/one/http_server.h @@ -111,6 +111,9 @@ class HTTPServer { response.write_status( sourcemeta::core::HTTP_STATUS_INTERNAL_SERVER_ERROR); response.send_without_content(); + request.observation().record( + sourcemeta::core::HTTP_STATUS_INTERNAL_SERVER_ERROR + .code); } });