diff --git a/docs/api.md b/docs/api.md index b823fd002..55163f1ec 100644 --- a/docs/api.md +++ b/docs/api.md @@ -51,6 +51,40 @@ GET /self/v1/health The HTTP method is not `GET` or `HEAD`. +### Metrics + +!!! success "Enterprise" + + This endpoint is only available in the [Enterprise](commercial.md) + edition. Learn more about [commercial licensing](commercial.md). + +*This endpoint reports instance telemetry in the [Prometheus exposition +format](https://prometheus.io/docs/instrumenting/exposition_formats/).* + +``` +GET /self/v1/metrics +``` + +Any Prometheus-compatible scraper reads this without configuration beyond the +path. Request behaviour follows the +[RED](https://grafana.com/blog/2018/08/02/the-red-method-how-to-instrument-your-services/) +method, with request and error counts reported per action and status code, and +duration reported per action. +This endpoint is not exempt from [authentication](#authentication), so a policy +covering its path gates it like any other route. + +=== "200" + + The metrics in the Prometheus exposition format. + +=== "403" + + The instance is running the Community edition. + +=== "405" + + The HTTP method is not `GET` or `HEAD`. + ### List *This endpoint lists the contents of a directory at the specified `{path}` diff --git a/enterprise/e2e/auth/compose.yml b/enterprise/e2e/auth/compose.yml index d5f21755a..925712ae6 100644 --- a/enterprise/e2e/auth/compose.yml +++ b/enterprise/e2e/auth/compose.yml @@ -45,3 +45,21 @@ services: condition: service_started ports: - "${PORT}:8001" + + # A real scraper rather than a reader of our own making, so what this + # instance publishes is judged by the software it claims to speak to. It + # scrapes a gated path, so it also proves a policy admits a scraper holding + # the right credential and nothing else + prometheus: + image: prom/prometheus:v3.1.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=1h + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9099:9090" + depends_on: + sandbox: + condition: service_started diff --git a/enterprise/e2e/auth/environment b/enterprise/e2e/auth/environment index 2cd8feba7..f6ac4b9a4 100644 --- a/enterprise/e2e/auth/environment +++ b/enterprise/e2e/auth/environment @@ -2,6 +2,7 @@ ONE_E2E_KEY_PRIMARY=primary-secret-key ONE_E2E_KEY_SECONDARY=secondary-secret-key ONE_E2E_KEY_INTERNAL=internal-secret-key ONE_E2E_KEY_SURFACE=surface-secret-key +ONE_E2E_KEY_METRICS=metrics-secret-key ONE_E2E_KEY_MIXED_PLAIN=mixed-plain-key ONE_E2E_KEY_MIXED_HASHED=2508e2ba355946ba6065f01eb183424aaea10ddaf7bbff90e38f3b702e89e23d ONE_E2E_KEY_BLANK= diff --git a/enterprise/e2e/auth/hurl/metrics.all.hurl b/enterprise/e2e/auth/hurl/metrics.all.hurl new file mode 100644 index 000000000..96eddb905 --- /dev/null +++ b/enterprise/e2e/auth/hurl/metrics.all.hurl @@ -0,0 +1,248 @@ +# A policy on /self/v1/metrics gates the scrape surface at dispatch: denied +# without a credential, with the canonical 401 shape +GET {{base}}/self/v1/metrics +HTTP 401 +Cache-Control: no-store +Content-Type: application/problem+json +WWW-Authenticate: Bearer realm="registry" +Link: ; rel="describedby" +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag, WWW-Authenticate +[Captures] +denied_body: body +denied_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:authentication-required" +jsonpath "$.title" == "Unauthorized" +jsonpath "$.status" == 401 +jsonpath "$.detail" == "This resource requires authentication" + +# The problem document validates against the error schema named by its Link +POST {{base}}/self/v1/api/schemas/evaluate{{denied_schema}} +``` +{{denied_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# A key that opens another surface does not open this one +GET {{base}}/self/v1/metrics +Authorization: Bearer surface-secret-key +HTTP 401 +Cache-Control: no-store +Content-Type: application/problem+json +WWW-Authenticate: Bearer realm="registry" +Link: ; rel="describedby" +[Captures] +crossover_body: body +crossover_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:authentication-required" +jsonpath "$.status" == 401 + +POST {{base}}/self/v1/api/schemas/evaluate{{crossover_schema}} +``` +{{crossover_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# The metrics key admits, and the exposition answers +GET {{base}}/self/v1/metrics +Authorization: Bearer metrics-secret-key +HTTP 200 +Cache-Control: no-store +Content-Type: text/plain; version=0.0.4; charset=utf-8 +[Asserts] +header "Vary" not exists +header "Link" not exists +body matches /# TYPE sourcemeta_one_build_info gauge\nsourcemeta_one_build_info\{version="[^"]+",edition="enterprise"\} 1\n/ +body matches /# TYPE process_start_time_seconds gauge\nprocess_start_time_seconds [0-9.]+\n/ +body matches /# TYPE process_resident_memory_bytes gauge\nprocess_resident_memory_bytes [0-9]+\n/ +body matches /# TYPE process_open_fds gauge\nprocess_open_fds [0-9]+\n/ +body matches /# TYPE sourcemeta_one_http_requests_in_flight gauge\n/ +body matches /# TYPE sourcemeta_one_metrics_dropped_total counter\nsourcemeta_one_metrics_dropped_total 0\n/ +body matches /# TYPE sourcemeta_one_http_requests_total counter\n/ +body matches /sourcemeta_one_http_requests_total\{action="metrics_v1",code="401"\} [0-9]+\n/ +body not matches /le="1e-04"/ + +# A CORS preflight carries no credentials and is never gated +OPTIONS {{base}}/self/v1/metrics +Origin: http://example.com +Access-Control-Request-Method: GET +HTTP 204 +Cache-Control: no-store +Access-Control-Allow-Origin: * +Access-Control-Allow-Methods: GET, HEAD, OPTIONS +Access-Control-Allow-Headers: Accept, Accept-Encoding +Access-Control-Max-Age: 3600 +Allow: GET, HEAD, OPTIONS + +# The gate precedes method validation: an unsupported method denies without the +# key rather than revealing the method is wrong +POST {{base}}/self/v1/metrics +HTTP 401 +Cache-Control: no-store +Content-Type: application/problem+json +WWW-Authenticate: Bearer realm="registry" +Link: ; rel="describedby" +[Captures] +method_denied_body: body +method_denied_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:authentication-required" +jsonpath "$.status" == 401 + +POST {{base}}/self/v1/api/schemas/evaluate{{method_denied_schema}} +``` +{{method_denied_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# With the key the surface admits and the method check then answers 405 +POST {{base}}/self/v1/metrics +Authorization: Bearer metrics-secret-key +HTTP 405 +Cache-Control: no-store +Content-Type: application/problem+json +Allow: GET, HEAD, OPTIONS +Link: ; rel="describedby" +Access-Control-Allow-Origin: * +[Captures] +method_allowed_body: body +method_allowed_schema: header "Link" regex "<([^>]+)>" +[Asserts] +jsonpath "$.type" == "urn:sourcemeta:one:method-not-allowed" +jsonpath "$.title" == "Method Not Allowed" +jsonpath "$.status" == 405 + +POST {{base}}/self/v1/api/schemas/evaluate{{method_allowed_schema}} +``` +{{method_allowed_body}} +``` +HTTP 200 +[Asserts] +jsonpath "$.valid" == true + +# Whether a scrape succeeded is the scraper's own verdict on what it read, and +# it is zero for a target it could not reach, could not authenticate against, +# or could not parse. Nothing else here proves as much in a single number +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: up{job="sourcemeta-one"} +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.resultType" == "vector" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "1" + +# The scraper reports on its own target, which says the gate admitted it rather +# than merely that something was listening +GET http://localhost:9099/api/v1/targets +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +state: active +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.activeTargets" count == 1 +jsonpath "$.data.activeTargets[0].health" == "up" +jsonpath "$.data.activeTargets[0].lastError" == "" +jsonpath "$.data.activeTargets[0].scrapePool" == "sourcemeta-one" + +# An info metric survives the round trip with its labels intact, which is the +# whole reason it is shaped as a gauge whose value says nothing +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_build_info +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].metric.edition" == "enterprise" +jsonpath "$.data.result[0].metric.__name__" == "sourcemeta_one_build_info" +jsonpath "$.data.result[0].value[1]" == "1" + +# A counter carrying both of its labels, which is what makes the series +# addressable by the questions an operator actually asks. The refusals above +# are counted like anything else +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_http_requests_total{action="metrics_v1",code="401"} +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].metric.action" == "metrics_v1" +jsonpath "$.data.result[0].metric.code" == "401" + +# Asking for a percentile is what proves the buckets are cumulative, ordered, +# and closed by an infinite bound. A malformed histogram answers nothing here +# even though it parses perfectly well as text +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: histogram_quantile(0.95, sum by (le) (rate(sourcemeta_one_http_request_duration_seconds_bucket[1m]))) +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" exists + +# The count a histogram carries and the number of requests counted separately +# are two ways of saying the same thing, and they disagree if either is wrong +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sum(sourcemeta_one_http_request_duration_seconds_count) - sum(sourcemeta_one_http_requests_total) +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "0" + +# Nothing was lost on the way to being counted +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: sourcemeta_one_metrics_dropped_total +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 +jsonpath "$.data.result[0].value[1]" == "0" + +# The process metrics use names shared across every language's client library, +# so a dashboard written against any of them finds these +GET http://localhost:9099/api/v1/query +[Options] +retry: 30 +retry-interval: 1000 +[QueryStringParams] +query: process_resident_memory_bytes > 0 and process_max_fds > 0 and process_start_time_seconds > 0 +HTTP 200 +[Asserts] +jsonpath "$.status" == "success" +jsonpath "$.data.result" count == 1 diff --git a/enterprise/e2e/auth/one.json b/enterprise/e2e/auth/one.json index 2764a9a6b..d9ed07e54 100644 --- a/enterprise/e2e/auth/one.json +++ b/enterprise/e2e/auth/one.json @@ -29,6 +29,13 @@ "paths": [ "/self/v1/health" ], "keys": [ { "environmentVariable": "ONE_E2E_KEY_SURFACE" } ] }, + { + "type": "apiKey", + "algorithm": "identity", + "name": "metrics", + "paths": [ "/self/v1/metrics" ], + "keys": [ { "environmentVariable": "ONE_E2E_KEY_METRICS" } ] + }, { "type": "apiKey", "algorithm": "identity", diff --git a/enterprise/e2e/auth/prometheus.yml b/enterprise/e2e/auth/prometheus.yml new file mode 100644 index 000000000..7f42147b8 --- /dev/null +++ b/enterprise/e2e/auth/prometheus.yml @@ -0,0 +1,18 @@ +# A scrape interval this short is unreasonable for a real deployment and +# exactly right here, where the point is for a series to exist before the +# assertions that read it run +global: + scrape_interval: 1s + evaluation_interval: 1s + +scrape_configs: + - job_name: sourcemeta-one + metrics_path: /self/v1/metrics + # The path is governed by a policy, so the scraper presents a credential + # like any other caller + authorization: + type: Bearer + credentials: metrics-secret-key + static_configs: + - targets: + - "sandbox:8001" diff --git a/enterprise/e2e/html/hurl/metrics.all.hurl b/enterprise/e2e/html/hurl/metrics.all.hurl new file mode 100644 index 000000000..4d7966030 --- /dev/null +++ b/enterprise/e2e/html/hurl/metrics.all.hurl @@ -0,0 +1,79 @@ +# A scrape reports on the requests that came before it, so this one gives it +# something to have counted +GET {{base}}/self/v1/health +HTTP 200 + +GET {{base}}/self/v1/metrics +HTTP 200 +Cache-Control: no-store +Content-Type: text/plain; version=0.0.4; charset=utf-8 +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ +body matches /# TYPE sourcemeta_one_build_info gauge\nsourcemeta_one_build_info\{version="[^"]+",edition="enterprise"\} 1\n/ +body matches /# TYPE process_start_time_seconds gauge\nprocess_start_time_seconds [0-9.]+\n/ +body matches /# TYPE process_cpu_seconds_total counter\nprocess_cpu_seconds_total [0-9.e-]+\n/ +body matches /# TYPE process_resident_memory_bytes gauge\nprocess_resident_memory_bytes [0-9]+\n/ +body matches /# TYPE process_virtual_memory_bytes gauge\nprocess_virtual_memory_bytes [0-9]+\n/ +body matches /# TYPE process_open_fds gauge\nprocess_open_fds [0-9]+\n/ +body matches /# TYPE process_max_fds gauge\nprocess_max_fds [0-9]+\n/ +body matches /# TYPE sourcemeta_one_http_requests_in_flight gauge\nsourcemeta_one_http_requests_in_flight [0-9]+\n/ +body matches /# TYPE sourcemeta_one_metrics_dropped_total counter\nsourcemeta_one_metrics_dropped_total 0\n/ +body matches /# TYPE sourcemeta_one_http_requests_total counter\n/ +body matches /sourcemeta_one_http_requests_total\{action="health_check_v1",code="200"\} [0-9]+\n/ +body matches /# TYPE sourcemeta_one_http_request_duration_seconds histogram\n/ +body matches /sourcemeta_one_http_request_duration_seconds_bucket\{action="health_check_v1",le="0.0001"\} [0-9]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_bucket\{action="health_check_v1",le="1.0"\} [0-9]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_bucket\{action="health_check_v1",le="\+Inf"\} [0-9]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_sum\{action="health_check_v1"\} [0-9.e-]+\n/ +body matches /sourcemeta_one_http_request_duration_seconds_count\{action="health_check_v1"\} [0-9]+\n/ +body not matches /le="1e-04"/ + +# The endpoint counts itself like anything else, one scrape behind +GET {{base}}/self/v1/metrics +HTTP 200 +[Asserts] +body matches /sourcemeta_one_http_requests_total\{action="metrics_v1",code="200"\} [0-9]+\n/ + +POST {{base}}/self/v1/metrics +HTTP 405 +Cache-Control: no-store +Content-Type: application/problem+json +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Allow: GET, HEAD, OPTIONS +Link: ; rel="describedby" +[Captures] +last_response: body +schema_path: header "Link" regex "<([^>]+)>" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +jsonpath "$.status" == 405 +jsonpath "$.type" == "urn:sourcemeta:one:method-not-allowed" +jsonpath "$.title" == "Method Not Allowed" + +POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}} +``` +{{last_response}} +``` +HTTP 200 +Cache-Control: no-store +Link: ; rel="describedby" +[Asserts] +jsonpath "$.valid" == true + +OPTIONS {{base}}/self/v1/metrics +HTTP 204 +Cache-Control: no-store +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Access-Control-Allow-Methods: GET, HEAD, OPTIONS +Access-Control-Allow-Headers: Accept, Accept-Encoding +Access-Control-Max-Age: 3600 +Allow: GET, HEAD, OPTIONS diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server.h b/enterprise/server/include/sourcemeta/one/enterprise_server.h index b135ad888..3ddc1e60f 100644 --- a/enterprise/server/include/sourcemeta/one/enterprise_server.h +++ b/enterprise/server/include/sourcemeta/one/enterprise_server.h @@ -8,5 +8,6 @@ #include #include #include +#include #endif diff --git a/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h new file mode 100644 index 000000000..77492b71c --- /dev/null +++ b/enterprise/server/include/sourcemeta/one/enterprise_server_action_metrics_v1.h @@ -0,0 +1,340 @@ +#ifndef SOURCEMETA_ONE_ENTERPRISE_SERVER_ACTION_METRICS_V1_H +#define SOURCEMETA_ONE_ENTERPRISE_SERVER_ACTION_METRICS_V1_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include // std::ranges::transform +#include // std::array +#include // std::uint64_t +#include // std::filesystem::path +#include // std::format +#include // std::span +#include // std::string +#include // std::string_view +#include // std::vector + +#if defined(__APPLE__) +#include // proc_pidinfo, PROC_PIDLISTFDS, PROC_PIDLISTFD_SIZE +#include // mach_task_self, task_info, MACH_TASK_BASIC_INFO +#include // getrusage, getrlimit, RUSAGE_SELF, RLIMIT_NOFILE +#include // getpid +#elif defined(__linux__) +#include // std::ifstream +#include // std::istringstream +#include // getrlimit, RLIMIT_NOFILE +#include // std::error_code +#include // sysconf, _SC_CLK_TCK, _SC_PAGESIZE +#endif + +class ActionMetrics_v1 : public sourcemeta::one::RouterAction { +public: + static constexpr std::string_view DESCRIPTION{ + "Report instance telemetry in the Prometheus exposition format"}; + static constexpr bool READ_ONLY{true}; + static constexpr bool DESTRUCTIVE{false}; + static constexpr bool IDEMPOTENT{true}; + static constexpr bool OPEN_WORLD{false}; + + // How a boundary is spelled is part of the name of the series it bounds, so + // it is decided here rather than by whatever a formatter does with the same + // number on a given platform + static constexpr std::array + BOUNDARIES{{"0.0001", "0.00025", "0.0005", "0.001", "0.0025", "0.005", + "0.01", "0.05", "0.25", "1.0"}}; + + ActionMetrics_v1( + const std::filesystem::path &base, + const sourcemeta::core::URITemplateRouterView &router, + const sourcemeta::core::URITemplateRouter::Identifier identifier, + sourcemeta::one::Router &dispatcher) + : sourcemeta::one::RouterAction{base, router.base_url(), dispatcher} { + router.arguments( + identifier, [this](const auto &key, const auto &value) -> void { + if (key == "errorSchema") { + this->error_schema_ = std::get(value); + } + }); + } + + auto rest(const std::span, + const sourcemeta::one::Authentication::Caller &, + sourcemeta::one::HTTPRequest &request, + sourcemeta::one::HTTPResponse &response) -> void override { + if (request.method() == "options") { + sourcemeta::one::cors_preflight(request, response, "GET, HEAD, OPTIONS", + "Accept, Accept-Encoding"); + return; + } + + if (request.method() != "get" && request.method() != "head") { + sourcemeta::one::json_error( + request, response, sourcemeta::core::HTTP_STATUS_METHOD_NOT_ALLOWED, + "urn:sourcemeta:one:method-not-allowed", + "This HTTP method is invalid for this URL", this->error_schema_, "*", + "GET, HEAD, OPTIONS"); + return; + } + + const auto payload{this->serialize()}; + response.write_status(sourcemeta::core::HTTP_STATUS_OK); + // The exposition format names its own revision in the media type, which is + // how a scraper knows what it is reading without asking + response.write_header("Content-Type", + "text/plain; version=0.0.4; charset=utf-8"); + // A scrape stands for the moment it was taken, so an answer kept and handed + // to the next one would report a past that never comes back + response.write_header("Cache-Control", + sourcemeta::one::cache_control_no_store()); + sourcemeta::one::send_response(sourcemeta::core::HTTP_STATUS_OK, request, + response, payload, + sourcemeta::one::Encoding::Identity); + } + + auto mcp(const sourcemeta::core::MCPProtocolVersion, + const sourcemeta::core::JSON &id, const sourcemeta::core::JSON &, + const sourcemeta::one::Authentication::Caller &) + -> sourcemeta::core::JSON override { + return sourcemeta::core::jsonrpc_make_error_method_not_found(id); + } + +private: + // What a process says about itself, which the platform answers rather than + // this program keeping count of. Anything a platform cannot cheaply say is + // left out rather than guessed at + struct Process { + double cpu_seconds{0}; + std::uint64_t resident_bytes{0}; + std::uint64_t virtual_bytes{0}; + std::uint64_t open_descriptors{0}; + std::uint64_t maximum_descriptors{0}; + }; + + [[nodiscard]] static auto descriptor_limit() -> std::uint64_t { + rlimit limit{}; + if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { + return 0; + } + + return static_cast(limit.rlim_cur); + } + +#if defined(__APPLE__) + + [[nodiscard]] static auto read_process() -> Process { + Process sample; + + rusage usage{}; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + sample.cpu_seconds = + static_cast(usage.ru_utime.tv_sec) + + static_cast(usage.ru_utime.tv_usec) / 1000000.0 + + static_cast(usage.ru_stime.tv_sec) + + static_cast(usage.ru_stime.tv_usec) / 1000000.0; + } + + mach_task_basic_info info{}; + mach_msg_type_number_t count{MACH_TASK_BASIC_INFO_COUNT}; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, + reinterpret_cast(&info), + &count) == KERN_SUCCESS) { + sample.resident_bytes = info.resident_size; + sample.virtual_bytes = info.virtual_size; + } + + const auto descriptors{ + proc_pidinfo(getpid(), PROC_PIDLISTFDS, 0, nullptr, 0)}; + if (descriptors > 0) { + sample.open_descriptors = + static_cast(descriptors) / PROC_PIDLISTFD_SIZE; + } + + sample.maximum_descriptors = descriptor_limit(); + return sample; + } + +#elif defined(__linux__) + + // The second field is a command name in parentheses that may itself contain + // spaces, so what follows it is found from the last parenthesis rather than + // by counting separators from the beginning + [[nodiscard]] static auto read_process() -> Process { + Process sample; + + std::ifstream stream{"/proc/self/stat"}; + if (stream.is_open()) { + std::string line; + std::getline(stream, line); + const auto comm{line.rfind(')')}; + if (comm != std::string::npos) { + std::istringstream fields{line.substr(comm + 1)}; + std::vector tokens; + std::string token; + while (fields >> token) { + tokens.push_back(token); + } + + const auto ticks{static_cast(sysconf(_SC_CLK_TCK))}; + if (tokens.size() > 21 && ticks > 0) { + sample.cpu_seconds = + (std::stod(tokens.at(11)) + std::stod(tokens.at(12))) / ticks; + sample.virtual_bytes = std::stoull(tokens.at(20)); + sample.resident_bytes = + std::stoull(tokens.at(21)) * + static_cast(sysconf(_SC_PAGESIZE)); + } + } + } + + // Reading the list takes a descriptor of its own, which the list then + // includes, so what is counted is one more than what was open + std::error_code error; + const std::filesystem::directory_iterator descriptors{"/proc/self/fd", + error}; + if (!error) { + std::uint64_t listed{0}; + for (const auto &entry : descriptors) { + static_cast(entry); + listed += 1; + } + + sample.open_descriptors = listed > 0 ? listed - 1 : 0; + } + + sample.maximum_descriptors = descriptor_limit(); + return sample; + } + +#else + + [[nodiscard]] static auto read_process() -> Process { return {}; } + +#endif + + static auto family(std::string &output, const std::string_view name, + const std::string_view help, const std::string_view type) + -> void { + output += + std::format("# HELP {} {}\n# TYPE {} {}\n", name, help, name, type); + } + + [[nodiscard]] auto serialize() const -> std::string { + const auto &metrics{sourcemeta::one::http_metrics()}; + const auto state{metrics.snapshot()}; + const auto process{read_process()}; + + std::string edition{sourcemeta::one::edition()}; + std::ranges::transform(edition, edition.begin(), + [](const char character) -> char { + return sourcemeta::core::to_lowercase(character); + }); + + std::string output; + output.reserve(8192); + + family(output, "sourcemeta_one_build_info", "Build information", "gauge"); + output += std::format( + "sourcemeta_one_build_info{{version=\"{}\",edition=\"{}\"}} 1\n\n", + sourcemeta::one::version(), edition); + + family(output, "process_start_time_seconds", + "Start time of the process since the Unix epoch", "gauge"); + output += + std::format("process_start_time_seconds {}\n\n", metrics.started()); + + family(output, "process_cpu_seconds_total", + "Total user and system CPU time spent", "counter"); + output += + std::format("process_cpu_seconds_total {}\n\n", process.cpu_seconds); + + family(output, "process_resident_memory_bytes", "Resident memory size", + "gauge"); + output += std::format("process_resident_memory_bytes {}\n\n", + process.resident_bytes); + + family(output, "process_virtual_memory_bytes", "Virtual memory size", + "gauge"); + output += std::format("process_virtual_memory_bytes {}\n\n", + process.virtual_bytes); + + family(output, "process_open_fds", "Number of open file descriptors", + "gauge"); + output += std::format("process_open_fds {}\n\n", process.open_descriptors); + + family(output, "process_max_fds", "Maximum number of open file descriptors", + "gauge"); + output += + std::format("process_max_fds {}\n\n", process.maximum_descriptors); + + family(output, "sourcemeta_one_http_requests_in_flight", + "Requests currently being served", "gauge"); + output += std::format("sourcemeta_one_http_requests_in_flight {}\n\n", + state.in_flight); + + family(output, "sourcemeta_one_metrics_dropped_total", + "Observations that could not be recorded", "counter"); + output += std::format("sourcemeta_one_metrics_dropped_total {}\n\n", + state.dropped); + + family(output, "sourcemeta_one_http_requests_total", + "Total HTTP requests handled", "counter"); + for (const auto &entry : state.requests) { + output += std::format("sourcemeta_one_http_requests_total{{action=\"{}\"," + "code=\"{}\"}} {}\n", + sourcemeta::one::ACTION_NAMES.at(entry.handler), + entry.status, entry.count); + } + + output += "\n"; + family(output, "sourcemeta_one_http_request_duration_seconds", + "Request duration", "histogram"); + for (std::size_t action = 0; action < state.buckets.size(); action++) { + std::uint64_t cumulative{0}; + for (const auto count : state.buckets[action]) { + cumulative += count; + } + + if (cumulative == 0) { + continue; + } + + cumulative = 0; + const auto &name{sourcemeta::one::ACTION_NAMES.at(action)}; + for (std::size_t bucket = 0; + bucket < sourcemeta::one::HTTPMetrics::BUCKET_COUNT; bucket++) { + cumulative += state.buckets[action][bucket]; + output += std::format("sourcemeta_one_http_request_duration_seconds_" + "bucket{{action=\"{}\",le=\"{}\"}} {}\n", + name, BOUNDARIES.at(bucket), cumulative); + } + + cumulative += + state.buckets[action][sourcemeta::one::HTTPMetrics::BUCKET_COUNT]; + output += std::format("sourcemeta_one_http_request_duration_seconds_" + "bucket{{action=\"{}\",le=\"+Inf\"}} {}\n", + name, cumulative); + output += std::format("sourcemeta_one_http_request_duration_seconds_sum{{" + "action=\"{}\"}} {}\n", + name, state.sums[action]); + output += + std::format("sourcemeta_one_http_request_duration_seconds_count{{" + "action=\"{}\"}} {}\n", + name, cumulative); + } + + return output; + } + + std::string_view error_schema_; +}; + +#endif diff --git a/src/actions/CMakeLists.txt b/src/actions/CMakeLists.txt index 8fdbd5eca..4c1c56724 100644 --- a/src/actions/CMakeLists.txt +++ b/src/actions/CMakeLists.txt @@ -4,6 +4,7 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT one NAME actions action_default_v1.h action_dependency_tree_v1.h action_health_check_v1.h + action_metrics_v1.h action_jsonschema_evaluate_v1.h action_jsonschema_rdf_v1.h action_jsonschema_serve_v1.h diff --git a/src/actions/action_metrics_v1.h b/src/actions/action_metrics_v1.h new file mode 100644 index 000000000..0a8e67478 --- /dev/null +++ b/src/actions/action_metrics_v1.h @@ -0,0 +1,84 @@ +#ifndef SOURCEMETA_ONE_ACTIONS_METRICS_V1_H +#define SOURCEMETA_ONE_ACTIONS_METRICS_V1_H + +#if defined(SOURCEMETA_ONE_ENTERPRISE) + +#include + +#else + +#include +#include +#include +#include + +#include +#include + +#include // std::filesystem::path +#include // std::span +#include // std::string_view + +class ActionMetrics_v1 : public sourcemeta::one::RouterAction { +public: + static constexpr std::string_view DESCRIPTION{ + "Report instance telemetry in the Prometheus exposition format"}; + static constexpr bool READ_ONLY{true}; + static constexpr bool DESTRUCTIVE{false}; + static constexpr bool IDEMPOTENT{true}; + static constexpr bool OPEN_WORLD{false}; + + ActionMetrics_v1( + const std::filesystem::path &base, + const sourcemeta::core::URITemplateRouterView &router, + const sourcemeta::core::URITemplateRouter::Identifier identifier, + sourcemeta::one::Router &dispatcher) + : sourcemeta::one::RouterAction{base, router.base_url(), dispatcher} { + router.arguments( + identifier, [this](const auto &key, const auto &value) -> void { + if (key == "errorSchema") { + this->error_schema_ = std::get(value); + } + }); + } + + auto rest(const std::span, + const sourcemeta::one::Authentication::Caller &, + sourcemeta::one::HTTPRequest &request, + sourcemeta::one::HTTPResponse &response) -> void override { + if (request.method() == "options") { + sourcemeta::one::cors_preflight(request, response, "GET, HEAD, OPTIONS", + "Accept, Accept-Encoding"); + return; + } + + if (request.method() != "get" && request.method() != "head") { + sourcemeta::one::json_error( + request, response, sourcemeta::core::HTTP_STATUS_METHOD_NOT_ALLOWED, + "urn:sourcemeta:one:method-not-allowed", + "This HTTP method is invalid for this URL", this->error_schema_, "*", + "GET, HEAD, OPTIONS"); + return; + } + + sourcemeta::one::json_error( + request, response, sourcemeta::core::HTTP_STATUS_FORBIDDEN, + "urn:sourcemeta:one:enterprise-required", + "This feature is only available in the Enterprise edition", + this->error_schema_, "*"); + } + + auto mcp(const sourcemeta::core::MCPProtocolVersion, + const sourcemeta::core::JSON &id, const sourcemeta::core::JSON &, + const sourcemeta::one::Authentication::Caller &) + -> sourcemeta::core::JSON override { + return sourcemeta::core::jsonrpc_make_error_method_not_found(id); + } + +private: + std::string_view error_schema_; +}; + +#endif + +#endif diff --git a/src/actions/actions.cc b/src/actions/actions.cc index 9dc89c11a..7596fe6f2 100644 --- a/src/actions/actions.cc +++ b/src/actions/actions.cc @@ -23,6 +23,7 @@ #include "action_list_directory_v1.h" #include "action_mcp_prm_v1.h" #include "action_mcp_v1.h" +#include "action_metrics_v1.h" #include "action_not_found_v1.h" #include "action_schema_search_v1.h" #include "action_serve_explorer_artifact_v1.h" @@ -39,7 +40,7 @@ struct ActionMetadata { bool open_world; }; -#define SOURCEMETA_ONE_DEFINE_METADATA(Name, Class) \ +#define SOURCEMETA_ONE_DEFINE_METADATA(Name, Class, Label) \ ActionMetadata{Class::DESCRIPTION, Class::READ_ONLY, Class::DESTRUCTIVE, \ Class::IDEMPOTENT, Class::OPEN_WORLD}, @@ -52,7 +53,7 @@ const std::array METADATA{ namespace sourcemeta::one { -#define SOURCEMETA_ONE_MAKE_CONSTRUCTOR_ENTRY(Name, Class) \ +#define SOURCEMETA_ONE_MAKE_CONSTRUCTOR_ENTRY(Name, Class, Label) \ table[ACTION_TYPE_##Name] = &make_router_action; const std::array CONSTRUCTORS{ diff --git a/src/actions/include/sourcemeta/one/actions.h b/src/actions/include/sourcemeta/one/actions.h index 250a3aefd..f6ad62852 100644 --- a/src/actions/include/sourcemeta/one/actions.h +++ b/src/actions/include/sourcemeta/one/actions.h @@ -9,34 +9,53 @@ namespace sourcemeta::one { +// New entries are appended rather than inserted, since the position of an +// entry is the identifier a built router records for it, and an identifier +// that moves points an already-built router at the wrong handler. +// +// The third column is what an action is called to anybody outside this +// program, which is a name this project promises rather than one it happens to +// use. It is written down rather than derived from the first, so that renaming +// a handler is an internal matter and renaming what an operator sees is a +// deliberate act #define SOURCEMETA_ONE_FOR_EACH_ACTION(X) \ - X(DEFAULT_V1, ActionDefault_v1) \ - X(HEALTH_CHECK_V1, ActionHealthCheck_v1) \ - X(NOT_FOUND_V1, ActionNotFound_v1) \ - X(SCHEMA_ARTIFACT_V1, ActionServeSchemaArtifact_v1) \ - X(EXPLORER_ARTIFACT_V1, ActionServeExplorerArtifact_v1) \ - X(GET_SCHEMA_HEALTH_V1, ActionGetSchemaHealth_v1) \ - X(GET_SCHEMA_LOCATIONS_V1, ActionGetSchemaLocations_v1) \ - X(GET_SCHEMA_POSITIONS_V1, ActionGetSchemaPositions_v1) \ - X(GET_SCHEMA_STATS_V1, ActionGetSchemaStats_v1) \ - X(GET_SCHEMA_METADATA_V1, ActionGetSchemaMetadata_v1) \ - X(LIST_DIRECTORY_V1, ActionListDirectory_v1) \ - X(DEPENDENCY_TREE_V1, ActionDependencyTree_v1) \ - X(GET_SCHEMA_DEPENDENCIES_V1, ActionGetSchemaDependencies_v1) \ - X(GET_SCHEMA_DEPENDENTS_V1, ActionGetSchemaDependents_v1) \ - X(JSONSCHEMA_EVALUATE_V1, ActionJSONSchemaEvaluate_v1) \ - X(JSONSCHEMA_RDF_V1, ActionJSONSchemaRDF_v1) \ - X(JSONSCHEMA_TRACE_V1, ActionJSONSchemaTrace_v1) \ - X(SCHEMA_SEARCH_V1, ActionSchemaSearch_v1) \ - X(SERVE_STATIC_V1, ActionServeStatic_v1) \ - X(MCP_V1, ActionMCP_v1) \ - X(AUTH_LOGOUT_V1, ActionAuthLogout_v1) \ - X(AUTH_LOGIN_V1, ActionAuthLogin_v1) \ - X(AUTH_LOGIN_PAGE_V1, ActionAuthLoginPage_v1) \ - X(AUTH_CALLBACK_V1, ActionAuthCallback_v1) \ - X(MCP_PROTECTED_RESOURCE_METADATA_V1, ActionMCPProtectedResourceMetadata_v1) - -#define SOURCEMETA_ONE_DEFINE_ACTION_TYPE(Name, Class) ACTION_TYPE_##Name, + X(DEFAULT_V1, ActionDefault_v1, "default_v1") \ + X(HEALTH_CHECK_V1, ActionHealthCheck_v1, "health_check_v1") \ + X(NOT_FOUND_V1, ActionNotFound_v1, "not_found_v1") \ + X(SCHEMA_ARTIFACT_V1, ActionServeSchemaArtifact_v1, "schema_artifact_v1") \ + X(EXPLORER_ARTIFACT_V1, ActionServeExplorerArtifact_v1, \ + "explorer_artifact_v1") \ + X(GET_SCHEMA_HEALTH_V1, ActionGetSchemaHealth_v1, "get_schema_health_v1") \ + X(GET_SCHEMA_LOCATIONS_V1, ActionGetSchemaLocations_v1, \ + "get_schema_locations_v1") \ + X(GET_SCHEMA_POSITIONS_V1, ActionGetSchemaPositions_v1, \ + "get_schema_positions_v1") \ + X(GET_SCHEMA_STATS_V1, ActionGetSchemaStats_v1, "get_schema_stats_v1") \ + X(GET_SCHEMA_METADATA_V1, ActionGetSchemaMetadata_v1, \ + "get_schema_metadata_v1") \ + X(LIST_DIRECTORY_V1, ActionListDirectory_v1, "list_directory_v1") \ + X(DEPENDENCY_TREE_V1, ActionDependencyTree_v1, "dependency_tree_v1") \ + X(GET_SCHEMA_DEPENDENCIES_V1, ActionGetSchemaDependencies_v1, \ + "get_schema_dependencies_v1") \ + X(GET_SCHEMA_DEPENDENTS_V1, ActionGetSchemaDependents_v1, \ + "get_schema_dependents_v1") \ + X(JSONSCHEMA_EVALUATE_V1, ActionJSONSchemaEvaluate_v1, \ + "jsonschema_evaluate_v1") \ + X(JSONSCHEMA_RDF_V1, ActionJSONSchemaRDF_v1, "jsonschema_rdf_v1") \ + X(JSONSCHEMA_TRACE_V1, ActionJSONSchemaTrace_v1, "jsonschema_trace_v1") \ + X(SCHEMA_SEARCH_V1, ActionSchemaSearch_v1, "schema_search_v1") \ + X(SERVE_STATIC_V1, ActionServeStatic_v1, "serve_static_v1") \ + X(MCP_V1, ActionMCP_v1, "mcp_v1") \ + X(AUTH_LOGOUT_V1, ActionAuthLogout_v1, "auth_logout_v1") \ + X(AUTH_LOGIN_V1, ActionAuthLogin_v1, "auth_login_v1") \ + X(AUTH_LOGIN_PAGE_V1, ActionAuthLoginPage_v1, "auth_login_page_v1") \ + X(AUTH_CALLBACK_V1, ActionAuthCallback_v1, "auth_callback_v1") \ + X(MCP_PROTECTED_RESOURCE_METADATA_V1, ActionMCPProtectedResourceMetadata_v1, \ + "mcp_protected_resource_metadata_v1") \ + X(METRICS_V1, ActionMetrics_v1, "metrics_v1") + +#define SOURCEMETA_ONE_DEFINE_ACTION_TYPE(Name, Class, Label) \ + ACTION_TYPE_##Name, enum : std::uint8_t { SOURCEMETA_ONE_FOR_EACH_ACTION(SOURCEMETA_ONE_DEFINE_ACTION_TYPE) @@ -45,6 +64,16 @@ enum : std::uint8_t { #undef SOURCEMETA_ONE_DEFINE_ACTION_TYPE +#define SOURCEMETA_ONE_DEFINE_ACTION_NAME(Name, Class, Label) \ + std::string_view{Label}, + +// Indexed by the same values the enum above defines, since both come from the +// one list and neither can name an action the other does not have +inline constexpr std::array ACTION_NAMES{ + {SOURCEMETA_ONE_FOR_EACH_ACTION(SOURCEMETA_ONE_DEFINE_ACTION_NAME)}}; + +#undef SOURCEMETA_ONE_DEFINE_ACTION_NAME + extern const std::array CONSTRUCTORS; diff --git a/src/http/CMakeLists.txt b/src/http/CMakeLists.txt index d192fc153..2581eb7ea 100644 --- a/src/http/CMakeLists.txt +++ b/src/http/CMakeLists.txt @@ -1,5 +1,5 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT one NAME http - PRIVATE_HEADERS uwebsockets.h request.h response.h helpers.h server.h) + PRIVATE_HEADERS uwebsockets.h request.h response.h metrics.h helpers.h server.h) target_link_libraries(sourcemeta_one_http INTERFACE sourcemeta::core::json) target_link_libraries(sourcemeta_one_http INTERFACE sourcemeta::core::time) diff --git a/src/http/include/sourcemeta/one/http.h b/src/http/include/sourcemeta/one/http.h index 87a288500..40ba14eb4 100644 --- a/src/http/include/sourcemeta/one/http.h +++ b/src/http/include/sourcemeta/one/http.h @@ -2,6 +2,7 @@ #define SOURCEMETA_ONE_HTTP_H #include +#include #include #include #include diff --git a/src/http/include/sourcemeta/one/http_helpers.h b/src/http/include/sourcemeta/one/http_helpers.h index 466047069..2934c0a77 100644 --- a/src/http/include/sourcemeta/one/http_helpers.h +++ b/src/http/include/sourcemeta/one/http_helpers.h @@ -14,8 +14,9 @@ #include // std::ranges::equal #include // std::array #include // assert -#include // std::chrono::system_clock +#include // std::chrono::system_clock, std::chrono::steady_clock #include // std::size_t +#include // std::uint8_t, std::uint16_t #include // std::format #include // std::mutex, std::scoped_lock #include // std::optional @@ -204,6 +205,7 @@ inline auto send_response(const sourcemeta::core::HTTPStatus &status, std::format("{} {} {}", status.wire, request.method(), request.path())}; response.send_without_content(); HTTP_LOG(line); + request.observation().record(status.code); } inline auto send_response( @@ -217,6 +219,7 @@ inline auto send_response( response.send(request, message, current_encoding, precomputed_compressed_size); HTTP_LOG(line); + request.observation().record(status.code); } // RFC 9110 ยง9.3.7: OPTIONS responses describe communication options diff --git a/src/http/include/sourcemeta/one/http_metrics.h b/src/http/include/sourcemeta/one/http_metrics.h new file mode 100644 index 000000000..c1a6226d8 --- /dev/null +++ b/src/http/include/sourcemeta/one/http_metrics.h @@ -0,0 +1,213 @@ +#ifndef SOURCEMETA_ONE_HTTP_METRICS_H +#define SOURCEMETA_ONE_HTTP_METRICS_H + +#include // std::ranges::lower_bound +#include // std::array +#include // std::atomic +#include // std::chrono::system_clock, std::chrono::duration +#include // std::size_t +#include // std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t +#include // std::distance +#include // std::map +#include // std::mutex, std::scoped_lock +#include // std::vector + +namespace sourcemeta::one { + +// What serving requests cost, kept as numbers and nothing else. +// +// A request is counted against a handler, which is a number this class never +// interprets. What a handler is, what it is called, and whether any of this is +// ever said out loud are all decided elsewhere, which is what lets this stay +// true of any server rather than of this one +class HTTPMetrics { +public: + static constexpr std::size_t BUCKET_COUNT{10}; + + // An order of magnitude finer at the low end than the usual boundaries, + // since answering out of a file takes well under a millisecond and the + // defaults would put nearly every request in the first bucket + static constexpr std::array BUCKETS{ + {0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.05, 0.25, 1.0}}; + + // How many requests one handler answered one way + struct Entry { + std::uint8_t handler{0}; + std::uint16_t status{0}; + std::uint64_t count{0}; + }; + + // Everything counted so far, taken in one pass so that what is read cannot + // disagree with itself. Entries are ordered, since two readings of an + // unchanged server should say the same thing in the same order + struct Snapshot { + std::uint64_t in_flight{0}; + std::uint64_t dropped{0}; + std::vector requests; + std::vector> buckets; + std::vector sums; + }; + + HTTPMetrics() = default; + + // To avoid mistakes + HTTPMetrics(const HTTPMetrics &) = delete; + HTTPMetrics(HTTPMetrics &&) = delete; + auto operator=(const HTTPMetrics &) -> HTTPMetrics & = delete; + auto operator=(HTTPMetrics &&) -> HTTPMetrics & = delete; + + // The server is coming up, with this many handlers it may ever count + // against, which whoever owns them is the only one to know. Nothing is + // counted before this is said + auto start(const std::size_t handlers) -> void { + this->handlers_ = handlers; + this->started_ = + std::chrono::duration{ + std::chrono::system_clock::now().time_since_epoch()} + .count(); + for (auto &shard : this->shards_) { + const std::scoped_lock guard{shard.mutex}; + shard.buckets.assign(handlers, {}); + shard.sums.assign(handlers, 0.0); + } + } + + // A request has arrived, which is what the in-flight count is the difference + // between + auto enter() noexcept -> void { + this->entered_.fetch_add(1, std::memory_order_relaxed); + } + + // A request has gone without ever being answered, which a caller that hung + // up mid-upload does. Nothing is counted against a handler, since nothing + // was served, but the in-flight count comes back down all the same + auto abandon() noexcept -> void { + this->answered_.fetch_add(1, std::memory_order_relaxed); + } + + // A request has been answered, which is both what is counted and what brings + // the in-flight count back down. A request answered before any handler was + // chosen carries none, and is counted as served without being attributed + auto observe(const std::uint8_t handler, const std::uint16_t status, + const double seconds) noexcept -> void { + this->answered_.fetch_add(1, std::memory_order_relaxed); + if (handler >= this->handlers_) [[unlikely]] { + return; + } + + const auto boundary{std::ranges::lower_bound(BUCKETS, seconds)}; + const auto bucket{ + static_cast(std::distance(BUCKETS.begin(), boundary))}; + + // Nothing said about a request may change how it was answered, and the + // answer has already gone out by the time this runs + try { + auto &target{this->shard()}; + const std::scoped_lock guard{target.mutex}; + target.requests[key(handler, status)] += 1; + target.buckets[handler][bucket] += 1; + target.sums[handler] += seconds; + } catch (...) { + // A gap somebody can see is worth more than one they cannot, so what + // could not be recorded is counted and said in the answer itself + this->dropped_.fetch_add(1, std::memory_order_relaxed); + } + } + + [[nodiscard]] auto snapshot() const -> Snapshot { + Snapshot result; + result.dropped = this->dropped_.load(std::memory_order_relaxed); + const auto entered{this->entered_.load(std::memory_order_relaxed)}; + const auto answered{this->answered_.load(std::memory_order_relaxed)}; + result.in_flight = entered > answered ? entered - answered : 0; + result.buckets.assign(this->handlers_, {}); + result.sums.assign(this->handlers_, 0.0); + + std::map totals; + for (const auto &shard : this->shards_) { + const std::scoped_lock guard{shard.mutex}; + for (const auto &[entry, count] : shard.requests) { + totals[entry] += count; + } + + for (std::size_t handler = 0; handler < shard.buckets.size(); handler++) { + for (std::size_t bucket = 0; bucket <= BUCKET_COUNT; bucket++) { + result.buckets[handler][bucket] += shard.buckets[handler][bucket]; + } + + result.sums[handler] += shard.sums[handler]; + } + } + + result.requests.reserve(totals.size()); + for (const auto &[entry, count] : totals) { + result.requests.push_back( + {.handler = static_cast(entry >> 16U), + .status = static_cast(entry & 0xFFFFU), + .count = count}); + } + + return result; + } + + // When this server began, as seconds since the Unix epoch + [[nodiscard]] auto started() const noexcept -> double { + return this->started_; + } + +private: + // A series is named by what answered and what it answered together, so + // neither alone identifies one + [[nodiscard]] static constexpr auto key(const std::uint8_t handler, + const std::uint16_t status) noexcept + -> std::uint32_t { + return (static_cast(handler) << 16U) | status; + } + + // Enough that threads rarely share one, and a fixed number rather than one + // per hardware thread so that building this allocates nothing and can + // therefore happen before anything else does. Threads beyond this many + // share, which the lock already accounts for + static constexpr std::size_t SHARD_COUNT{16}; + + // Answering a request touches a line no other thread is writing to. Reading + // them is a scrape, which happens once every several seconds and can afford + // to visit each in turn + struct Shard { + mutable std::mutex mutex; + std::map requests; + std::vector> buckets; + std::vector sums; + }; + + // A thread takes the next place in line the first time it answers anything, + // and keeps it, so which line to touch costs nothing to work out afterwards + [[nodiscard]] auto shard() const noexcept -> Shard & { + static std::atomic next{0}; + thread_local const std::size_t assigned{ + next.fetch_add(1, std::memory_order_relaxed)}; + return this->shards_[assigned % SHARD_COUNT]; + } + + std::size_t handlers_{0}; + mutable std::array shards_; + std::atomic entered_{0}; + std::atomic answered_{0}; + std::atomic dropped_{0}; + double started_{0}; +}; + +// What this process has served, which there is one of because there is one +// server. It lives here rather than on the server itself, since what a request +// is worth is read where a request is answered and that is not there. It is +// built before anything runs, so reaching it never costs a check +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +inline HTTPMetrics HTTP_METRICS; + +[[nodiscard]] inline auto http_metrics() noexcept -> HTTPMetrics & { + return HTTP_METRICS; +} + +} // namespace sourcemeta::one + +#endif diff --git a/src/http/include/sourcemeta/one/http_request.h b/src/http/include/sourcemeta/one/http_request.h index 5ec99dac1..d7f7ba9a2 100644 --- a/src/http/include/sourcemeta/one/http_request.h +++ b/src/http/include/sourcemeta/one/http_request.h @@ -4,13 +4,16 @@ #include #include +#include #include #include -#include // std::chrono::system_clock +#include // std::chrono::system_clock, std::chrono::steady_clock #include // std::invocable #include // std::size_t +#include // std::uint8_t, std::uint16_t #include // std::exception_ptr, std::current_exception +#include // std::numeric_limits #include // std::shared_ptr, std::make_shared #include // std::optional #include // std::string @@ -25,12 +28,50 @@ namespace sourcemeta::one { inline constexpr std::size_t MAX_REQUEST_BODY_BYTES{ static_cast(4) * 1024 * 1024}; +// What is remembered about a request so that whoever answers it can say what +// it cost. Which handler answered is a value nothing owns until one is chosen, +// so a request refused ahead of routing carries none and goes unattributed +struct Observation { + std::chrono::steady_clock::time_point started{}; + std::uint8_t handler{std::numeric_limits::max()}; + // A request arrives once and leaves once, but there is more than one way for + // it to leave and more than one place that notices. Settling here rather + // than at each of them is what keeps the in-flight count honest + mutable bool settled{false}; + + auto record(const std::uint16_t status) const -> void { + if (this->settled) { + return; + } + + this->settled = true; + http_metrics().observe(this->handler, status, + std::chrono::duration{ + std::chrono::steady_clock::now() - this->started} + .count()); + } + + // Nothing was served, so nothing is counted against a handler, but the + // in-flight count comes back down all the same + auto abandon() const -> void { + if (this->settled) { + return; + } + + this->settled = true; + http_metrics().abandon(); + } +}; + class HTTPRequest { public: // Primary constructor from raw uWebSockets pointers HTTPRequest(uWS::HttpRequest *request, uWS::HttpResponse *response) noexcept - : request_{request}, response_{response} {} + : request_{request}, response_{response} { + this->observation_.started = std::chrono::steady_clock::now(); + http_metrics().enter(); + } // Snapshot constructor for async contexts where uWS::HttpRequest is gone HTTPRequest(std::string method, std::string path, @@ -53,6 +94,14 @@ class HTTPRequest { : sourcemeta::one::Encoding::Identity; } + [[nodiscard]] auto observation() noexcept -> Observation & { + return this->observation_; + } + + [[nodiscard]] auto observation() const noexcept -> const Observation & { + return this->observation_; + } + [[nodiscard]] auto method() const noexcept -> std::string_view { return this->request_ ? this->request_->getMethod() : this->method_; } @@ -150,11 +199,14 @@ class HTTPRequest { auto snapshot = std::make_shared( std::string{this->method()}, std::string{this->path()}, this->response_encoding_, raw_response); + snapshot->observation_ = this->observation_; auto buffer = std::make_shared(); auto completed = std::make_shared(false); - raw_response->onAborted( - [completed]() mutable -> void { *completed = true; }); + raw_response->onAborted([completed, snapshot]() mutable -> void { + *completed = true; + snapshot->observation_.abandon(); + }); raw_response->onData( // NOLINTNEXTLINE(bugprone-exception-escape) @@ -204,6 +256,7 @@ class HTTPRequest { bool satisfiable_encoding_{true}; sourcemeta::one::Encoding response_encoding_{ sourcemeta::one::Encoding::Identity}; + Observation observation_{}; }; } // namespace sourcemeta::one diff --git a/src/http/include/sourcemeta/one/http_server.h b/src/http/include/sourcemeta/one/http_server.h index 55b544ba5..3ad7b394d 100644 --- a/src/http/include/sourcemeta/one/http_server.h +++ b/src/http/include/sourcemeta/one/http_server.h @@ -111,6 +111,9 @@ class HTTPServer { response.write_status( sourcemeta::core::HTTP_STATUS_INTERNAL_SERVER_ERROR); response.send_without_content(); + request.observation().record( + sourcemeta::core::HTTP_STATUS_INTERNAL_SERVER_ERROR + .code); } }); diff --git a/src/index/endpoints.h b/src/index/endpoints.h index 03252d0d7..21181e03c 100644 --- a/src/index/endpoints.h +++ b/src/index/endpoints.h @@ -34,6 +34,7 @@ inline constexpr std::string_view ENDPOINT_SCHEMA_TRACE{ inline constexpr std::string_view ENDPOINT_SCHEMA_SEARCH{ "/self/v1/api/schemas/search"}; inline constexpr std::string_view ENDPOINT_HEALTH{"/self/v1/health"}; +inline constexpr std::string_view ENDPOINT_METRICS{"/self/v1/metrics"}; inline constexpr std::string_view ENDPOINT_AUTH_LOGOUT{"/self/v1/auth/logout"}; inline constexpr std::string_view ENDPOINT_AUTH_LOGIN_PAGE{ "/self/v1/auth/login"}; diff --git a/src/index/generators.h b/src/index/generators.h index 28f309a74..68321eeb9 100644 --- a/src/index/generators.h +++ b/src/index/generators.h @@ -900,6 +900,11 @@ struct GENERATE_URITEMPLATE_ROUTES { next_id++, sourcemeta::one::ACTION_TYPE_HEALTH_CHECK_V1, health_check_arguments); + const sourcemeta::core::URITemplateRouter::Argument metrics_arguments[] = + {{"errorSchema", std::string_view{error_schema}}}; + router.add(sourcemeta::one::ENDPOINT_METRICS, "server_metrics", next_id++, + sourcemeta::one::ACTION_TYPE_METRICS_V1, metrics_arguments); + const sourcemeta::core::URITemplateRouter::Argument auth_logout_arguments[] = { {"errorSchema", std::string_view{error_schema}}}; diff --git a/src/router/router.cc b/src/router/router.cc index 88f0c4082..617ed29a4 100644 --- a/src/router/router.cc +++ b/src/router/router.cc @@ -4,6 +4,7 @@ #include #include // std::chrono::seconds +#include // std::uint8_t #include // std::make_unique #include // std::call_once #include // std::optional, std::nullopt @@ -70,6 +71,8 @@ Router::Router(const std::filesystem::path &base, authentication_{ sourcemeta::one::Authentication::Table{base / "authentication.bin"}, provider_fetcher()} { + // Only whoever holds the handler table knows how many there can be + sourcemeta::one::http_metrics().start(constructors.size()); router.arguments(0, [this](const auto &key, const auto &value) -> void { if (key == "errorSchema") { this->default_error_schema_ = std::get(value); @@ -119,6 +122,10 @@ auto Router::dispatch( const std::span matches, sourcemeta::one::HTTPRequest &request, sourcemeta::one::HTTPResponse &response) -> void { + // Which handler answers is the one thing about a request that only routing + // knows, so it is the one thing said here + request.observation().handler = static_cast(context); + auto *instance{this->action(identifier, context)}; if (instance == nullptr) [[unlikely]] { this->error(request, response, diff --git a/test/e2e/html/hurl/metrics.community.hurl b/test/e2e/html/hurl/metrics.community.hurl new file mode 100644 index 000000000..6956443ee --- /dev/null +++ b/test/e2e/html/hurl/metrics.community.hurl @@ -0,0 +1,81 @@ +GET {{base}}/self/v1/metrics +HTTP 403 +Cache-Control: no-store +Content-Type: application/problem+json +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Link: ; rel="describedby" +[Captures] +last_response: body +schema_path: header "Link" regex "<([^>]+)>" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ +jsonpath "$.status" == 403 +jsonpath "$.type" == "urn:sourcemeta:one:enterprise-required" +jsonpath "$.title" == "Forbidden" +jsonpath "$.detail" == "This feature is only available in the Enterprise edition" + +POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}} +``` +{{last_response}} +``` +HTTP 200 +Cache-Control: no-store +Link: ; rel="describedby" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +jsonpath "$.valid" == true + +POST {{base}}/self/v1/metrics +HTTP 405 +Cache-Control: no-store +Content-Type: application/problem+json +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Allow: GET, HEAD, OPTIONS +Link: ; rel="describedby" +[Captures] +last_response: body +schema_path: header "Link" regex "<([^>]+)>" +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/ +jsonpath "$.status" == 405 +jsonpath "$.type" == "urn:sourcemeta:one:method-not-allowed" +jsonpath "$.title" == "Method Not Allowed" + +POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}} +``` +{{last_response}} +``` +HTTP 200 +Cache-Control: no-store +Link: ; rel="describedby" +[Asserts] +jsonpath "$.valid" == true + +OPTIONS {{base}}/self/v1/metrics +HTTP 204 +Cache-Control: no-store +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: Link, ETag +Access-Control-Allow-Methods: GET, HEAD, OPTIONS +Access-Control-Allow-Headers: Accept, Accept-Encoding +Access-Control-Max-Age: 3600 +Allow: GET, HEAD, OPTIONS +[Asserts] +header "Vary" not exists +header "Referrer-Policy" not exists +header "Content-Security-Policy" not exists +header "X-Frame-Options" not exists +header "Date" matches /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), (0[1-9]|[12][0-9]|3[01]) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} ([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9] GMT$/