From b4e4ae0c7071e4e2ebef7052026628cec8508569 Mon Sep 17 00:00:00 2001 From: Amiel Peled Date: Thu, 20 Aug 2026 19:43:59 +0300 Subject: [PATCH 1/4] fix(deploy): preserve runnable API artifact --- artifacts/api-server/.replit-artifact/artifact.toml | 4 ++-- server/tests/test_replit_deployment.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/artifacts/api-server/.replit-artifact/artifact.toml b/artifacts/api-server/.replit-artifact/artifact.toml index 8ff9fd5..46ed04a 100644 --- a/artifacts/api-server/.replit-artifact/artifact.toml +++ b/artifacts/api-server/.replit-artifact/artifact.toml @@ -10,7 +10,7 @@ name = "API Server" paths = ["/api"] [services.development] -run = "cd ../../server && uv run uvicorn app.main:app --host 0.0.0.0 --port 8080 --reload" +run = "uv run --directory ../../server uvicorn app.main:app --host 0.0.0.0 --port 8080 --reload" [services.production] @@ -21,7 +21,7 @@ args = ["uv", "sync", "--project", "server", "--frozen", "--no-dev"] NODE_ENV = "production" [services.production.run] -args = ["sh", "-c", "cd ../../server && uv run uvicorn app.main:app --host 0.0.0.0 --port 8080"] +args = ["uv", "run", "--directory", "../../server", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] [services.production.run.env] PORT = "8080" diff --git a/server/tests/test_replit_deployment.py b/server/tests/test_replit_deployment.py index 6624fb3..e2b92e5 100644 --- a/server/tests/test_replit_deployment.py +++ b/server/tests/test_replit_deployment.py @@ -15,7 +15,7 @@ def test_replit_runs_the_api_locally_instead_of_relaying_to_an_absent_backend() service = artifact["services"][0] assert service["paths"] == ["/api"] run_args = service["production"]["run"]["args"] - assert run_args[:2] == ["sh", "-c"] - assert run_args[2].startswith("cd ../../server && uv run uvicorn app.main:app ") + assert run_args[:4] == ["uv", "run", "--directory", "../../server"] + assert run_args[4:6] == ["uvicorn", "app.main:app"] assert service["production"]["health"]["startup"]["path"] == "/api/healthz" assert "UPSTREAM_API_BASE_URL" not in service["production"]["run"]["env"] From 68dde9a6388a02c4716cd97704cdf7db4d34b5c5 Mon Sep 17 00:00:00 2001 From: Amiel Peled Date: Thu, 20 Aug 2026 20:04:34 +0300 Subject: [PATCH 2/4] fix(server): stop a broken quota store from looking like an exhausted quota MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sign-in fails on the live deployment, and so does every AI request. The cause is not in the sign-in code: SharedRateLimiter is failing closed. In production the limiter counts in Postgres, keyed by an HMAC of the caller's identity, and needs RATE_LIMIT_SALT (or METRICS_ID_SALT) to build that key. Without one it returns (False, 0) for every caller — the right call, since a quota that cannot count must not wave paid API calls through — but it did so silently, and the caller saw a bare 429. That is indistinguishable from a real quota, which is why the outage sat there: /api/readyz reported "ready", the logs said nothing, and the site told visitors "Too many sign-in attempts." Diagnosed against the running deployment rather than guessed: /api/readyz reports the database available, and POST /api/ai/generate answers 429 with no X-AI-Quota-* headers — the burst branch, which returns before those headers exist — and still does after a full 60s window has rolled, which an exhausted bucket could not. - rate_limit.py: shared_quota_problem() names why a production quota cannot count, and hit() logs it once per cause rather than per request. - /api/readyz reports it in a new optional `rateLimiting` field. The status code still follows the database alone: Fly health-checks this path, so turning a degraded quota into a 503 would stop the machine and make it worse. Spec, Zod and client regenerated. - ops.py takes the database probe through Depends, so the readiness tests cover both branches instead of skipping when no database is reachable. The runbook is what let this ship: the first-deploy `fly secrets set` block never listed RATE_LIMIT_SALT, and it was mentioned later only as an aside under monitoring. It is in the block now, with what breaks without it, and the post-deploy check says to confirm `rateLimiting` is absent. This does not by itself restore the deployment — the secret still has to be set on the host — but the next deployment that misses it says so. 133 pytest, 242 unit, 58 api, 72 contract pass. Co-Authored-By: Claude Opus 5 (1M context) --- deploy-logs.md | 406 ++++++++++++++++++ deploy/README.md | 13 +- .../src/generated/api.schemas.ts | 2 + lib/api-spec/openapi.yaml | 7 + lib/api-zod/src/generated/api.ts | 6 + .../src/generated/types/readinessStatus.ts | 2 + replit.md | 1 + server/app/dependencies.py | 10 +- server/app/rate_limit.py | 40 +- server/app/routes/ops.py | 17 +- server/tests/test_rate_limit_config.py | 137 ++++++ 11 files changed, 631 insertions(+), 10 deletions(-) create mode 100644 deploy-logs.md create mode 100644 server/tests/test_rate_limit_config.py diff --git a/deploy-logs.md b/deploy-logs.md new file mode 100644 index 0000000..a38d8f1 --- /dev/null +++ b/deploy-logs.md @@ -0,0 +1,406 @@ +- generic [ref=f61e5579]: + - generic [ref=f61e5580]: + - checkbox "Select all log lines" [ref=f61e5584] [cursor=pointer] + - generic [ref=f61e5585]: Time + - generic [ref=f61e5591]: Log + - button "Select log line 08-20 14:36:41 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5592]: + - checkbox "Select log line" [ref=f61e5595] [cursor=pointer] + - generic [ref=f61e5596]: 08-20 14:36:41 + - generic [ref=f61e5597]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:41 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5599]: + - checkbox "Select log line" [ref=f61e5602] [cursor=pointer] + - generic [ref=f61e5603]: 08-20 14:36:41 + - generic [ref=f61e5604]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:41 artifact process started pid=22 artifact=artifacts/api-server" [ref=f61e5606]: + - checkbox "Select log line" [ref=f61e5609] [cursor=pointer] + - generic [ref=f61e5610]: 08-20 14:36:41 + - generic [ref=f61e5611]: artifact process started pid=22 artifact=artifacts/api-server + - button "Select log line 08-20 14:36:41 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5612]: + - checkbox "Select log line" [ref=f61e5615] [cursor=pointer] + - generic [ref=f61e5616]: 08-20 14:36:41 + - generic [ref=f61e5617]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:41 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5619]: + - checkbox "Select log line" [ref=f61e5622] [cursor=pointer] + - generic [ref=f61e5623]: 08-20 14:36:41 + - generic [ref=f61e5624]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:44 File \"/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/importer.py\", line 19, in import_from_string" [ref=f61e5626]: + - checkbox "Select log line" [ref=f61e5629] [cursor=pointer] + - generic [ref=f61e5630]: 08-20 14:36:44 + - generic [ref=f61e5631]: File "/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/importer.py", line 19, in import_from_string + - button "Select log line 08-20 14:36:44 ^" [ref=f61e5633]: + - checkbox "Select log line" [ref=f61e5636] [cursor=pointer] + - generic [ref=f61e5637]: 08-20 14:36:44 + - generic [ref=f61e5638]: ^ + - button "Select log line 08-20 14:36:44 config.load_app()" [ref=f61e5640]: + - checkbox "Select log line" [ref=f61e5643] [cursor=pointer] + - generic [ref=f61e5644]: 08-20 14:36:44 + - generic [ref=f61e5645]: config.load_app() + - button "Select log line 08-20 14:36:44 File \"/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/config.py\", line 428, in load_app" [ref=f61e5647]: + - checkbox "Select log line" [ref=f61e5650] [cursor=pointer] + - generic [ref=f61e5651]: 08-20 14:36:44 + - generic [ref=f61e5652]: File "/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/config.py", line 428, in load_app + - button "Select log line 08-20 14:36:44 raise exc from None" [ref=f61e5654]: + - checkbox "Select log line" [ref=f61e5657] [cursor=pointer] + - generic [ref=f61e5658]: 08-20 14:36:44 + - generic [ref=f61e5659]: raise exc from None + - button "Select log line 08-20 14:36:44 File \"/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/main.py\", line 609, in run" [ref=f61e5661]: + - checkbox "Select log line" [ref=f61e5664] [cursor=pointer] + - generic [ref=f61e5665]: 08-20 14:36:44 + - generic [ref=f61e5666]: File "/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/main.py", line 609, in run + - button "Select log line 08-20 14:36:44 File \"/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/importer.py\", line 22, in import_from_string" [ref=f61e5668]: + - checkbox "Select log line" [ref=f61e5671] [cursor=pointer] + - generic [ref=f61e5672]: 08-20 14:36:44 + - generic [ref=f61e5673]: File "/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/uvicorn/importer.py", line 22, in import_from_string + - button "Select log line 08-20 14:36:44 module = importlib.import_module(module_str)" [ref=f61e5675]: + - checkbox "Select log line" [ref=f61e5678] [cursor=pointer] + - generic [ref=f61e5679]: 08-20 14:36:44 + - generic [ref=f61e5680]: module = importlib.import_module(module_str) + - button "Select log line 08-20 14:36:44 )" [ref=f61e5682]: + - checkbox "Select log line" [ref=f61e5685] [cursor=pointer] + - generic [ref=f61e5686]: 08-20 14:36:44 + - generic [ref=f61e5687]: ) + - button "Select log line 08-20 14:36:44 ~~~~~~~~~~~~~~~^^" [ref=f61e5689]: + - checkbox "Select log line" [ref=f61e5692] [cursor=pointer] + - generic [ref=f61e5693]: 08-20 14:36:44 + - generic [ref=f61e5694]: ~~~~~~~~~~~~~~~^^ + - button "Select log line 08-20 14:36:44 return _bootstrap._gcd_import(name[level:], package, level)" [ref=f61e5696]: + - checkbox "Select log line" [ref=f61e5699] [cursor=pointer] + - generic [ref=f61e5700]: 08-20 14:36:44 + - generic [ref=f61e5701]: return _bootstrap._gcd_import(name[level:], package, level) + - button "Select log line 08-20 14:36:44 app," [ref=f61e5703]: + - checkbox "Select log line" [ref=f61e5706] [cursor=pointer] + - generic [ref=f61e5707]: 08-20 14:36:44 + - generic [ref=f61e5708]: app, + - button "Select log line 08-20 14:36:44 return import_from_string(self.app)" [ref=f61e5710]: + - checkbox "Select log line" [ref=f61e5713] [cursor=pointer] + - generic [ref=f61e5714]: 08-20 14:36:44 + - generic [ref=f61e5715]: return import_from_string(self.app) + - button "Select log line 08-20 14:36:44 File \"/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/click/core.py\", line 907, in invoke" [ref=f61e5717]: + - checkbox "Select log line" [ref=f61e5720] [cursor=pointer] + - generic [ref=f61e5721]: 08-20 14:36:44 + - generic [ref=f61e5722]: File "/home/runner/workspace/.pythonlibs/lib/python3.13/site-packages/click/core.py", line 907, in invoke + - button "Select log line 08-20 14:36:44 File \"\", line 1387, in _gcd_import" [ref=f61e5724]: + - checkbox "Select log line" [ref=f61e5727] [cursor=pointer] + - generic [ref=f61e5728]: 08-20 14:36:44 + - generic [ref=f61e5729]: File "", line 1387, in _gcd_import + - button "Select log line 08-20 14:36:44 ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" [ref=f61e5731]: + - checkbox "Select log line" [ref=f61e5734] [cursor=pointer] + - generic [ref=f61e5735]: 08-20 14:36:44 + - generic [ref=f61e5736]: ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + - button "Select log line 08-20 14:36:44 ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" [ref=f61e5738]: + - checkbox "Select log line" [ref=f61e5741] [cursor=pointer] + - generic [ref=f61e5742]: 08-20 14:36:44 + - generic [ref=f61e5743]: ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + - button "Select log line 08-20 14:36:46 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5745]: + - checkbox "Select log line" [ref=f61e5748] [cursor=pointer] + - generic [ref=f61e5749]: 08-20 14:36:46 + - generic [ref=f61e5750]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:47 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5752]: + - checkbox "Select log line" [ref=f61e5755] [cursor=pointer] + - generic [ref=f61e5756]: 08-20 14:36:47 + - generic [ref=f61e5757]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:49 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5759]: + - checkbox "Select log line" [ref=f61e5762] [cursor=pointer] + - generic [ref=f61e5763]: 08-20 14:36:49 + - generic [ref=f61e5764]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:50 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5766]: + - checkbox "Select log line" [ref=f61e5769] [cursor=pointer] + - generic [ref=f61e5770]: 08-20 14:36:50 + - generic [ref=f61e5771]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:52 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5773]: + - checkbox "Select log line" [ref=f61e5776] [cursor=pointer] + - generic [ref=f61e5777]: 08-20 14:36:52 + - generic [ref=f61e5778]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:53 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5780]: + - checkbox "Select log line" [ref=f61e5783] [cursor=pointer] + - generic [ref=f61e5784]: 08-20 14:36:53 + - generic [ref=f61e5785]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:55 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5787]: + - checkbox "Select log line" [ref=f61e5790] [cursor=pointer] + - generic [ref=f61e5791]: 08-20 14:36:55 + - generic [ref=f61e5792]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:55 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5794]: + - checkbox "Select log line" [ref=f61e5797] [cursor=pointer] + - generic [ref=f61e5798]: 08-20 14:36:55 + - generic [ref=f61e5799]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:55 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5801]: + - checkbox "Select log line" [ref=f61e5804] [cursor=pointer] + - generic [ref=f61e5805]: 08-20 14:36:55 + - generic [ref=f61e5806]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:57 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5808]: + - checkbox "Select log line" [ref=f61e5811] [cursor=pointer] + - generic [ref=f61e5812]: 08-20 14:36:57 + - generic [ref=f61e5813]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:57 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5815]: + - checkbox "Select log line" [ref=f61e5818] [cursor=pointer] + - generic [ref=f61e5819]: 08-20 14:36:57 + - generic [ref=f61e5820]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:58 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5822]: + - checkbox "Select log line" [ref=f61e5825] [cursor=pointer] + - generic [ref=f61e5826]: 08-20 14:36:58 + - generic [ref=f61e5827]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:36:58 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5829]: + - checkbox "Select log line" [ref=f61e5832] [cursor=pointer] + - generic [ref=f61e5833]: 08-20 14:36:58 + - generic [ref=f61e5834]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:00 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5836]: + - checkbox "Select log line" [ref=f61e5839] [cursor=pointer] + - generic [ref=f61e5840]: 08-20 14:37:00 + - generic [ref=f61e5841]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:00 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5843]: + - checkbox "Select log line" [ref=f61e5846] [cursor=pointer] + - generic [ref=f61e5847]: 08-20 14:37:00 + - generic [ref=f61e5848]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:00 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5850]: + - checkbox "Select log line" [ref=f61e5853] [cursor=pointer] + - generic [ref=f61e5854]: 08-20 14:37:00 + - generic [ref=f61e5855]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:02 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5857]: + - checkbox "Select log line" [ref=f61e5860] [cursor=pointer] + - generic [ref=f61e5861]: 08-20 14:37:02 + - generic [ref=f61e5862]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:04 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5864]: + - checkbox "Select log line" [ref=f61e5867] [cursor=pointer] + - generic [ref=f61e5868]: 08-20 14:37:04 + - generic [ref=f61e5869]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:06 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5871]: + - checkbox "Select log line" [ref=f61e5874] [cursor=pointer] + - generic [ref=f61e5875]: 08-20 14:37:06 + - generic [ref=f61e5876]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:06 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5878]: + - checkbox "Select log line" [ref=f61e5881] [cursor=pointer] + - generic [ref=f61e5882]: 08-20 14:37:06 + - generic [ref=f61e5883]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:08 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5885]: + - checkbox "Select log line" [ref=f61e5888] [cursor=pointer] + - generic [ref=f61e5889]: 08-20 14:37:08 + - generic [ref=f61e5890]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:10 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5892]: + - checkbox "Select log line" [ref=f61e5895] [cursor=pointer] + - generic [ref=f61e5896]: 08-20 14:37:10 + - generic [ref=f61e5897]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:10 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5899]: + - checkbox "Select log line" [ref=f61e5902] [cursor=pointer] + - generic [ref=f61e5903]: 08-20 14:37:10 + - generic [ref=f61e5904]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:10 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5906]: + - checkbox "Select log line" [ref=f61e5909] [cursor=pointer] + - generic [ref=f61e5910]: 08-20 14:37:10 + - generic [ref=f61e5911]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:12 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5913]: + - checkbox "Select log line" [ref=f61e5916] [cursor=pointer] + - generic [ref=f61e5917]: 08-20 14:37:12 + - generic [ref=f61e5918]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:12 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5920]: + - checkbox "Select log line" [ref=f61e5923] [cursor=pointer] + - generic [ref=f61e5924]: 08-20 14:37:12 + - generic [ref=f61e5925]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:14 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5927]: + - checkbox "Select log line" [ref=f61e5930] [cursor=pointer] + - generic [ref=f61e5931]: 08-20 14:37:14 + - generic [ref=f61e5932]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:14 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5934]: + - checkbox "Select log line" [ref=f61e5937] [cursor=pointer] + - generic [ref=f61e5938]: 08-20 14:37:14 + - generic [ref=f61e5939]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:14 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5941]: + - checkbox "Select log line" [ref=f61e5944] [cursor=pointer] + - generic [ref=f61e5945]: 08-20 14:37:14 + - generic [ref=f61e5946]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:15 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5948]: + - checkbox "Select log line" [ref=f61e5951] [cursor=pointer] + - generic [ref=f61e5952]: 08-20 14:37:15 + - generic [ref=f61e5953]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:16 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5955]: + - checkbox "Select log line" [ref=f61e5958] [cursor=pointer] + - generic [ref=f61e5959]: 08-20 14:37:16 + - generic [ref=f61e5960]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:16 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5962]: + - checkbox "Select log line" [ref=f61e5965] [cursor=pointer] + - generic [ref=f61e5966]: 08-20 14:37:16 + - generic [ref=f61e5967]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:17 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5969]: + - checkbox "Select log line" [ref=f61e5972] [cursor=pointer] + - generic [ref=f61e5973]: 08-20 14:37:17 + - generic [ref=f61e5974]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:18 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5976]: + - checkbox "Select log line" [ref=f61e5979] [cursor=pointer] + - generic [ref=f61e5980]: 08-20 14:37:18 + - generic [ref=f61e5981]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:19 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5983]: + - checkbox "Select log line" [ref=f61e5986] [cursor=pointer] + - generic [ref=f61e5987]: 08-20 14:37:19 + - generic [ref=f61e5988]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:19 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5990]: + - checkbox "Select log line" [ref=f61e5993] [cursor=pointer] + - generic [ref=f61e5994]: 08-20 14:37:19 + - generic [ref=f61e5995]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:20 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e5997]: + - checkbox "Select log line" [ref=f61e6000] [cursor=pointer] + - generic [ref=f61e6001]: 08-20 14:37:20 + - generic [ref=f61e6002]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:21 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6004]: + - checkbox "Select log line" [ref=f61e6007] [cursor=pointer] + - generic [ref=f61e6008]: 08-20 14:37:21 + - generic [ref=f61e6009]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:22 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6011]: + - checkbox "Select log line" [ref=f61e6014] [cursor=pointer] + - generic [ref=f61e6015]: 08-20 14:37:22 + - generic [ref=f61e6016]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:22 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6018]: + - checkbox "Select log line" [ref=f61e6021] [cursor=pointer] + - generic [ref=f61e6022]: 08-20 14:37:22 + - generic [ref=f61e6023]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:24 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6025]: + - checkbox "Select log line" [ref=f61e6028] [cursor=pointer] + - generic [ref=f61e6029]: 08-20 14:37:24 + - generic [ref=f61e6030]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:25 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6032]: + - checkbox "Select log line" [ref=f61e6035] [cursor=pointer] + - generic [ref=f61e6036]: 08-20 14:37:25 + - generic [ref=f61e6037]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:27 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6039]: + - checkbox "Select log line" [ref=f61e6042] [cursor=pointer] + - generic [ref=f61e6043]: 08-20 14:37:27 + - generic [ref=f61e6044]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:28 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6046]: + - checkbox "Select log line" [ref=f61e6049] [cursor=pointer] + - generic [ref=f61e6050]: 08-20 14:37:28 + - generic [ref=f61e6051]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:29 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6053]: + - checkbox "Select log line" [ref=f61e6056] [cursor=pointer] + - generic [ref=f61e6057]: 08-20 14:37:29 + - generic [ref=f61e6058]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:31 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6060]: + - checkbox "Select log line" [ref=f61e6063] [cursor=pointer] + - generic [ref=f61e6064]: 08-20 14:37:31 + - generic [ref=f61e6065]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:32 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6067]: + - checkbox "Select log line" [ref=f61e6070] [cursor=pointer] + - generic [ref=f61e6071]: 08-20 14:37:32 + - generic [ref=f61e6072]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:33 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6074]: + - checkbox "Select log line" [ref=f61e6077] [cursor=pointer] + - generic [ref=f61e6078]: 08-20 14:37:33 + - generic [ref=f61e6079]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:34 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6081]: + - checkbox "Select log line" [ref=f61e6084] [cursor=pointer] + - generic [ref=f61e6085]: 08-20 14:37:34 + - generic [ref=f61e6086]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:36 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6088]: + - checkbox "Select log line" [ref=f61e6091] [cursor=pointer] + - generic [ref=f61e6092]: 08-20 14:37:36 + - generic [ref=f61e6093]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:37 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6095]: + - checkbox "Select log line" [ref=f61e6098] [cursor=pointer] + - generic [ref=f61e6099]: 08-20 14:37:37 + - generic [ref=f61e6100]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:38 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6102]: + - checkbox "Select log line" [ref=f61e6105] [cursor=pointer] + - generic [ref=f61e6106]: 08-20 14:37:38 + - generic [ref=f61e6107]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:40 healthcheck failed error=healthcheck /api returned status 500" [ref=f61e6109]: + - checkbox "Select log line" [ref=f61e6112] [cursor=pointer] + - generic [ref=f61e6113]: 08-20 14:37:40 + - generic [ref=f61e6114]: healthcheck failed error=healthcheck /api returned status 500 + - button "Select log line 08-20 14:37:41 seccomp port detection incomplete, trying /proc/net/tcp fallback expected=[8080] seccompDetected=0" [ref=f61e6116]: + - checkbox "Select log line" [ref=f61e6119] [cursor=pointer] + - generic [ref=f61e6120]: 08-20 14:37:41 + - generic [ref=f61e6121]: seccomp port detection incomplete, trying /proc/net/tcp fallback expected=[8080] seccompDetected=0 + - button "Select log line 08-20 14:37:41 sending SIGTERM to artifact process artifact=artifacts/api-server" [ref=f61e6122]: + - checkbox "Select log line" [ref=f61e6125] [cursor=pointer] + - generic [ref=f61e6126]: 08-20 14:37:41 + - generic [ref=f61e6127]: sending SIGTERM to artifact process artifact=artifacts/api-server + - button "Select log line 08-20 14:37:41 all artifact processes stopped" [ref=f61e6128]: + - checkbox "Select log line" [ref=f61e6131] [cursor=pointer] + - generic [ref=f61e6132]: 08-20 14:37:41 + - generic [ref=f61e6133]: all artifact processes stopped + - button "Select log line 08-20 14:37:41 not all artifact ports opened within timeout expected=[8080] detected=0" [ref=f61e6134]: + - checkbox "Select log line" [ref=f61e6137] [cursor=pointer] + - generic [ref=f61e6138]: 08-20 14:37:41 + - generic [ref=f61e6139]: not all artifact ports opened within timeout expected=[8080] detected=0 + - button "Select log line 08-20 14:37:41 a port configuration was specified but the required port was never opened" [ref=f61e6141]: + - checkbox "Select log line" [ref=f61e6144] [cursor=pointer] + - generic [ref=f61e6145]: 08-20 14:37:41 + - generic [ref=f61e6146]: a port configuration was specified but the required port was never opened + - button "Select log line 08-20 16:36:56 received signal, shutting down signal=terminated" [ref=f61e6148]: + - checkbox "Select log line" [ref=f61e6151] [cursor=pointer] + - generic [ref=f61e6152]: 08-20 16:36:56 + - generic [ref=f61e6153]: received signal, shutting down signal=terminated + - button "Select log line 08-20 16:36:56 main done, exiting" [ref=f61e6154]: + - checkbox "Select log line" [ref=f61e6157] [cursor=pointer] + - generic [ref=f61e6158]: 08-20 16:36:56 + - generic [ref=f61e6159]: main done, exiting + - button "Select log line 08-20 16:45:15 starting up user application" [ref=f61e6160]: + - checkbox "Select log line" [ref=f61e6163] [cursor=pointer] + - generic [ref=f61e6164]: 08-20 16:45:15 + - generic [ref=f61e6165]: starting up user application + - 'button "Select log line 08-20 16:45:15 metasidecar: loaded enterprise status from environment is_enterprise=false" [ref=f61e6166]': + - checkbox "Select log line" [ref=f61e6169] [cursor=pointer] + - generic [ref=f61e6170]: 08-20 16:45:15 + - generic [ref=f61e6171]: "metasidecar: loaded enterprise status from environment is_enterprise=false" + - button "Select log line 08-20 16:45:15 artifact mode enabled runnable=0 static=12" [ref=f61e6172]: + - checkbox "Select log line" [ref=f61e6175] [cursor=pointer] + - generic [ref=f61e6176]: 08-20 16:45:15 + - generic [ref=f61e6177]: artifact mode enabled runnable=0 static=12 + - button "Select log line 08-20 16:45:15 starting artifact processes for monorepo deployment" [ref=f61e6178]: + - checkbox "Select log line" [ref=f61e6181] [cursor=pointer] + - generic [ref=f61e6182]: 08-20 16:45:15 + - generic [ref=f61e6183]: starting artifact processes for monorepo deployment + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-1/ publicDir=artifacts/ai-testing-lecture-1/dist/public" [ref=f61e6184]: + - checkbox "Select log line" [ref=f61e6187] [cursor=pointer] + - generic [ref=f61e6188]: 08-20 16:45:15 + - generic [ref=f61e6189]: registered static handler for artifact path=/ai-testing-lecture-1/ publicDir=artifacts/ai-testing-lecture-1/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-10/ publicDir=artifacts/ai-testing-lecture-10/dist/public" [ref=f61e6190]: + - checkbox "Select log line" [ref=f61e6193] [cursor=pointer] + - generic [ref=f61e6194]: 08-20 16:45:15 + - generic [ref=f61e6195]: registered static handler for artifact path=/ai-testing-lecture-10/ publicDir=artifacts/ai-testing-lecture-10/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact publicDir=artifacts/ai-testing-lecture-5/dist/public path=/ai-testing-lecture-5/" [ref=f61e6196]: + - checkbox "Select log line" [ref=f61e6199] [cursor=pointer] + - generic [ref=f61e6200]: 08-20 16:45:15 + - generic [ref=f61e6201]: registered static handler for artifact publicDir=artifacts/ai-testing-lecture-5/dist/public path=/ai-testing-lecture-5/ + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-2/ publicDir=artifacts/ai-testing-lecture-2/dist/public" [ref=f61e6202]: + - checkbox "Select log line" [ref=f61e6205] [cursor=pointer] + - generic [ref=f61e6206]: 08-20 16:45:15 + - generic [ref=f61e6207]: registered static handler for artifact path=/ai-testing-lecture-2/ publicDir=artifacts/ai-testing-lecture-2/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-3/ publicDir=artifacts/ai-testing-lecture-3/dist/public Copy" [ref=f61e6208]: + - checkbox "Select log line" [ref=f61e6211] [cursor=pointer] + - generic [ref=f61e6212]: 08-20 16:45:15 + - generic [ref=f61e6213]: registered static handler for artifact path=/ai-testing-lecture-3/ publicDir=artifacts/ai-testing-lecture-3/dist/public + - button "Copy" [ref=f61e6214] [cursor=pointer] + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-7/ publicDir=artifacts/ai-testing-lecture-7/dist/public" [ref=f61e6218]: + - checkbox "Select log line" [ref=f61e6221] [cursor=pointer] + - generic [ref=f61e6222]: 08-20 16:45:15 + - generic [ref=f61e6223]: registered static handler for artifact path=/ai-testing-lecture-7/ publicDir=artifacts/ai-testing-lecture-7/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact publicDir=artifacts/ai-testing-lecture-6/dist/public path=/ai-testing-lecture-6/" [ref=f61e6224]: + - checkbox "Select log line" [ref=f61e6227] [cursor=pointer] + - generic [ref=f61e6228]: 08-20 16:45:15 + - generic [ref=f61e6229]: registered static handler for artifact publicDir=artifacts/ai-testing-lecture-6/dist/public path=/ai-testing-lecture-6/ + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-4/ publicDir=artifacts/ai-testing-lecture-4/dist/public" [ref=f61e6230]: + - checkbox "Select log line" [ref=f61e6233] [cursor=pointer] + - generic [ref=f61e6234]: 08-20 16:45:15 + - generic [ref=f61e6235]: registered static handler for artifact path=/ai-testing-lecture-4/ publicDir=artifacts/ai-testing-lecture-4/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-8/ publicDir=artifacts/ai-testing-lecture-8/dist/public" [ref=f61e6236]: + - checkbox "Select log line" [ref=f61e6239] [cursor=pointer] + - generic [ref=f61e6240]: 08-20 16:45:15 + - generic [ref=f61e6241]: registered static handler for artifact path=/ai-testing-lecture-8/ publicDir=artifacts/ai-testing-lecture-8/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-lecture-9/ publicDir=artifacts/ai-testing-lecture-9/dist/public" [ref=f61e6242]: + - checkbox "Select log line" [ref=f61e6245] [cursor=pointer] + - generic [ref=f61e6246]: 08-20 16:45:15 + - generic [ref=f61e6247]: registered static handler for artifact path=/ai-testing-lecture-9/ publicDir=artifacts/ai-testing-lecture-9/dist/public + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ai-testing-academy/ publicDir=artifacts/ai-testing-academy/dist/public" [ref=f61e6248]: + - checkbox "Select log line" [ref=f61e6251] [cursor=pointer] + - generic [ref=f61e6252]: 08-20 16:45:15 + - generic [ref=f61e6253]: registered static handler for artifact path=/ai-testing-academy/ publicDir=artifacts/ai-testing-academy/dist/public + - button "Select log line 08-20 16:45:15 forwarding local port 0 to external port 80 (mapped as 1104)" [ref=f61e6254]: + - checkbox "Select log line" [ref=f61e6257] [cursor=pointer] + - generic [ref=f61e6258]: 08-20 16:45:15 + - generic [ref=f61e6259]: forwarding local port 0 to external port 80 (mapped as 1104) + - button "Select log line 08-20 16:45:15 registered static handler for artifact path=/ publicDir=artifacts/portfolio/dist/public" [ref=f61e6260]: + - checkbox "Select log line" [ref=f61e6263] [cursor=pointer] + - generic [ref=f61e6264]: 08-20 16:45:15 + - generic [ref=f61e6265]: registered static handler for artifact path=/ publicDir=artifacts/portfolio/dist/public + - button "Select log line 08-20 16:45:15 static-only deployment, waiting for signal" [ref=f61e6266]: + - checkbox "Select log line" [ref=f61e6269] [cursor=pointer] + - generic [ref=f61e6270]: 08-20 16:45:15 + - generic [ref=f61e6271]: static-only deployment, waiting for signal \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md index 41b377c..fcfa10a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -87,6 +87,7 @@ fly secrets set \ GEMINI_API_KEY=... \ GOOGLE_CLIENT_ID=... \ SESSION_SECRET=... \ + RATE_LIMIT_SALT=... \ SUPABASE_DB_PASSWORD=... \ SUPABASE_URL=... \ SUPABASE_ANON_KEY=... \ @@ -100,8 +101,16 @@ fly deploy --config server/fly.toml \ The trailing `.` matters because it is the Docker build context. The image installs uv, syncs the locked Python dependencies, copies `server/app`, and runs Uvicorn as a non-root user. +`RATE_LIMIT_SALT` is not optional, and it is the one whose absence is easiest to miss. In +production every quota is counted in Postgres keyed by an HMAC of the caller's identity, and +without a salt there is no key — so `SharedRateLimiter` fails closed and **every rate-limited +route refuses every caller**: Google sign-in, the AI proxy, and the admin seed route. The +refusal is a `429`, which reads to a visitor exactly like a quota they have exhausted. Any long +random string works, and `METRICS_ID_SALT` is accepted in its place. + Verify that `curl https://.fly.dev/api/healthz` returns `{"status":"ok"}` and -`/api/readyz` reports the database available, then open +`/api/readyz` reports the database available **and carries no `rateLimiting` field** — that +field appears only when quotas cannot count, and names what is missing. Then open `https://.fly.dev/api/docs` and confirm Scalar loads the runtime OpenAPI document. The current `server/fly.toml` names the app `ata-api`. Create or rename that Fly application @@ -127,7 +136,7 @@ Also set long, independent random `METRICS_TOKEN` and `METRICS_ID_SALT` values w production returns 404 without it. The salt HMAC-pseudonymizes verified user IDs for Grafana and must remain only on Fly. -Set a separate random `RATE_LIMIT_SALT` as well. Before enabling paid sales, configure and +Before enabling paid sales, configure and verify all of the following, then change `SALES_ENABLED` to `true`: - `STRIPE_COURSE_PRICE_ID`, `STRIPE_COURSE_PRODUCT_ID`, `STRIPE_COURSE_AMOUNT`, and diff --git a/lib/api-client-react/src/generated/api.schemas.ts b/lib/api-client-react/src/generated/api.schemas.ts index 87d24d3..f36a756 100644 --- a/lib/api-client-react/src/generated/api.schemas.ts +++ b/lib/api-client-react/src/generated/api.schemas.ts @@ -12,6 +12,8 @@ export interface HealthStatus { export interface ReadinessStatus { status: string; database: string; + /** Why production rate limiting cannot count, when it cannot. Absent on a healthy deployment. Present means every rate-limited route — sign-in and the AI proxy included — is refusing with 429 regardless of actual usage, so it is reported here rather than only in the logs. */ + rateLimiting?: string; } export interface AiProviderConfig { diff --git a/lib/api-spec/openapi.yaml b/lib/api-spec/openapi.yaml index ef9f232..1c152a4 100644 --- a/lib/api-spec/openapi.yaml +++ b/lib/api-spec/openapi.yaml @@ -365,6 +365,13 @@ components: type: string database: type: string + rateLimiting: + type: string + description: >- + Why production rate limiting cannot count, when it cannot. Absent on + a healthy deployment. Present means every rate-limited route — + sign-in and the AI proxy included — is refusing with 429 regardless + of actual usage, so it is reported here rather than only in the logs. required: [status, database] AiProviderConfig: type: object diff --git a/lib/api-zod/src/generated/api.ts b/lib/api-zod/src/generated/api.ts index c946e16..100cacc 100644 --- a/lib/api-zod/src/generated/api.ts +++ b/lib/api-zod/src/generated/api.ts @@ -21,6 +21,12 @@ export const HealthCheckResponse = zod.object({ export const ReadinessCheckResponse = zod.object({ status: zod.string(), database: zod.string(), + rateLimiting: zod + .string() + .optional() + .describe( + 'Why production rate limiting cannot count, when it cannot. Absent on a healthy deployment. Present means every rate-limited route — sign-in and the AI proxy included — is refusing with 429 regardless of actual usage, so it is reported here rather than only in the logs.', + ), }); /** diff --git a/lib/api-zod/src/generated/types/readinessStatus.ts b/lib/api-zod/src/generated/types/readinessStatus.ts index a899b59..02294b7 100644 --- a/lib/api-zod/src/generated/types/readinessStatus.ts +++ b/lib/api-zod/src/generated/types/readinessStatus.ts @@ -9,4 +9,6 @@ export interface ReadinessStatus { status: string; database: string; + /** Why production rate limiting cannot count, when it cannot. Absent on a healthy deployment. Present means every rate-limited route — sign-in and the AI proxy included — is refusing with 429 regardless of actual usage, so it is reported here rather than only in the logs. */ + rateLimiting?: string; } diff --git a/replit.md b/replit.md index faf113a..c31319a 100644 --- a/replit.md +++ b/replit.md @@ -63,6 +63,7 @@ is absent fails closed with a controlled 4xx/5xx response. The package scripts d | `UPSTREAM_API_BASE_URL` | Secretless Replit API artifact relays `/api/*` to Fly | FastAPI handles routes locally | | `METRICS_TOKEN` | Protects production `/metrics` scrapes | Metrics are available only outside production | | `METRICS_ID_SALT` | HMAC-pseudonymizes user labels in metrics | Authenticated users are labeled `redacted` | +| `RATE_LIMIT_SALT` | Production quotas are counted in Postgres | **In production every rate-limited route refuses every caller with 429** — sign-in, the AI proxy and the admin seed route. `METRICS_ID_SALT` is accepted instead. `/api/readyz` names it in a `rateLimiting` field | ### API structure diff --git a/server/app/dependencies.py b/server/app/dependencies.py index 430c746..85c9887 100644 --- a/server/app/dependencies.py +++ b/server/app/dependencies.py @@ -11,6 +11,7 @@ import hmac import logging import os +from collections.abc import Awaitable, Callable from typing import Annotated from fastapi import Cookie, Depends, Header, Request @@ -20,7 +21,7 @@ from .commerce import PurchaseRecorder, StripeGateway from .config import env from .content_store import ContentService, SupabaseContentStore -from .database import find_course_access, record_purchase +from .database import database_ready, find_course_access, record_purchase from .entitlements import EntitlementService from .errors import ServiceError from .google_auth import GoogleUser, verify_google_id_token @@ -30,6 +31,8 @@ logger = logging.getLogger(__name__) +DatabaseProbeFn = Callable[[], Awaitable[bool]] + burst_limiter = SharedRateLimiter("ai-burst", BURST_LIMIT, BURST_WINDOW) daily_limiter = SharedRateLimiter("ai-daily", DAILY_QUOTA, 24 * 60 * 60) admin_limiter = SharedRateLimiter("admin", 20, 15 * 60) @@ -97,6 +100,10 @@ def get_catalog() -> CourseCatalog | None: return course_catalog() +def get_database_probe() -> DatabaseProbeFn: + return database_ready + + def get_ai_gateway() -> AiGateway: return AiGateway() @@ -125,4 +132,5 @@ def get_entitlement_service() -> EntitlementService: Stripe = Annotated[StripeGateway, Depends(get_stripe_gateway)] Purchases = Annotated[PurchaseRecorder, Depends(get_purchase_recorder)] Entitlements = Annotated[EntitlementService, Depends(get_entitlement_service)] +DatabaseProbe = Annotated[DatabaseProbeFn, Depends(get_database_probe)] AdminOnly = Depends(require_admin) diff --git a/server/app/rate_limit.py b/server/app/rate_limit.py index 3fb03aa..0adf5f3 100644 --- a/server/app/rate_limit.py +++ b/server/app/rate_limit.py @@ -3,6 +3,7 @@ import asyncio import hashlib import hmac +import logging import os import time from collections import defaultdict, deque @@ -10,6 +11,25 @@ from .config import database_url, env from .database import hit_rate_limit +logger = logging.getLogger(__name__) + + +def shared_quota_problem() -> str | None: + """Why a production quota cannot be counted, or None when it can. + + A production deployment missing either of these refuses every rate-limited + request — sign-in and the AI proxy included — and the refusal is a 429 that + reads exactly like an exhausted quota. Naming the cause is what stops that + being invisible: it is reported by /api/readyz and logged once per bucket. + """ + if os.getenv("NODE_ENV") != "production": + return None + if not database_url(): + return "no database is configured, so quotas cannot be shared between workers" + if not (env("RATE_LIMIT_SALT") or env("METRICS_ID_SALT")): + return "RATE_LIMIT_SALT (or METRICS_ID_SALT) is not set" + return None + class MemoryRateLimiter: def __init__(self, limit: int, window_seconds: float) -> None: @@ -38,17 +58,29 @@ def __init__(self, bucket: str, limit: int, window_seconds: float) -> None: self.limit = limit self.window_seconds = window_seconds self.memory = MemoryRateLimiter(limit, window_seconds) + self._warned: set[str] = set() async def hit(self, key: str) -> tuple[bool, int]: if os.getenv("NODE_ENV") != "production": return await self.memory.hit(key) - salt = env("RATE_LIMIT_SALT") or env("METRICS_ID_SALT") - if not database_url() or not salt: + if problem := shared_quota_problem(): # A production quota without a shared database and private salt is - # not a quota. Fail closed instead of silently exposing paid APIs. + # not a quota. Fail closed instead of silently exposing paid APIs — + # but say so, because the caller only sees "too many requests". + self._warn_once(f"{problem}; every {self.bucket!r} request is refused until it is set") return False, 0 - digest = hmac.new(salt.encode(), key.encode(), hashlib.sha256).hexdigest() + salt = env("RATE_LIMIT_SALT") or env("METRICS_ID_SALT") + digest = hmac.new(salt.encode(), key.encode(), hashlib.sha256).hexdigest() # type: ignore[union-attr] try: return await hit_rate_limit(self.bucket, digest, self.limit, self.window_seconds) except Exception: + self._warn_once(f"the {self.bucket!r} quota store could not be reached") + logger.exception("Shared rate limit lookup failed for bucket %r", self.bucket) return False, 0 + + def _warn_once(self, message: str) -> None: + """Loud, but once per cause — this runs on the hot path of every request.""" + if message in self._warned: + return + self._warned.add(message) + logger.error("Rate limiting is failing closed: %s", message) diff --git a/server/app/routes/ops.py b/server/app/routes/ops.py index 3b39ca3..6495bce 100644 --- a/server/app/routes/ops.py +++ b/server/app/routes/ops.py @@ -8,9 +8,10 @@ from fastapi.responses import JSONResponse from scalar_fastapi import AgentScalarConfig, get_scalar_api_reference -from ..database import database_ready +from ..dependencies import DatabaseProbe from ..errors import error_response from ..metrics import metrics_authorized, prometheus_response +from ..rate_limit import shared_quota_problem router = APIRouter() @@ -45,7 +46,17 @@ async def health() -> dict[str, str]: @router.get("/api/readyz") -async def readiness(): +async def readiness(database_ready: DatabaseProbe): + """Readiness, plus anything degraded that a 200 would otherwise hide. + + The status code stays a function of the database alone, because Fly health- + checks this path and an unhealthy answer stops the machine. A deployment + whose quotas cannot count is broken for its users but still serving, so it + is named in the body instead of being turned into an outage. + """ if not await database_ready(): return JSONResponse({"status": "not_ready", "database": "unavailable"}, status_code=503) - return {"status": "ready", "database": "available"} + body = {"status": "ready", "database": "available"} + if problem := shared_quota_problem(): + body["rateLimiting"] = problem + return body diff --git a/server/tests/test_rate_limit_config.py b/server/tests/test_rate_limit_config.py new file mode 100644 index 0000000..32f87d2 --- /dev/null +++ b/server/tests/test_rate_limit_config.py @@ -0,0 +1,137 @@ +"""What a production deployment does when its quota store is not configured. + +This is the failure that took sign-in and the AI proxy down on the live site +and stayed invisible for it: `SharedRateLimiter` fails closed, which is right — +a quota that cannot count must not wave paid API calls through — but the caller +only ever saw `429 Too many requests`, indistinguishable from a real quota, and +nothing in the logs or on /api/readyz said otherwise. +""" + +from __future__ import annotations + +import logging + +import pytest + +from app.dependencies import get_database_probe +from app.rate_limit import SharedRateLimiter, shared_quota_problem + + +@pytest.fixture +def database_available(override_dependency): + """A reachable database, so readiness gets as far as reporting the quota.""" + + async def probe() -> bool: + return True + + override_dependency(get_database_probe, lambda: probe) + + +@pytest.fixture +def production(monkeypatch: pytest.MonkeyPatch): + """A production deployment whose configuration a test then removes a piece of.""" + monkeypatch.setenv("NODE_ENV", "production") + monkeypatch.setenv("DATABASE_URL", "postgresql://fixture/quotas") + monkeypatch.setenv("RATE_LIMIT_SALT", "fixture-salt") + monkeypatch.delenv("METRICS_ID_SALT", raising=False) + return monkeypatch + + +def test_a_fully_configured_production_deployment_reports_no_problem(production) -> None: + assert shared_quota_problem() is None + + +def test_local_and_test_runs_never_report_a_problem(monkeypatch: pytest.MonkeyPatch) -> None: + """Outside production the in-memory limiter is the intended path, not a fault.""" + monkeypatch.setenv("NODE_ENV", "development") + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.delenv("RATE_LIMIT_SALT", raising=False) + + assert shared_quota_problem() is None + + +def test_a_missing_salt_is_named(production) -> None: + production.delenv("RATE_LIMIT_SALT", raising=False) + + assert shared_quota_problem() == "RATE_LIMIT_SALT (or METRICS_ID_SALT) is not set" + + +def test_the_metrics_salt_is_accepted_in_its_place(production) -> None: + production.delenv("RATE_LIMIT_SALT", raising=False) + production.setenv("METRICS_ID_SALT", "fixture-salt") + + assert shared_quota_problem() is None + + +def test_a_missing_database_is_named(production) -> None: + production.delenv("DATABASE_URL", raising=False) + production.delenv("SUPABASE_DB_PASSWORD", raising=False) + + assert shared_quota_problem() is not None + assert "database" in shared_quota_problem() + + +@pytest.mark.asyncio +async def test_an_unconfigured_production_quota_refuses_every_caller(production) -> None: + """The behaviour is deliberate: no quota means no paid API calls get through.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + limiter = SharedRateLimiter("ai-burst", 15, 60) + + allowed, remaining = await limiter.hit("ip:198.51.100.4") + + assert allowed is False + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_the_refusal_is_logged_with_its_cause( + production, caplog: pytest.LogCaptureFixture +) -> None: + """Without this the outage is a 429 and nothing else, which is how it hid.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + limiter = SharedRateLimiter("login", 10, 300) + + with caplog.at_level(logging.ERROR, logger="app.rate_limit"): + await limiter.hit("ip:198.51.100.4") + + assert any("RATE_LIMIT_SALT" in record.message for record in caplog.records) + assert any("login" in record.message for record in caplog.records) + + +@pytest.mark.asyncio +async def test_the_cause_is_logged_once_rather_than_per_request( + production, caplog: pytest.LogCaptureFixture +) -> None: + """It runs on the hot path of every request; one line is a report, thousands are noise.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + limiter = SharedRateLimiter("ai-burst", 15, 60) + + with caplog.at_level(logging.ERROR, logger="app.rate_limit"): + for _ in range(5): + await limiter.hit("ip:198.51.100.4") + + assert len(caplog.records) == 1 + + +@pytest.mark.asyncio +async def test_readiness_names_the_problem_without_reporting_an_outage( + api_client, production, database_available +) -> None: + """Fly health-checks this path: a 503 here stops the machine, which is worse.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + + response = await api_client.get("/api/readyz") + + assert response.status_code == 200, "a broken quota must not be turned into an outage" + assert "RATE_LIMIT_SALT" in response.json()["rateLimiting"] + assert response.json()["status"] == "ready" + + +@pytest.mark.asyncio +async def test_readiness_stays_quiet_when_quotas_can_count( + api_client, production, database_available +) -> None: + response = await api_client.get("/api/readyz") + + assert response.status_code == 200 + assert "rateLimiting" not in response.json() From 06a0aeca5571aed3a262c0b2bd24cc8de7ba6d1b Mon Sep 17 00:00:00 2001 From: Amiel Peled Date: Thu, 20 Aug 2026 20:13:33 +0300 Subject: [PATCH 3/4] fix: three defects behind the sign-in failure, at their own layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to 68dde9a, which made the broken quota store visible but left it taking sign-in down. These are the three things that turned one missing secret into a silent, unexplained outage. 1. Failing closed was a single policy for four different quotas. SharedRateLimiter now takes `when_unavailable`. The two AI quotas keep refusing: they guard a key billed per call, and a limiter that cannot count must not wave those through. The login and admin quotas degrade to the in-memory limiter instead, because they bound brute force against credentials that are verified independently — a Google signature, an admin token compared with hmac.compare_digest — so refusing everyone protects nothing and takes authentication down. Degrading keeps a real per-worker bound; it loses only the sharing between workers and across restarts. "refuse" stays the default, so a new bucket opts into leniency. 2. The relay handed visitors control of their own quota identity. It forwarded every inbound header except the country, and the origin keys its quotas on `fly-client-ip`. A visitor who set that through the relay chose a fresh identity per request and never reached a limit. It is now dropped along with x-forwarded-for, x-real-ip, cf-connecting-ip and true-client-ip, for the reason the country header already was: an inbound value is the caller's claim about themselves, not an observation of them. 3. The client threw away what the server said. Every non-2xx became "Sign-in failed. Please try again." — including the 429 this whole outage produced, which told visitors to retry in a loop that could not succeed while implying the fault was theirs. AuthContext now keeps a reason: 429 is busy, 5xx is unavailable, everything else is the original wording, which is what a rejected credential actually is. 143 pytest, 242 unit, 172 component, 58 api, 72 contract, 69 e2e. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/components/account/GoogleSignIn.tsx | 9 ++- .../src/context/AuthContext.tsx | 42 ++++++++++-- .../ai-testing-academy/src/lib/locales/en.ts | 2 + .../ai-testing-academy/src/lib/locales/he.ts | 2 + server/app/dependencies.py | 9 ++- server/app/rate_limit.py | 64 +++++++++++++++---- server/app/relay.py | 17 ++++- server/tests/test_rate_limit_config.py | 49 ++++++++++++-- server/tests/test_relay.py | 36 +++++++++++ tests/component/GoogleSignIn.spec.tsx | 53 +++++++++++++++ tests/component/fixtures.tsx | 12 ++++ 11 files changed, 265 insertions(+), 30 deletions(-) diff --git a/artifacts/ai-testing-academy/src/components/account/GoogleSignIn.tsx b/artifacts/ai-testing-academy/src/components/account/GoogleSignIn.tsx index da723ee..309fb1d 100644 --- a/artifacts/ai-testing-academy/src/components/account/GoogleSignIn.tsx +++ b/artifacts/ai-testing-academy/src/components/account/GoogleSignIn.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef } from 'react'; -import { useAuth } from '../../context/AuthContext'; +import { useAuth, type AuthFailure } from '../../context/AuthContext'; import { useLocale } from '../../context/LocaleContext'; /** @@ -16,6 +16,11 @@ export function GoogleSignIn() { const { configured, user, authenticating, authError, signOut, renderButton } = useAuth(); const { lang, S } = useLocale(); const buttonHost = useRef(null); + const errorText: Record = { + busy: S.signInBusy, + unavailable: S.signInUnavailable, + failed: S.signInError, + }; useEffect(() => { const host = buttonHost.current; @@ -89,7 +94,7 @@ export function GoogleSignIn() { /> {authError && (

- {S.signInError} + {errorText[authError]}

)} diff --git a/artifacts/ai-testing-academy/src/context/AuthContext.tsx b/artifacts/ai-testing-academy/src/context/AuthContext.tsx index 95b4cfa..12e2ab4 100644 --- a/artifacts/ai-testing-academy/src/context/AuthContext.tsx +++ b/artifacts/ai-testing-academy/src/context/AuthContext.tsx @@ -6,12 +6,23 @@ import { type GoogleUser, } from '../lib/googleIdentity'; +/** + * Why a sign-in did not complete. + * + * The three are not interchangeable to the person reading them: `busy` will + * pass on its own and is worth waiting out, `unavailable` is the server's + * problem and no amount of retrying helps, and `failed` is everything else. + * Collapsing them into one line is how a server-side outage looked to a + * visitor like their own sign-in going wrong. + */ +export type AuthFailure = 'busy' | 'unavailable' | 'failed'; + interface AuthContextValue { /** False when the site was built without a client ID; sign-in stays hidden. */ configured: boolean; user: GoogleUser | null; authenticating: boolean; - authError: boolean; + authError: AuthFailure | null; signOut: () => Promise; /** * Renders Google's own button into `parent`. Google draws it itself — the @@ -23,6 +34,20 @@ interface AuthContextValue { const AuthContext = createContext(null); +/** + * What the server's status code means for the person who just clicked. + * + * 429 is the one that matters: the API refuses sign-in when its rate limiter + * cannot count, which is a server misconfiguration that presents as a quota. + * Telling the visitor to simply try again sends them round a loop that cannot + * succeed; telling them it is busy at least matches what they are seeing. + */ +function failureFor(status: number): AuthFailure { + if (status === 429) return 'busy'; + if (status >= 500) return 'unavailable'; + return 'failed'; +} + const BUTTON_OPTIONS: Omit = { type: 'standard', theme: 'outline', @@ -47,7 +72,7 @@ export function AuthProvider({ children, clientId = googleClientId() }: AuthProv const configured = resolvedClientId !== ''; const [user, setUser] = useState(null); const [authenticating, setAuthenticating] = useState(false); - const [authError, setAuthError] = useState(false); + const [authError, setAuthError] = useState(null); // A build-time client ID remains supported for local/offline builds. When it // is absent, ask the same-origin API for the public ID at runtime so static @@ -70,7 +95,7 @@ export function AuthProvider({ children, clientId = googleClientId() }: AuthProv const handleCredential = useCallback(async (credential: string) => { setAuthenticating(true); - setAuthError(false); + setAuthError(null); try { const response = await fetch('/api/auth/google', { method: 'POST', @@ -78,12 +103,15 @@ export function AuthProvider({ children, clientId = googleClientId() }: AuthProv headers: { 'content-type': 'application/json' }, body: JSON.stringify({ credential }), }); - if (!response.ok) throw new Error('The server did not accept the Google credential'); + if (!response.ok) { + setAuthError(failureFor(response.status)); + return; + } const body = (await response.json()) as { user?: GoogleUser }; if (!body.user) throw new Error('The server returned no signed-in user'); setUser(body.user); } catch { - setAuthError(true); + setAuthError('failed'); } finally { setAuthenticating(false); } @@ -93,7 +121,7 @@ export function AuthProvider({ children, clientId = googleClientId() }: AuthProv // account without being asked, so the next click is a real choice. const signOut = useCallback(async () => { setAuthenticating(true); - setAuthError(false); + setAuthError(null); // Stop Google's automatic account selection immediately. The verified // local session remains visible until our server confirms deletion below. window.google?.accounts.id.disableAutoSelect(); @@ -105,7 +133,7 @@ export function AuthProvider({ children, clientId = googleClientId() }: AuthProv if (!response.ok) throw new Error('The server did not end the session'); setUser(null); } catch { - setAuthError(true); + setAuthError('failed'); } finally { setAuthenticating(false); } diff --git a/artifacts/ai-testing-academy/src/lib/locales/en.ts b/artifacts/ai-testing-academy/src/lib/locales/en.ts index ab38534..e929b20 100644 --- a/artifacts/ai-testing-academy/src/lib/locales/en.ts +++ b/artifacts/ai-testing-academy/src/lib/locales/en.ts @@ -245,6 +245,8 @@ export const en = { signInAria: 'Sign in with Google', signingInStatus: 'Signing in…', signInError: 'Sign-in failed. Please try again.', + signInBusy: 'Too many sign-in attempts right now. Please wait a minute and try again.', + signInUnavailable: 'Sign-in is temporarily unavailable on this server.', signOutBtn: 'Sign out', uploadPrompt: '📁 Click or drag your resume here — PDF, DOCX, or TXT', uploadLoadedMid: ' · ', diff --git a/artifacts/ai-testing-academy/src/lib/locales/he.ts b/artifacts/ai-testing-academy/src/lib/locales/he.ts index 7bb62b5..b0c4052 100644 --- a/artifacts/ai-testing-academy/src/lib/locales/he.ts +++ b/artifacts/ai-testing-academy/src/lib/locales/he.ts @@ -241,6 +241,8 @@ export const he: Locale = { signInAria: 'התחברות עם Google', signingInStatus: 'מתחבר…', signInError: 'ההתחברות נכשלה. נסה שוב.', + signInBusy: 'יותר מדי ניסיונות התחברות כרגע. יש להמתין דקה ולנסות שוב.', + signInUnavailable: 'ההתחברות אינה זמינה כרגע בשרת הזה.', signOutBtn: 'התנתקות', uploadPrompt: '📁 לחץ או גרור את קורות החיים לכאן — PDF, DOCX, או TXT', uploadLoadedMid: ' · ', diff --git a/server/app/dependencies.py b/server/app/dependencies.py index 85c9887..2bda431 100644 --- a/server/app/dependencies.py +++ b/server/app/dependencies.py @@ -33,10 +33,15 @@ DatabaseProbeFn = Callable[[], Awaitable[bool]] +# The two AI quotas guard a key that is billed per call, so they refuse rather +# than guess when the shared store is unavailable. The two credential quotas +# guard secrets that are verified independently — a Google signature, an admin +# token compared in constant time — where refusing everyone is an authentication +# outage that protects nothing, so they fall back to a per-worker bound. burst_limiter = SharedRateLimiter("ai-burst", BURST_LIMIT, BURST_WINDOW) daily_limiter = SharedRateLimiter("ai-daily", DAILY_QUOTA, 24 * 60 * 60) -admin_limiter = SharedRateLimiter("admin", 20, 15 * 60) -login_limiter = SharedRateLimiter("login", 10, 5 * 60) +admin_limiter = SharedRateLimiter("admin", 20, 15 * 60, when_unavailable="degrade") +login_limiter = SharedRateLimiter("login", 10, 5 * 60, when_unavailable="degrade") def bearer_token(authorization: str | None) -> str: diff --git a/server/app/rate_limit.py b/server/app/rate_limit.py index 0adf5f3..7c15cfc 100644 --- a/server/app/rate_limit.py +++ b/server/app/rate_limit.py @@ -7,6 +7,7 @@ import os import time from collections import defaultdict, deque +from typing import Literal from .config import database_url, env from .database import hit_rate_limit @@ -50,13 +51,40 @@ async def hit(self, key: str) -> tuple[bool, int]: return True, self.limit - len(hits) +WhenUnavailable = Literal["refuse", "degrade"] + + class SharedRateLimiter: - """Postgres-backed in production, deterministic in-memory in local/test runs.""" + """Postgres-backed in production, deterministic in-memory in local/test runs. + + `when_unavailable` decides what happens if the shared store cannot be used, + and the right answer differs by what the quota protects: + + "refuse" the quota guards something that costs money on every call. A + limiter that cannot count must not wave those through, so the + caller is turned away. This is the default: a new bucket has to + opt into being lenient rather than inherit it. - def __init__(self, bucket: str, limit: int, window_seconds: float) -> None: + "degrade" the quota is a brute-force bound on a credential that is itself + verified — a Google-signed token, an admin token compared with + hmac.compare_digest. Refusing everyone here does not protect + anything; it takes authentication down. Falling back to the + in-memory limiter keeps a real bound per worker, and loses only + the sharing between workers and across restarts. + """ + + def __init__( + self, + bucket: str, + limit: int, + window_seconds: float, + *, + when_unavailable: WhenUnavailable = "refuse", + ) -> None: self.bucket = bucket self.limit = limit self.window_seconds = window_seconds + self.when_unavailable = when_unavailable self.memory = MemoryRateLimiter(limit, window_seconds) self._warned: set[str] = set() @@ -64,23 +92,31 @@ async def hit(self, key: str) -> tuple[bool, int]: if os.getenv("NODE_ENV") != "production": return await self.memory.hit(key) if problem := shared_quota_problem(): - # A production quota without a shared database and private salt is - # not a quota. Fail closed instead of silently exposing paid APIs — - # but say so, because the caller only sees "too many requests". - self._warn_once(f"{problem}; every {self.bucket!r} request is refused until it is set") - return False, 0 + self._warn_once(problem) + return await self._unavailable(key) salt = env("RATE_LIMIT_SALT") or env("METRICS_ID_SALT") - digest = hmac.new(salt.encode(), key.encode(), hashlib.sha256).hexdigest() # type: ignore[union-attr] + assert salt is not None # shared_quota_problem() has just established this + digest = hmac.new(salt.encode(), key.encode(), hashlib.sha256).hexdigest() try: return await hit_rate_limit(self.bucket, digest, self.limit, self.window_seconds) except Exception: - self._warn_once(f"the {self.bucket!r} quota store could not be reached") + self._warn_once("the shared quota store could not be reached") logger.exception("Shared rate limit lookup failed for bucket %r", self.bucket) - return False, 0 + return await self._unavailable(key) + + async def _unavailable(self, key: str) -> tuple[bool, int]: + if self.when_unavailable == "degrade": + return await self.memory.hit(key) + return False, 0 - def _warn_once(self, message: str) -> None: + def _warn_once(self, problem: str) -> None: """Loud, but once per cause — this runs on the hot path of every request.""" - if message in self._warned: + if problem in self._warned: return - self._warned.add(message) - logger.error("Rate limiting is failing closed: %s", message) + self._warned.add(problem) + consequence = ( + f"every {self.bucket!r} request is refused" + if self.when_unavailable == "refuse" + else f"the {self.bucket!r} quota is per-worker only" + ) + logger.error("Shared rate limiting is unavailable: %s; %s", problem, consequence) diff --git a/server/app/relay.py b/server/app/relay.py index 5d618e3..aa6a6cb 100644 --- a/server/app/relay.py +++ b/server/app/relay.py @@ -26,6 +26,21 @@ "upgrade", } +# Headers that name who the caller is. The origin trusts `fly-client-ip` to key +# its quotas, so a visitor who can set it through this relay picks their own +# quota identity and never runs out. They are dropped here for the same reason +# `x-academy-client-country` is: an inbound value is the visitor's claim about +# themselves, not an observation of them. +_CLIENT_IDENTITY = { + "cf-connecting-ip", + "fly-client-ip", + "true-client-ip", + "x-academy-client-country", + "x-client-ip", + "x-forwarded-for", + "x-real-ip", +} + def upstream_api_base_url() -> str | None: value = os.getenv("UPSTREAM_API_BASE_URL", "").strip().rstrip("/") @@ -53,7 +68,7 @@ async def relay_api_request(request: Request, upstream: str) -> Response: headers = { name: value for name, value in request.headers.items() - if name.lower() not in _HOP_BY_HOP and name.lower() != "x-academy-client-country" + if name.lower() not in _HOP_BY_HOP and name.lower() not in _CLIENT_IDENTITY } headers["x-forwarded-host"] = request.headers.get("host", "") headers["x-forwarded-proto"] = request.url.scheme diff --git a/server/tests/test_rate_limit_config.py b/server/tests/test_rate_limit_config.py index 32f87d2..dc9e30b 100644 --- a/server/tests/test_rate_limit_config.py +++ b/server/tests/test_rate_limit_config.py @@ -72,7 +72,7 @@ def test_a_missing_database_is_named(production) -> None: @pytest.mark.asyncio -async def test_an_unconfigured_production_quota_refuses_every_caller(production) -> None: +async def test_a_quota_guarding_a_billed_key_refuses_every_caller(production) -> None: """The behaviour is deliberate: no quota means no paid API calls get through.""" production.delenv("RATE_LIMIT_SALT", raising=False) limiter = SharedRateLimiter("ai-burst", 15, 60) @@ -83,19 +83,60 @@ async def test_an_unconfigured_production_quota_refuses_every_caller(production) assert remaining == 0 +@pytest.mark.asyncio +async def test_a_credential_quota_degrades_instead_of_taking_authentication_down( + production, +) -> None: + """Refusing every sign-in protects nothing — the token is verified either way.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + limiter = SharedRateLimiter("login", 10, 300, when_unavailable="degrade") + + allowed, remaining = await limiter.hit("ip:198.51.100.4") + + assert allowed is True + assert remaining == 9 + + +@pytest.mark.asyncio +async def test_a_degraded_quota_is_still_a_quota(production) -> None: + """Degrading is a fallback to a per-worker bound, not to no bound at all.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + limiter = SharedRateLimiter("login", 3, 300, when_unavailable="degrade") + + results = [await limiter.hit("ip:198.51.100.4") for _ in range(5)] + + assert [allowed for allowed, _ in results] == [True, True, True, False, False] + + +@pytest.mark.asyncio +async def test_a_degraded_quota_still_separates_callers(production) -> None: + production.delenv("RATE_LIMIT_SALT", raising=False) + limiter = SharedRateLimiter("login", 1, 300, when_unavailable="degrade") + + first, _ = await limiter.hit("ip:198.51.100.4") + other, _ = await limiter.hit("ip:203.0.113.9") + + assert (first, other) == (True, True), "one caller must not spend another's allowance" + + +def test_refusing_is_the_default_so_a_new_bucket_cannot_inherit_leniency() -> None: + assert SharedRateLimiter("something-new", 5, 60).when_unavailable == "refuse" + + @pytest.mark.asyncio async def test_the_refusal_is_logged_with_its_cause( production, caplog: pytest.LogCaptureFixture ) -> None: """Without this the outage is a 429 and nothing else, which is how it hid.""" production.delenv("RATE_LIMIT_SALT", raising=False) - limiter = SharedRateLimiter("login", 10, 300) + limiter = SharedRateLimiter("login", 10, 300, when_unavailable="degrade") with caplog.at_level(logging.ERROR, logger="app.rate_limit"): await limiter.hit("ip:198.51.100.4") - assert any("RATE_LIMIT_SALT" in record.message for record in caplog.records) - assert any("login" in record.message for record in caplog.records) + logged = " ".join(record.getMessage() for record in caplog.records) + assert "RATE_LIMIT_SALT" in logged + assert "login" in logged @pytest.mark.asyncio diff --git a/server/tests/test_relay.py b/server/tests/test_relay.py index 70a5ef3..e1386eb 100644 --- a/server/tests/test_relay.py +++ b/server/tests/test_relay.py @@ -45,3 +45,39 @@ def test_production_relay_requires_https(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("UPSTREAM_API_BASE_URL", "http://api.example.test") with pytest.raises(RuntimeError, match="must use HTTPS"): relay.upstream_api_base_url() + + +@pytest.mark.parametrize( + "header", + ["fly-client-ip", "x-forwarded-for", "x-real-ip", "cf-connecting-ip", "true-client-ip"], +) +@pytest.mark.asyncio +async def test_relay_drops_a_client_supplied_identity_header( + api_client, relay_requests, header: str +) -> None: + """The origin keys its quotas on these, so a forwarded one is a free quota. + + The relay is the only thing between a visitor and the API, and everything it + forwards arrives at the origin looking like the origin's own edge told it so. + A visitor who can set `fly-client-ip` through it picks a fresh quota identity + per request and never reaches a limit. + """ + await api_client.get("/api/healthz2", headers={header: "203.0.113.77"}) + + assert relay_requests, "the request should have been relayed" + assert header not in relay_requests[0].headers + + +@pytest.mark.asyncio +async def test_relay_still_forwards_the_headers_the_api_needs(api_client, relay_requests) -> None: + """The strip is narrow: authentication and content headers have to survive it.""" + await api_client.post( + "/api/auth/google", + json={"credential": "x"}, + headers={"authorization": "Bearer token", "fly-client-ip": "203.0.113.77"}, + ) + + forwarded = relay_requests[0].headers + assert forwarded["authorization"] == "Bearer token" + assert forwarded["content-type"] == "application/json" + assert "fly-client-ip" not in forwarded diff --git a/tests/component/GoogleSignIn.spec.tsx b/tests/component/GoogleSignIn.spec.tsx index 1266c14..74fd5fa 100644 --- a/tests/component/GoogleSignIn.spec.tsx +++ b/tests/component/GoogleSignIn.spec.tsx @@ -252,3 +252,56 @@ test.describe('the credential’s lifetime', () => { await expect(component.locator('.nav-account-name')).toBeVisible(); }); }); + +test.describe('when the server refuses the credential', () => { + /** + * The three refusals are not the same problem, and for a long time they all + * produced one line: "Sign-in failed. Please try again." + * + * The 429 is the case that matters. The API refuses sign-in when its rate + * limiter cannot reach its store — a server misconfiguration — and that + * arrives as a quota response. A visitor told to try again does exactly that, + * in a loop that cannot succeed, while the message insists the problem is + * theirs. Naming it costs one string and is the difference between a visitor + * waiting and a visitor concluding the site is broken. + */ + test('a rate-limited sign-in says to wait, not to try again', async ({ googleSignIn }) => { + const component = await googleSignIn.mount(); + googleSignIn.refuseNextSignInWith(429); + + await googleSignIn.signInWith(validCredential()); + + await expect(component.getByRole('alert')).toHaveText(en.s.signInBusy); + await expect(component.getByRole('alert')).not.toHaveText(en.s.signInError); + }); + + test('a server fault says so, rather than blaming the visitor', async ({ googleSignIn }) => { + const component = await googleSignIn.mount(); + googleSignIn.refuseNextSignInWith(503); + + await googleSignIn.signInWith(validCredential()); + + await expect(component.getByRole('alert')).toHaveText(en.s.signInUnavailable); + }); + + test('a rejected credential keeps the generic message', async ({ googleSignIn }) => { + // 401 is the visitor's own credential being refused, which is what the + // original wording actually describes. + const component = await googleSignIn.mount(); + googleSignIn.refuseNextSignInWith(401); + + await googleSignIn.signInWith(validCredential()); + + await expect(component.getByRole('alert')).toHaveText(en.s.signInError); + }); + + test('the button stays available, whatever the reason', async ({ googleSignIn }) => { + const component = await googleSignIn.mount(); + googleSignIn.refuseNextSignInWith(429); + + await googleSignIn.signInWith(validCredential()); + + await expect(component.locator('#fakeGoogleButton')).toBeVisible(); + await expect(component.locator('.nav-account-name')).toHaveCount(0); + }); +}); diff --git a/tests/component/fixtures.tsx b/tests/component/fixtures.tsx index a6f1406..35cd85c 100644 --- a/tests/component/fixtures.tsx +++ b/tests/component/fixtures.tsx @@ -86,6 +86,8 @@ type GoogleSignInHarness = { /** Seeds the API session and a legacy browser credential before mounting. */ seedCredential: (credential: string) => Promise; /** Clicks Google's button, which hands `credential` back through its callback. */ + /** Make the next credential POST fail with this status, as a broken server would. */ + refuseNextSignInWith: (status: number) => void; signInWith: (credential: string) => Promise; /** Starts sign-in and holds the API response until the returned function runs. */ beginSignInWith: (credential: string) => Promise<() => Promise>; @@ -268,7 +270,14 @@ export const test = base.extend({ await page.route('**/api/auth/config', route => route.fulfill({ status: 200, json: { clientId: runtimeClientId } }), ); + let refuseNextLoginWith: number | null = null; await page.route('**/api/auth/google', async route => { + if (refuseNextLoginWith !== null) { + const status = refuseNextLoginWith; + refuseNextLoginWith = null; + await route.fulfill({ status, json: { error: 'refused by the fixture' } }); + return; + } const body = route.request().postDataJSON() as { credential?: string }; sessionUser = body.credential ? verifiedUser(body.credential) : null; if (pauseNextLogin) { @@ -312,6 +321,9 @@ export const test = base.extend({ sessionUser = verifiedUser(credential); await setBeforeMount('ata_google_credential', credential); }, + refuseNextSignInWith: (status: number) => { + refuseNextLoginWith = status; + }, signInWith: async credential => { await page.evaluate(value => { (window as GoogleStubWindow).__credential = value; From f877dd6f06f9d190e085631a3482bb5f2733f5de Mon Sep 17 00:00:00 2001 From: Amiel Peled Date: Thu, 20 Aug 2026 20:15:46 +0300 Subject: [PATCH 4/4] test: pin the reported outage to the route it broke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The limiter tests cover the policy; these cover what a visitor actually hit. A production deployment with no RATE_LIMIT_SALT answered every credential POST with 429, and the other half of the trade — the billed AI key staying shut under the same conditions — is asserted alongside it, so a future edit cannot recover sign-in by making everything lenient. Co-Authored-By: Claude Opus 5 (1M context) --- server/tests/test_rate_limit_config.py | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/server/tests/test_rate_limit_config.py b/server/tests/test_rate_limit_config.py index dc9e30b..3980402 100644 --- a/server/tests/test_rate_limit_config.py +++ b/server/tests/test_rate_limit_config.py @@ -176,3 +176,37 @@ async def test_readiness_stays_quiet_when_quotas_can_count( assert response.status_code == 200 assert "rateLimiting" not in response.json() + + +@pytest.mark.asyncio +async def test_sign_in_survives_a_production_deployment_with_no_salt( + api_client, production +) -> None: + """The reported outage, as a test. + + A production deployment with no RATE_LIMIT_SALT used to answer every + credential POST with 429, so nobody could sign in and the site told them + they had tried too often. The login quota degrades now, so the request + reaches verification and is judged on the credential itself — 401 here, + because this one is not a Google token. Any status but 429 is the point. + """ + production.delenv("RATE_LIMIT_SALT", raising=False) + + response = await api_client.post("/api/auth/google", json={"credential": "not-a-token"}) + + assert response.status_code != 429, "a missing salt must not present as an exhausted quota" + assert response.status_code == 401 + + +@pytest.mark.asyncio +async def test_the_ai_proxy_still_refuses_when_the_quota_cannot_count( + api_client, production +) -> None: + """The other half of the trade: sign-in recovers, the billed key stays shut.""" + production.delenv("RATE_LIMIT_SALT", raising=False) + + response = await api_client.post( + "/api/ai/generate", json={"messages": [{"role": "user", "content": "hi"}]} + ) + + assert response.status_code == 429