From 8bcf91f4e25dfe1da7f8ac3fe02e9a3e4d18da84 Mon Sep 17 00:00:00 2001 From: Krish Date: Sat, 22 Aug 2026 16:25:36 +0530 Subject: [PATCH] test: cover the quiz pipeline The suite was one assertion against the root route. Every defect fixed in PRs #1 through #4 shipped past it, including one that returned HTTP 200 while generating questions about empty files - a status-code check could never have caught that. 61 tests: 51 backend under pytest, 10 frontend under node:test. The cases encode the bugs that actually happened rather than chasing coverage. Selection is pinned against the ascending sort that picked empty __init__.py files, and against the prefix match that let nested vendored directories through - together with the substring fix that was rejected, since "dist/" must not match "mydist/". Parsing is pinned against a malformed complexity object reaching a required schema field and 500ing the request. The API tests pin that /submit returns a follow-up and not a score, so anything relying on the old contract fails loudly, and that a client omitting the paste fields still succeeds. One test documents rather than asserts correctness: pasting after a long pause clears the 100ms timing guard. Pinning it makes a future change to the thresholds a decision instead of an accident. Every test is hermetic. Gemini and Mongo are mocked throughout, so the suite spends no API quota, needs no credentials, and passes in CI where no .env exists - verified by running it from the repo root the way CI invokes it, where .env does not resolve. Three things beyond the tests themselves: tests/test_quiz.py is removed. Its docstring described it as a placeholder to replace once mocked tests existed, and its only assertion now lives in test_quiz_api.py. backend/pytest.ini adds asyncio_mode = auto for the async service tests, and filters the warning the EOL google-generativeai package prints on every import. CI now runs the frontend tests, which otherwise would not run at all, and uses npm ci rather than npm install. That is the correct pairing with the lockfile committed in PR #2 - npm install can silently update it, which defeats the point of committing it. The frontend suite uses Node's built-in runner, so this adds no dependency. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PET9qKZXhgjEbZK7MReYQj --- .github/workflows/ci.yml | 4 +- backend/pytest.ini | 5 + backend/tests/conftest.py | 52 ++++++ backend/tests/test_gemini_client.py | 80 +++++++++ backend/tests/test_github_client.py | 143 ++++++++++++++++ backend/tests/test_quiz.py | 16 -- backend/tests/test_quiz_api.py | 162 ++++++++++++++++++ backend/tests/test_quiz_service.py | 161 +++++++++++++++++ frontend/package.json | 3 +- .../src/features/quiz/pasteDetect.test.js | 83 +++++++++ 10 files changed, 691 insertions(+), 18 deletions(-) create mode 100644 backend/pytest.ini create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_gemini_client.py create mode 100644 backend/tests/test_github_client.py delete mode 100644 backend/tests/test_quiz.py create mode 100644 backend/tests/test_quiz_api.py create mode 100644 backend/tests/test_quiz_service.py create mode 100644 frontend/src/features/quiz/pasteDetect.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e5b633..a002863 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,9 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "20" - - run: npm install + - run: npm ci + working-directory: frontend + - run: npm test working-directory: frontend - run: npm run build working-directory: frontend diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..fdc6975 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +asyncio_mode = auto +filterwarnings = + ignore::FutureWarning + ignore::DeprecationWarning diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..877c35d --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,52 @@ +""" +Shared fixtures. + +Every test here is hermetic: Gemini and Mongo are always mocked, so the suite +never spends API quota, never needs credentials, and passes in CI where no .env +exists. If a test in this package makes a real network call, that is a bug in the +test. +""" +import pytest + +QUESTIONS = [ + {"id": "q1", "question": "Why a Session?", "file_reference": "sessions.py", "category": "problem"}, + {"id": "q2", "question": "How does preparation work?", "file_reference": "models.py", "category": "logic"}, + {"id": "q3", "question": "Why urllib3?", "file_reference": "adapters.py", "category": "stack"}, +] + +COMPLEXITY = {"tier": "complex", "reasoning": "many interacting parts"} + + +@pytest.fixture +def questions(): + return [dict(q) for q in QUESTIONS] + + +@pytest.fixture +def complexity(): + return dict(COMPLEXITY) + + +@pytest.fixture +def attempt(questions, complexity): + """A stored quiz_attempts document mid-flow, awaiting its follow-up.""" + return { + "_id": "quiz-1", + "repo_url": "https://github.com/psf/requests", + "user_id": "u1", + "questions": questions, + "complexity": complexity, + "answers": [ + {"question_id": "q1", "answer": "typed reply", "seconds_left": 4.0, + "flagged_paste": False, "paste_delta": 0}, + {"question_id": "q2", "answer": "x" * 300, "seconds_left": 68.0, + "flagged_paste": True, "paste_delta": 300}, + ], + "followup": { + "id": "f1", + "question": "You said X — what happens if Y throws?", + "targets_question_id": "q2", + "answer": None, + }, + "status": "awaiting_followup", + } diff --git a/backend/tests/test_gemini_client.py b/backend/tests/test_gemini_client.py new file mode 100644 index 0000000..5ce779f --- /dev/null +++ b/backend/tests/test_gemini_client.py @@ -0,0 +1,80 @@ +""" +Parsing of the model's response. + +These cover the degradation paths rather than the prompt text. `complexity` is a +required field on QuizGenerateResponse, so a single malformed reply that reached +the schema unparsed would be a ValidationError and a 500 for the candidate. +""" +import json + +import pytest + +from app.integrations.gemini_client import ( + UNKNOWN_COMPLEXITY, + _parse_quiz_payload, + _strip_code_fence, +) + + +def test_well_formed_object(): + questions, complexity = _parse_quiz_payload(json.dumps({ + "questions": [{"question": "q", "category": "logic"}], + "complexity": {"tier": "complex", "reasoning": "many parts"}, + })) + assert len(questions) == 1 + assert complexity == {"tier": "complex", "reasoning": "many parts"} + + +def test_bare_array_still_yields_questions(): + """The model ignoring the object wrapper must not lose the quiz.""" + questions, complexity = _parse_quiz_payload(json.dumps([{"question": "q"}])) + assert len(questions) == 1 + assert complexity["tier"] == "unknown" + + +@pytest.mark.parametrize("payload", [ + {"questions": [{"question": "q"}]}, # complexity absent + {"questions": [{"question": "q"}], "complexity": "hard"}, # not a dict + {"questions": [{"question": "q"}], "complexity": None}, +]) +def test_unusable_complexity_degrades_but_keeps_questions(payload): + questions, complexity = _parse_quiz_payload(json.dumps(payload)) + assert len(questions) == 1 + assert complexity == UNKNOWN_COMPLEXITY + + +@pytest.mark.parametrize("given,expected", [ + ("complex", "complex"), + ("Moderate", "moderate"), # case is normalised + (" TRIVIAL ", "trivial"), # whitespace is stripped + ("extreme", "unknown"), # invented tiers are not passed through + ("", "unknown"), +]) +def test_tier_normalisation(given, expected): + _, complexity = _parse_quiz_payload(json.dumps({ + "questions": [], "complexity": {"tier": given, "reasoning": "r"}, + })) + assert complexity["tier"] == expected + + +def test_missing_reasoning_gets_a_default(): + _, complexity = _parse_quiz_payload(json.dumps({ + "questions": [], "complexity": {"tier": "trivial"}, + })) + assert complexity["tier"] == "trivial" + assert complexity["reasoning"] + + +def test_missing_questions_key_yields_empty_list(): + questions, _ = _parse_quiz_payload(json.dumps({"complexity": {"tier": "trivial", "reasoning": "r"}})) + assert questions == [] + + +@pytest.mark.parametrize("raw,expected", [ + ('{"a": 1}', '{"a": 1}'), + ('```json\n{"a": 1}\n```', '{"a": 1}'), + ('```\n{"a": 1}\n```', '{"a": 1}'), + (' {"a": 1} ', '{"a": 1}'), +]) +def test_strip_code_fence(raw, expected): + assert _strip_code_fence(raw) == expected diff --git a/backend/tests/test_github_client.py b/backend/tests/test_github_client.py new file mode 100644 index 0000000..e730f28 --- /dev/null +++ b/backend/tests/test_github_client.py @@ -0,0 +1,143 @@ +""" +Repo file selection. + +Both cases here are regressions. The original code sorted ascending and took the +smallest files, which selected empty __init__.py files and produced questions +asking why blank files were blank — while still returning HTTP 200. The SKIP_DIRS +check used a path prefix, so nested vendored directories were never excluded; +harmless while the smallest files were preferred, load-bearing once the largest are. +""" +import base64 + +import pytest + +from app.integrations import github_client + + +class FakeResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code != 200: + raise AssertionError(f"unexpected status {self.status_code}") + + +class FakeClient: + """Stands in for httpx.AsyncClient: serves one tree and file contents.""" + + def __init__(self, tree, missing_branches=(), **kwargs): + self.tree = tree + self.missing_branches = missing_branches + self.requested = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url): + self.requested.append(url) + if "/git/trees/" in url: + branch = url.split("/git/trees/")[1].split("?")[0] + if branch in self.missing_branches: + return FakeResponse({}, status_code=404) + return FakeResponse({"tree": self.tree}) + path = url.split("/contents/")[1] + return FakeResponse({ + "encoding": "base64", + "content": base64.b64encode(f"source of {path}".encode()).decode(), + }) + + +def blob(path, size): + return {"type": "blob", "path": path, "size": size} + + +@pytest.fixture +def patch_client(monkeypatch): + def apply(tree, missing_branches=()): + holder = {} + + def factory(**kwargs): + holder["client"] = FakeClient(tree, missing_branches, **kwargs) + return holder["client"] + + monkeypatch.setattr(github_client.httpx, "AsyncClient", factory) + return holder + + return apply + + +async def test_prefers_largest_files_and_drops_stubs(patch_client): + patch_client([ + blob("app/__init__.py", 0), + blob("app/schemas/__init__.py", 0), + blob("py.typed", 12), + blob("app/service.py", 4000), + blob("app/client.py", 2000), + blob("app/tiny.py", 150), # under the 200-byte floor + ]) + files = await github_client.fetch_repo_files("https://github.com/o/r") + + paths = [f["path"] for f in files] + assert paths == ["app/service.py", "app/client.py"] + assert not any("__init__" in p for p in paths), "empty stubs must never be selected" + + +async def test_excludes_nested_vendored_directories(patch_client): + patch_client([ + blob("frontend/node_modules/react/index.js", 9000), + blob("backend/dist/bundle.js", 8000), + blob("src/build/out.js", 7000), + blob("backend/__pycache__/x.pyc", 6000), + blob("app/main.py", 1000), + ]) + files = await github_client.fetch_repo_files("https://github.com/o/r") + assert [f["path"] for f in files] == ["app/main.py"] + + +async def test_does_not_exclude_lookalike_paths(patch_client): + """`dist/` must not match `mydist/`, and a file named build.py is fine.""" + patch_client([ + blob("mydist/helper.py", 3000), + blob("rebuild/tool.py", 2500), + blob("app/build.py", 2000), + ]) + files = await github_client.fetch_repo_files("https://github.com/o/r") + assert sorted(f["path"] for f in files) == ["app/build.py", "mydist/helper.py", "rebuild/tool.py"] + + +async def test_skips_binaries_and_docs(patch_client): + patch_client([ + blob("README.md", 5000), + blob("logo.png", 4000), + blob("uv.lock", 9000), + blob("app/main.py", 1000), + ]) + files = await github_client.fetch_repo_files("https://github.com/o/r") + assert [f["path"] for f in files] == ["app/main.py"] + + +async def test_falls_back_to_master_branch(patch_client): + holder = patch_client([blob("app/main.py", 1000)], missing_branches=("main",)) + files = await github_client.fetch_repo_files("https://github.com/o/r") + assert [f["path"] for f in files] == ["app/main.py"] + assert any("trees/master" in u for u in holder["client"].requested) + + +async def test_returns_empty_when_nothing_qualifies(patch_client): + """Drives the 400 the endpoint returns for a repo with no usable source.""" + patch_client([blob("README.md", 5000), blob("app/__init__.py", 0)]) + assert await github_client.fetch_repo_files("https://github.com/o/r") == [] + + +async def test_respects_max_files(patch_client): + patch_client([blob(f"app/mod{i}.py", 1000 + i) for i in range(30)]) + files = await github_client.fetch_repo_files("https://github.com/o/r", max_files=4) + assert len(files) == 4 diff --git a/backend/tests/test_quiz.py b/backend/tests/test_quiz.py deleted file mode 100644 index 5b7c619..0000000 --- a/backend/tests/test_quiz.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Placeholder test — proves the app boots and the route is wired. -Extend with mocked github_client/gemini_client calls before relying -on this for real coverage. -""" -from fastapi.testclient import TestClient - -from app.main import app - -client = TestClient(app) - - -def test_root_ok(): - resp = client.get("/") - assert resp.status_code == 200 - assert resp.json()["status"] == "ok" diff --git a/backend/tests/test_quiz_api.py b/backend/tests/test_quiz_api.py new file mode 100644 index 0000000..2777bbb --- /dev/null +++ b/backend/tests/test_quiz_api.py @@ -0,0 +1,162 @@ +""" +HTTP contract for the quiz endpoints. + +The flow is generate -> submit -> followup. /submit deliberately returns a +follow-up question rather than a score; anything relying on the old +score-at-submit behaviour should fail loudly here. +""" +from unittest.mock import AsyncMock + +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.services import quiz_service + +client = TestClient(app) + + +def test_root_ok(): + resp = client.get("/") + assert resp.status_code == 200 + assert resp.json()["status"] == "ok" + + +# --- generate -------------------------------------------------------------- + +def test_generate_returns_questions_complexity_and_time_limit(monkeypatch, questions, complexity): + monkeypatch.setattr(quiz_service, "create_quiz", AsyncMock(return_value={ + "quiz_id": "quiz-1", "repo_url": "https://github.com/o/r", + "questions": questions, "complexity": complexity, "time_limit_seconds": 75, + })) + resp = client.post("/api/v1/quiz/generate", json={"repo_url": "https://github.com/o/r"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["complexity"] == complexity + assert body["time_limit_seconds"] == 75 + assert [q["category"] for q in body["questions"]] == ["problem", "logic", "stack"] + + +def test_generate_preserves_category_through_the_response_model(monkeypatch, questions, complexity): + """response_model strips undeclared fields — category must be declared.""" + monkeypatch.setattr(quiz_service, "create_quiz", AsyncMock(return_value={ + "quiz_id": "quiz-1", "repo_url": "r", "questions": questions, + "complexity": complexity, "time_limit_seconds": 75, + })) + body = client.post("/api/v1/quiz/generate", json={"repo_url": "r"}).json() + assert all(q["category"] for q in body["questions"]) + + +def test_generate_maps_no_source_files_to_400(monkeypatch): + monkeypatch.setattr(quiz_service, "create_quiz", AsyncMock(side_effect=ValueError("no_source_files"))) + resp = client.post("/api/v1/quiz/generate", json={"repo_url": "https://github.com/o/r"}) + assert resp.status_code == 400 + + +def test_generate_requires_repo_url(): + assert client.post("/api/v1/quiz/generate", json={}).status_code == 422 + + +# --- submit ---------------------------------------------------------------- + +def test_submit_returns_a_followup_and_no_score(monkeypatch): + monkeypatch.setattr(quiz_service, "start_followup", AsyncMock(return_value={ + "quiz_id": "quiz-1", + "followup": {"id": "f1", "question": "You said X?", "targets_question_id": "q2"}, + "time_limit_seconds": 75, + })) + resp = client.post("/api/v1/quiz/submit", json={ + "quiz_id": "quiz-1", + "answers": [{"question_id": "q1", "answer": "a"}], + }) + + assert resp.status_code == 200 + body = resp.json() + assert body["followup"]["targets_question_id"] == "q2" + assert "score" not in body + + +def test_submit_forwards_paste_flags_and_timing(monkeypatch): + """The anti-gaming signals are useless if they do not survive the wire.""" + spy = AsyncMock(return_value={ + "quiz_id": "quiz-1", + "followup": {"id": "f1", "question": "q?", "targets_question_id": "q1"}, + "time_limit_seconds": 75, + }) + monkeypatch.setattr(quiz_service, "start_followup", spy) + client.post("/api/v1/quiz/submit", json={"quiz_id": "quiz-1", "answers": [ + {"question_id": "q1", "answer": "a", "seconds_left": 61.5, + "flagged_paste": True, "paste_delta": 220}, + ]}) + + sent = spy.call_args.args[1][0] + assert sent["flagged_paste"] is True + assert sent["paste_delta"] == 220 + assert sent["seconds_left"] == 61.5 + + +def test_submit_defaults_paste_flags_when_client_omits_them(monkeypatch): + """An older client must not 500 the request.""" + spy = AsyncMock(return_value={ + "quiz_id": "quiz-1", + "followup": {"id": "f1", "question": "q?", "targets_question_id": "q1"}, + "time_limit_seconds": 75, + }) + monkeypatch.setattr(quiz_service, "start_followup", spy) + resp = client.post("/api/v1/quiz/submit", json={ + "quiz_id": "quiz-1", "answers": [{"question_id": "q1", "answer": "a"}], + }) + + assert resp.status_code == 200 + sent = spy.call_args.args[1][0] + assert sent["flagged_paste"] is False + assert sent["paste_delta"] == 0 + assert sent["seconds_left"] is None + + +def test_submit_unknown_quiz_is_404(monkeypatch): + monkeypatch.setattr(quiz_service, "start_followup", AsyncMock(side_effect=LookupError())) + resp = client.post("/api/v1/quiz/submit", json={ + "quiz_id": "nope", "answers": [{"question_id": "q1", "answer": "a"}], + }) + assert resp.status_code == 404 + + +def test_submit_with_no_answers_is_400(monkeypatch): + monkeypatch.setattr(quiz_service, "start_followup", AsyncMock(side_effect=ValueError())) + resp = client.post("/api/v1/quiz/submit", json={"quiz_id": "quiz-1", "answers": []}) + assert resp.status_code == 400 + + +# --- followup -------------------------------------------------------------- + +def test_followup_returns_the_final_score_and_breakdown(monkeypatch): + monkeypatch.setattr(quiz_service, "grade_quiz", AsyncMock(return_value={ + "overall_score": 82.0, + "breakdown": [{"question": "q1", "score": 8, "note": "solid"}], + })) + resp = client.post("/api/v1/quiz/followup", json={ + "quiz_id": "quiz-1", "answer": "my defence", "seconds_left": 12.0, + }) + + assert resp.status_code == 200 + body = resp.json() + assert body["score"] == 82.0 + assert body["feedback"] == body["breakdown"]["details"] + assert body["feedback"][0]["note"] == "solid" + + +def test_followup_accepts_a_blank_answer(monkeypatch): + """A timed-out follow-up submits blank — that is a result, not an error.""" + monkeypatch.setattr(quiz_service, "grade_quiz", + AsyncMock(return_value={"overall_score": 0.0, "breakdown": []})) + resp = client.post("/api/v1/quiz/followup", json={"quiz_id": "quiz-1", "answer": ""}) + assert resp.status_code == 200 + assert resp.json()["score"] == 0.0 + + +def test_followup_unknown_quiz_is_404(monkeypatch): + monkeypatch.setattr(quiz_service, "grade_quiz", AsyncMock(side_effect=LookupError())) + resp = client.post("/api/v1/quiz/followup", json={"quiz_id": "nope", "answer": "a"}) + assert resp.status_code == 404 diff --git a/backend/tests/test_quiz_service.py b/backend/tests/test_quiz_service.py new file mode 100644 index 0000000..fab3eef --- /dev/null +++ b/backend/tests/test_quiz_service.py @@ -0,0 +1,161 @@ +""" +Service orchestration and suspect selection. + +Suspect selection decides who gets interrogated, so it is the part of the +anti-gaming work that actually has teeth — a wrong pick means the pasted answer +goes unchallenged. +""" +from unittest.mock import AsyncMock + +import pytest + +from app.services import quiz_service +from app.services.quiz_service import pick_suspect_answer + + +def answer(qid, text="typed", seconds_left=None, flagged=False, delta=0): + return {"question_id": qid, "answer": text, "seconds_left": seconds_left, + "flagged_paste": flagged, "paste_delta": delta} + + +# --- suspect selection ----------------------------------------------------- + +def test_recorded_paste_beats_the_timing_heuristic(): + """A flag is evidence; typing pace is only an inference.""" + picked = pick_suspect_answer([ + answer("fast", "x" * 900, seconds_left=70), # very suspicious timing + answer("pasted", "y" * 300, seconds_left=5, flagged=True, delta=300), + ]) + assert picked["question_id"] == "pasted" + + +def test_largest_injection_wins_among_flagged(): + picked = pick_suspect_answer([ + answer("small", "a" * 100, flagged=True, delta=60), + answer("big", "b" * 500, flagged=True, delta=480), + answer("mid", "c" * 200, flagged=True, delta=150), + ]) + assert picked["question_id"] == "big" + + +def test_falls_back_to_fast_and_long_when_nothing_flagged(): + picked = pick_suspect_answer([ + answer("slow", "a" * 80, seconds_left=3), + answer("fast", "b" * 800, seconds_left=66), + ]) + assert picked["question_id"] == "fast" + + +def test_fast_one_liner_does_not_outrank_a_fast_essay(): + """Rate alone would pick the one-liner; weighting by length must not.""" + picked = pick_suspect_answer([ + answer("oneliner", "yes", seconds_left=74), + answer("essay", "y" * 600, seconds_left=50), + ]) + assert picked["question_id"] == "essay" + + +def test_blank_answers_are_never_selected_even_when_flagged(): + picked = pick_suspect_answer([ + answer("blank", "", flagged=True, delta=900), + answer("real", "typed out", seconds_left=4), + ]) + assert picked["question_id"] == "real" + + +def test_no_timing_data_still_selects_something(): + """The follow-up round must happen even with no telemetry at all.""" + picked = pick_suspect_answer([answer("a", "first"), answer("b", "second")]) + assert picked["question_id"] == "a" + + +def test_all_blank_returns_first_so_the_round_still_runs(): + picked = pick_suspect_answer([answer("a", ""), answer("b", " ")]) + assert picked["question_id"] == "a" + + +def test_no_answers_returns_none(): + assert pick_suspect_answer([]) is None + + +# --- create_quiz ----------------------------------------------------------- + +async def test_create_quiz_persists_complexity_and_assigns_ids(monkeypatch, complexity): + monkeypatch.setattr(quiz_service.github_client, "fetch_repo_files", + AsyncMock(return_value=[{"path": "a.py", "content": "x"}])) + monkeypatch.setattr(quiz_service.gemini_client, "generate_quiz_questions", + AsyncMock(return_value=([{"question": "q", "category": "logic"}], complexity))) + saved = AsyncMock() + monkeypatch.setattr(quiz_service.quiz_repository, "save_attempt", saved) + + result = await quiz_service.create_quiz("https://github.com/o/r", "u1") + + assert result["complexity"] == complexity + assert result["time_limit_seconds"] == 75 + assert result["questions"][0]["id"], "questions must be given ids" + doc = saved.call_args.args[0] + assert doc["complexity"] == complexity + assert doc["status"] == "generated" + + +async def test_create_quiz_rejects_a_repo_with_no_source(monkeypatch): + monkeypatch.setattr(quiz_service.github_client, "fetch_repo_files", AsyncMock(return_value=[])) + with pytest.raises(ValueError): + await quiz_service.create_quiz("https://github.com/o/r", None) + + +# --- follow-up round ------------------------------------------------------- + +async def test_start_followup_targets_the_flagged_answer(monkeypatch, attempt): + monkeypatch.setattr(quiz_service.quiz_repository, "get_attempt", AsyncMock(return_value=attempt)) + gen = AsyncMock(return_value="You said X — what if Y throws?") + monkeypatch.setattr(quiz_service.gemini_client, "generate_followup_question", gen) + monkeypatch.setattr(quiz_service.quiz_repository, "update_followup", AsyncMock()) + + out = await quiz_service.start_followup("quiz-1", attempt["answers"]) + + assert out["followup"]["targets_question_id"] == "q2" + # the pasted answer's text is what the model was asked to push on + assert gen.call_args.args[1] == attempt["answers"][1]["answer"] + + +async def test_start_followup_does_not_grade(monkeypatch, attempt): + """Grading here would let a candidate bank a score and skip the round.""" + monkeypatch.setattr(quiz_service.quiz_repository, "get_attempt", AsyncMock(return_value=attempt)) + monkeypatch.setattr(quiz_service.gemini_client, "generate_followup_question", AsyncMock(return_value="q?")) + monkeypatch.setattr(quiz_service.quiz_repository, "update_followup", AsyncMock()) + graded = AsyncMock() + monkeypatch.setattr(quiz_service.gemini_client, "grade_answers", graded) + + out = await quiz_service.start_followup("quiz-1", attempt["answers"]) + + graded.assert_not_called() + assert "score" not in out + + +async def test_start_followup_unknown_quiz(monkeypatch): + monkeypatch.setattr(quiz_service.quiz_repository, "get_attempt", AsyncMock(return_value=None)) + with pytest.raises(LookupError): + await quiz_service.start_followup("nope", [answer("q1")]) + + +# --- final grading --------------------------------------------------------- + +async def test_grade_quiz_passes_the_followup_defence_to_the_grader(monkeypatch, attempt): + monkeypatch.setattr(quiz_service.quiz_repository, "get_attempt", AsyncMock(return_value=attempt)) + grade = AsyncMock(return_value={"overall_score": 88, "breakdown": []}) + monkeypatch.setattr(quiz_service.gemini_client, "grade_answers", grade) + monkeypatch.setattr(quiz_service.quiz_repository, "update_result", AsyncMock()) + + result = await quiz_service.grade_quiz("quiz-1", "my defence", seconds_left=9.0) + + assert result["overall_score"] == 88 + sent = grade.call_args.kwargs["followup"] + assert sent["answer"] == "my defence" + assert sent["targets_question_id"] == "q2" + + +async def test_grade_quiz_unknown_quiz(monkeypatch): + monkeypatch.setattr(quiz_service.quiz_repository, "get_attempt", AsyncMock(return_value=None)) + with pytest.raises(LookupError): + await quiz_service.grade_quiz("nope", "answer") diff --git a/frontend/package.json b/frontend/package.json index e2cac8a..84e3976 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,7 +6,8 @@ "scripts": { "dev": "vite", "build": "vite build", - "preview": "vite preview" + "preview": "vite preview", + "test": "node --test" }, "dependencies": { "react": "^18.3.1", diff --git a/frontend/src/features/quiz/pasteDetect.test.js b/frontend/src/features/quiz/pasteDetect.test.js new file mode 100644 index 0000000..a9bca75 --- /dev/null +++ b/frontend/src/features/quiz/pasteDetect.test.js @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createInputTracker, PASTE_MIN_DELTA, PASTE_MAX_GAP_MS } from "./pasteDetect.js"; + +/** Replay a stream of [chunk, gapMs] events; "\b" is a backspace. */ +function drive(events) { + const tracker = createInputTracker(); + let text = ""; + let now = 1000; + for (const [chunk, gap] of events) { + now += gap; + const next = chunk === "\b" ? text.slice(0, -1) : text + chunk; + tracker.record(text, next, now); + text = next; + } + return { ...tracker.snapshot(), text }; +} + +const typed = (s, gap = 120) => [...s].map((ch) => [ch, gap]); +const PARAGRAPH = "x".repeat(480); + +test("a paste into an empty box is flagged", () => { + const r = drive([[PARAGRAPH, 0]]); + assert.equal(r.flagged_paste, true); + assert.equal(r.paste_delta, 480); +}); + +test("a paste immediately after a keystroke is flagged", () => { + const r = drive([["hello", 100], ["a", 90], [PARAGRAPH, 30]]); + assert.equal(r.flagged_paste, true); +}); + +test("ordinary typing is not flagged", () => { + const r = drive(typed("a".repeat(240))); + assert.equal(r.flagged_paste, false); + assert.equal(r.paste_delta, 0); + assert.equal(r.text.length, 240); +}); + +test("typing with the candidate's own backspaces is not flagged", () => { + const events = []; + for (let i = 0; i < 200; i++) { + events.push(["a", 110]); + if (i % 7 === 0) events.push(["\b", 180]); + } + const r = drive(events); + assert.equal(r.flagged_paste, false); +}); + +test("a very fast typist is not flagged", () => { + // Still one character per event, which is what distinguishes typing from paste. + assert.equal(drive(typed("a".repeat(300), 40)).flagged_paste, false); +}); + +test("an IME commit after a human pause is not flagged", () => { + const r = drive(Array.from({ length: 30 }, () => ["abcdefgh", 400])); + assert.equal(r.flagged_paste, false); +}); + +test("deleting a large selection is not flagged", () => { + // A negative delta must never look like an injection. + const tracker = createInputTracker(); + tracker.record("y".repeat(500), "", 1000); + assert.equal(tracker.snapshot().flagged_paste, false); +}); + +test("the largest injection is the one reported", () => { + const r = drive([["a".repeat(100), 0], ["b", 500], ["c".repeat(300), 20]]); + assert.equal(r.paste_delta, 300); +}); + +test("a delta just under the threshold is not flagged", () => { + const r = drive([["z".repeat(PASTE_MIN_DELTA), 0]]); + assert.equal(r.flagged_paste, false); +}); + +test("known gap: pasting after a long pause clears the timing guard", () => { + // Documents current behaviour rather than endorsing it. Closing this means + // judging by implied typing rate instead of a fixed gap, which risks flagging + // dictation software. See the README's anti-gaming section. + const r = drive([["I think ", 150], ["a", 140], [PARAGRAPH, PASTE_MAX_GAP_MS * 40]]); + assert.equal(r.flagged_paste, false); +});