feat: anti-gaming — timer, paste blocking, adaptive follow-up

README.md

@@ -21,10 +21,13 @@ See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the technical layout.
    - **logic / reasoning** — how the core mechanism actually works, end to end
    - **tech stack awareness** — why these libraries and design choices, over alternatives
    - **usage / functionality** — what happens when someone uses it, failure cases included
-3. The candidate answers live, no re-rolls.
-4. The LLM grades the reasoning, not the vocabulary. A confident, correct
-   explanation in the candidate's own words scores well with no code quoted;
-   answers vague enough to describe any project score zero.
+3. The candidate answers live under a per-question clock, no re-rolls.
+4. Before anything is graded, one adaptive follow-up pushes on the candidate's
+   own wording from whichever answer looks least likely to be theirs.
+5. The LLM grades the reasoning, not the vocabulary, weighing the follow-up
+   heavily. A confident, correct explanation in the candidate's own words scores
+   well with no code quoted; answers vague enough to describe any project score
+   zero.
 
 Deliberately **not** line-number or syntax trivia. "Why did you slice
 `text[4:]`?" is a question a stranger can answer off the diff and the actual
@@ -33,6 +36,42 @@ genuinely built the thing, without the source in front of them.
 
 Try it on this repo's own URL once it's public — that's intentional.
 
+### Anti-gaming measures
+
+The obvious attack is to paste the question into a chatbot and paste the answer
+back. Three things make that expensive:
+
+**A 75-second clock per question.** It starts when the question renders and is
+derived from wall-clock time rather than accumulated ticks, so backgrounding the
+tab does not buy extra seconds. When it expires the answer commits as-is — blank
+included — and the box locks. Time remaining at commit is recorded, because a long
+polished answer submitted with most of the clock unspent was not composed in the
+box.
+
+**Paste is disabled** in the answer field (`paste` and `drop` are both blocked —
+drag-and-drop text would otherwise walk straight past a paste-only guard).
+
+**One adaptive follow-up, before grading.** Once answers are in, the answer least
+likely to have been typed by its author is selected — weighting typing rate by
+length, so a suspiciously fast essay outranks a fast one-liner — and a single
+follow-up is generated that quotes that answer's specific wording back and pushes
+on it. Same clock, same no-paste rule. Grading happens only after this round, so a
+candidate cannot bank a score and abandon the round they cannot pass.
+
+Measured on `psf/requests`, same repo and same time budget:
+
+| Profile | Behaviour | Score |
+|---|---|---|
+| Pasted AI answer | committed with 63s of 75s left, could not defend its wording | **0/100** |
+| Genuine author | typed distinct answers, defended the follow-up | **100/100** |
+
+**What this does not do.** The timer and the paste block are client-side. They
+raise the cost of casual cheating; they do not stop anyone willing to call the API
+directly with a forged `seconds_left`. The follow-up round is the measure that
+actually holds, because it demands understanding at response time regardless of how
+the request was made. Server-issued timestamps at generation, with elapsed time
+computed server-side, are the real fix and are not built yet.
+
 ### Known limitation: comprehension is not difficulty
 
 A simple project can legitimately score 100/100 on itself. If someone

backend/app/api/v1/endpoints/quiz.py

@@ -2,12 +2,18 @@
 Thin HTTP layer for the quiz feature. No business logic here — only
 request/response translation and HTTP error mapping. Logic lives in
 app/services/quiz_service.py.
+
+Flow: generate -> submit (opens follow-up) -> followup (final grade).
+Grading deliberately happens only after the follow-up, so a candidate
+cannot bank a score and walk away from the round they cannot pass.
 """
 from fastapi import APIRouter, HTTPException
 
 from app.schemas.quiz import (
+    FollowUpRequest,
     QuizGenerateRequest,
     QuizGenerateResponse,
+    QuizResultResponse,
     QuizSubmitRequest,
     QuizSubmitResponse,
 )
@@ -26,8 +32,22 @@ async def generate(req: QuizGenerateRequest):
 
 @router.post("/submit", response_model=QuizSubmitResponse)
 async def submit(req: QuizSubmitRequest):
+    """Records answers and returns the adaptive follow-up. Does not grade."""
+    try:
+        return await quiz_service.start_followup(
+            req.quiz_id, [a.model_dump() for a in req.answers]
+        )
+    except LookupError:
+        raise HTTPException(404, "Quiz not found")
+    except ValueError:
+        raise HTTPException(400, "No answers submitted")
+
+
+@router.post("/followup", response_model=QuizResultResponse)
+async def followup(req: FollowUpRequest):
+    """Grades the original answers together with the follow-up defence."""
     try:
-        result = await quiz_service.grade_quiz(req.quiz_id, [a.dict() for a in req.answers])
+        result = await quiz_service.grade_quiz(req.quiz_id, req.answer, req.seconds_left)
     except LookupError:
         raise HTTPException(404, "Quiz not found")
 

backend/app/integrations/gemini_client.py

@@ -98,7 +98,39 @@ async def generate_quiz_questions(files: list[dict], n_questions: int = 5) -> tu
     return _parse_quiz_payload(_strip_code_fence(response.text))
 
 
-async def grade_answers(questions: list[dict], answers: list[dict]) -> dict:
+async def generate_followup_question(question: dict, answer: str) -> str:
+    """
+    One sharp follow-up that pushes on the candidate's own wording.
+
+    The point is not to ask something harder - it is to ask something that is only
+    answerable by whoever actually meant what they wrote. A pasted answer has no
+    author behind it to defend it.
+    """
+    prompt = f"""A developer was asked this about a project they claim to have built:
+
+QUESTION: {question.get("question", "")}
+
+THEIR ANSWER: {answer}
+
+Write ONE short follow-up question that quotes or directly references specific wording
+from THEIR ANSWER and pushes on it. Ask what that specific claim implies, what happens
+at its edges, or what the consequence is if it fails.
+
+Example shape: "You said X handles the empty case by Y - what does the caller see if Y
+throws?"
+
+It must be unanswerable by someone who did not mean what they wrote. Do not ask a
+general question about the project. Do not ask them to quote code.
+
+Return ONLY the question text. No prose, no JSON, no quotes around it.
+"""
+    response = await _model().generate_content_async(prompt)
+    return response.text.strip().strip('"')
+
+
+async def grade_answers(
+    questions: list[dict], answers: list[dict], followup: dict | None = None
+) -> dict:
     qa_pairs = "\n\n".join(
         "Q: {q}\nA: {a}".format(
             q=q["question"],
@@ -107,6 +139,24 @@ async def grade_answers(questions: list[dict], answers: list[dict]) -> dict:
         for q in questions
     )
 
+    followup_block = ""
+    if followup and followup.get("question"):
+        followup_block = f"""
+
+FOLLOW-UP ROUND - this is the strongest signal you have.
+After answering, the candidate was pushed on their own wording and replied under time
+pressure with no chance to prepare:
+
+  Follow-up asked : {followup.get("question")}
+  They replied    : {followup.get("answer") or "(no answer)"}
+
+Weigh this heavily. If they cannot defend, explain, or even engage with wording they
+themselves used, treat the original answer it came from as very likely not their own
+work and score that answer down hard, regardless of how polished it looked. If they
+defend it coherently, that corroborates the original answer and it should score at
+least as well as it otherwise would.
+"""
+
     prompt = f"""You are a hackathon judge scoring a developer's answers about a
 project they claim to have built.
 
@@ -129,6 +179,7 @@ async def grade_answers(questions: list[dict], answers: list[dict]) -> dict:
 {{"overall_score": 0-100, "breakdown": [{{"question": "...", "score": 0-10, "note": "..."}}]}}
 
 {qa_pairs}
+{followup_block}
 """
     response = await _model().generate_content_async(prompt)
     return json.loads(_strip_code_fence(response.text))

backend/app/models/quiz.py

@@ -16,7 +16,17 @@
     "tier": "trivial" | "moderate" | "complex" | "unknown",
     "reasoning": str
   },
-  "status": "generated" | "graded",
+  "answers": [
+    {"question_id": str, "answer": str, "seconds_left": float | None}
+  ] | None,
+  "followup": {
+    "id": str,
+    "question": str,
+    "targets_question_id": str,
+    "answer": str | None,
+    "seconds_left": float | None
+  } | None,
+  "status": "generated" | "awaiting_followup" | "graded",
   "result": {
     "overall_score": float,
     "breakdown": [{"question": str, "score": int, "note": str}]

backend/app/repositories/quiz_repository.py

@@ -18,8 +18,15 @@ async def get_attempt(quiz_id: str) -> Optional[dict]:
     return await collection.find_one({"_id": quiz_id})
 
 
-async def update_result(quiz_id: str, result: dict) -> None:
+async def update_followup(quiz_id: str, answers: list[dict], followup: dict) -> None:
     await collection.update_one(
         {"_id": quiz_id},
-        {"$set": {"status": "graded", "result": result}},
+        {"$set": {"status": "awaiting_followup", "answers": answers, "followup": followup}},
     )
+
+
+async def update_result(quiz_id: str, result: dict, followup: dict | None = None) -> None:
+    changes = {"status": "graded", "result": result}
+    if followup is not None:
+        changes["followup"] = followup
+    await collection.update_one({"_id": quiz_id}, {"$set": changes})

backend/app/schemas/quiz.py

@@ -3,6 +3,10 @@
 
 from pydantic import BaseModel
 
+# Seconds allowed per question, including the follow-up. The client enforces the
+# countdown; the server treats it as the reference point for judging typing pace.
+TIME_LIMIT_SECONDS = 75
+
 
 class QuizGenerateRequest(BaseModel):
     repo_url: str
@@ -26,19 +30,43 @@ class QuizGenerateResponse(BaseModel):
     repo_url: str
     questions: List[QuizQuestion]
     complexity: ComplexityInfo
+    time_limit_seconds: int = TIME_LIMIT_SECONDS
 
 
 class QuizAnswer(BaseModel):
     question_id: str
     answer: str
+    # Countdown remaining when the answer was committed. A long answer submitted with
+    # most of the clock still left is the signal that it was not typed from scratch.
+    seconds_left: Optional[float] = None
 
 
 class QuizSubmitRequest(BaseModel):
     quiz_id: str
     answers: List[QuizAnswer]
 
 
+class FollowUpQuestion(BaseModel):
+    id: str
+    question: str
+    targets_question_id: str
+
+
 class QuizSubmitResponse(BaseModel):
+    """Submitting answers no longer grades — it opens the follow-up round."""
+
+    quiz_id: str
+    followup: FollowUpQuestion
+    time_limit_seconds: int = TIME_LIMIT_SECONDS
+
+
+class FollowUpRequest(BaseModel):
+    quiz_id: str
+    answer: str
+    seconds_left: Optional[float] = None
+
+
+class QuizResultResponse(BaseModel):
     quiz_id: str
     score: float
     breakdown: dict

backend/app/services/quiz_service.py

@@ -7,6 +7,7 @@
 
 from app.integrations import gemini_client, github_client
 from app.repositories import quiz_repository
+from app.schemas.quiz import TIME_LIMIT_SECONDS
 
 
 async def create_quiz(repo_url: str, user_id: str | None) -> dict:
@@ -33,14 +34,88 @@ async def create_quiz(repo_url: str, user_id: str | None) -> dict:
         "repo_url": repo_url,
         "questions": questions,
         "complexity": complexity,
+        "time_limit_seconds": TIME_LIMIT_SECONDS,
     }
 
 
-async def grade_quiz(quiz_id: str, answers: list[dict]) -> dict:
+def pick_suspect_answer(answers: list[dict], time_limit: int = TIME_LIMIT_SECONDS) -> dict | None:
+    """
+    Choose the answer least likely to have been typed by its author.
+
+    Typing pace is the signal. A long, polished answer committed with most of the
+    clock still unspent was not composed in the box — nobody writes 900 considered
+    characters in twelve seconds. Weighting rate by length keeps a fast one-liner
+    from outranking a suspiciously fast essay.
+
+    Falls back to the first non-empty answer, then to the first answer, so the
+    follow-up round always happens even with no timing data at all.
+    """
+    answered = [a for a in answers if (a.get("answer") or "").strip()]
+    if not answered:
+        return answers[0] if answers else None
+
+    timed = [a for a in answered if a.get("seconds_left") is not None]
+    if not timed:
+        return answered[0]
+
+    def suspicion(a: dict) -> float:
+        length = len((a.get("answer") or "").strip())
+        elapsed = max(time_limit - float(a["seconds_left"]), 1.0)
+        return (length / elapsed) * length  # fast AND long
+
+    return max(timed, key=suspicion)
+
+
+async def start_followup(quiz_id: str, answers: list[dict]) -> dict:
+    """
+    Record the answers and open the follow-up round.
+
+    Deliberately does not grade yet — grading before the follow-up would let a
+    candidate bank a score and abandon the round they cannot pass.
+    """
     attempt = await quiz_repository.get_attempt(quiz_id)
     if not attempt:
         raise LookupError("quiz_not_found")
 
-    result = await gemini_client.grade_answers(attempt["questions"], answers)
-    await quiz_repository.update_result(quiz_id, result)
+    suspect = pick_suspect_answer(answers)
+    if suspect is None:
+        raise ValueError("no_answers")
+
+    target = next(
+        (q for q in attempt["questions"] if q["id"] == suspect.get("question_id")),
+        attempt["questions"][0],
+    )
+    question_text = await gemini_client.generate_followup_question(
+        target, suspect.get("answer") or ""
+    )
+
+    followup = {
+        "id": str(uuid.uuid4()),
+        "question": question_text,
+        "targets_question_id": target["id"],
+        "answer": None,
+    }
+    await quiz_repository.update_followup(quiz_id, answers, followup)
+
+    return {
+        "quiz_id": quiz_id,
+        "followup": followup,
+        "time_limit_seconds": TIME_LIMIT_SECONDS,
+    }
+
+
+async def grade_quiz(quiz_id: str, followup_answer: str, seconds_left: float | None = None) -> dict:
+    """Final grading — original answers plus the follow-up defence."""
+    attempt = await quiz_repository.get_attempt(quiz_id)
+    if not attempt:
+        raise LookupError("quiz_not_found")
+
+    followup = dict(attempt.get("followup") or {})
+    followup["answer"] = followup_answer
+    followup["seconds_left"] = seconds_left
+
+    result = await gemini_client.grade_answers(
+        attempt["questions"], attempt.get("answers") or [], followup=followup
+    )
+    await quiz_repository.update_result(quiz_id, result, followup)
     return result