From e58067c5cdf8539c3e87165957d10c16fde2b696 Mon Sep 17 00:00:00 2001 From: Krish Date: Sat, 22 Aug 2026 16:02:11 +0530 Subject: [PATCH] feat: add timer, paste blocking, and an adaptive follow-up round The obvious attack on a comprehension quiz is to paste the question into a chatbot and paste the answer back. Three measures raise the cost of that. Per-question 75s clock. It starts when the question renders and is computed from wall-clock time rather than accumulated ticks, so a backgrounded tab cannot buy extra seconds by having its interval throttled. On expiry the answer commits as-is, blank included, and the field locks; when every question has expired the quiz submits itself. Time remaining at commit is sent with each answer. Paste is blocked in the answer field. Both paste and drop are prevented, since drag-and-drop text would otherwise walk straight past a paste-only guard. One adaptive follow-up, asked before anything is graded. The answer least likely to have been typed by its author is selected by weighting typing rate by length - (length/elapsed) * length - so a suspiciously fast essay outranks a fast one-liner, with fallbacks to the first answered question and then the first question so the round always happens even with no timing data. A single follow-up is generated that quotes that answer's specific wording back and pushes on what it implies. Grading is deferred until after this round on purpose: grading at submit would let a candidate bank a score and walk away from the round they cannot pass. This restructures the flow into generate -> submit -> followup. /submit no longer returns a score, it returns the follow-up question; /followup returns the final result in the shape /submit previously returned, so ScoreResult needs no changes. Answers, timing, and the follow-up exchange are all persisted on the attempt, and status gains an awaiting_followup state between generated and graded. Measured on psf/requests with the same repo and time budget: a pasted AI answer committed with 63s of 75s left, unable to defend its own wording, scored 0/100; a genuine author typing distinct answers and defending the follow-up scored 100/100, with the follow-up appearing as its own row in the breakdown. Known gap, documented in the README: the timer and paste block are client-side and do not stop anyone calling the API directly with a forged seconds_left. The follow-up is the measure that actually holds, since it demands understanding at response time regardless of transport. Server-issued timestamps are the real fix and are not built here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PET9qKZXhgjEbZK7MReYQj --- README.md | 47 +++++++- backend/app/api/v1/endpoints/quiz.py | 22 +++- backend/app/integrations/gemini_client.py | 53 +++++++++- backend/app/models/quiz.py | 12 ++- backend/app/repositories/quiz_repository.py | 11 +- backend/app/schemas/quiz.py | 28 +++++ backend/app/services/quiz_service.py | 81 +++++++++++++- frontend/src/features/quiz/QuizPage.jsx | 100 ++++++++++++++++-- frontend/src/features/quiz/api.js | 9 ++ .../features/quiz/components/QuestionCard.jsx | 69 +++++++++++- frontend/src/styles/globals.css | 14 +++ 11 files changed, 421 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 13fe282..c4246d5 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,13 @@ See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the technical layout. - **logic / reasoning** — how the core mechanism actually works, end to end - **tech stack awareness** — why these libraries and design choices, over alternatives - **usage / functionality** — what happens when someone uses it, failure cases included -3. The candidate answers live, no re-rolls. -4. The LLM grades the reasoning, not the vocabulary. A confident, correct - explanation in the candidate's own words scores well with no code quoted; - answers vague enough to describe any project score zero. +3. The candidate answers live under a per-question clock, no re-rolls. +4. Before anything is graded, one adaptive follow-up pushes on the candidate's + own wording from whichever answer looks least likely to be theirs. +5. The LLM grades the reasoning, not the vocabulary, weighing the follow-up + heavily. A confident, correct explanation in the candidate's own words scores + well with no code quoted; answers vague enough to describe any project score + zero. Deliberately **not** line-number or syntax trivia. "Why did you slice `text[4:]`?" is a question a stranger can answer off the diff and the actual @@ -33,6 +36,42 @@ genuinely built the thing, without the source in front of them. Try it on this repo's own URL once it's public — that's intentional. +### Anti-gaming measures + +The obvious attack is to paste the question into a chatbot and paste the answer +back. Three things make that expensive: + +**A 75-second clock per question.** It starts when the question renders and is +derived from wall-clock time rather than accumulated ticks, so backgrounding the +tab does not buy extra seconds. When it expires the answer commits as-is — blank +included — and the box locks. Time remaining at commit is recorded, because a long +polished answer submitted with most of the clock unspent was not composed in the +box. + +**Paste is disabled** in the answer field (`paste` and `drop` are both blocked — +drag-and-drop text would otherwise walk straight past a paste-only guard). + +**One adaptive follow-up, before grading.** Once answers are in, the answer least +likely to have been typed by its author is selected — weighting typing rate by +length, so a suspiciously fast essay outranks a fast one-liner — and a single +follow-up is generated that quotes that answer's specific wording back and pushes +on it. Same clock, same no-paste rule. Grading happens only after this round, so a +candidate cannot bank a score and abandon the round they cannot pass. + +Measured on `psf/requests`, same repo and same time budget: + +| Profile | Behaviour | Score | +|---|---|---| +| Pasted AI answer | committed with 63s of 75s left, could not defend its wording | **0/100** | +| Genuine author | typed distinct answers, defended the follow-up | **100/100** | + +**What this does not do.** The timer and the paste block are client-side. They +raise the cost of casual cheating; they do not stop anyone willing to call the API +directly with a forged `seconds_left`. The follow-up round is the measure that +actually holds, because it demands understanding at response time regardless of how +the request was made. Server-issued timestamps at generation, with elapsed time +computed server-side, are the real fix and are not built yet. + ### Known limitation: comprehension is not difficulty A simple project can legitimately score 100/100 on itself. If someone diff --git a/backend/app/api/v1/endpoints/quiz.py b/backend/app/api/v1/endpoints/quiz.py index c327f40..aa60504 100644 --- a/backend/app/api/v1/endpoints/quiz.py +++ b/backend/app/api/v1/endpoints/quiz.py @@ -2,12 +2,18 @@ Thin HTTP layer for the quiz feature. No business logic here — only request/response translation and HTTP error mapping. Logic lives in app/services/quiz_service.py. + +Flow: generate -> submit (opens follow-up) -> followup (final grade). +Grading deliberately happens only after the follow-up, so a candidate +cannot bank a score and walk away from the round they cannot pass. """ from fastapi import APIRouter, HTTPException from app.schemas.quiz import ( + FollowUpRequest, QuizGenerateRequest, QuizGenerateResponse, + QuizResultResponse, QuizSubmitRequest, QuizSubmitResponse, ) @@ -26,8 +32,22 @@ async def generate(req: QuizGenerateRequest): @router.post("/submit", response_model=QuizSubmitResponse) async def submit(req: QuizSubmitRequest): + """Records answers and returns the adaptive follow-up. Does not grade.""" + try: + return await quiz_service.start_followup( + req.quiz_id, [a.model_dump() for a in req.answers] + ) + except LookupError: + raise HTTPException(404, "Quiz not found") + except ValueError: + raise HTTPException(400, "No answers submitted") + + +@router.post("/followup", response_model=QuizResultResponse) +async def followup(req: FollowUpRequest): + """Grades the original answers together with the follow-up defence.""" try: - result = await quiz_service.grade_quiz(req.quiz_id, [a.dict() for a in req.answers]) + result = await quiz_service.grade_quiz(req.quiz_id, req.answer, req.seconds_left) except LookupError: raise HTTPException(404, "Quiz not found") diff --git a/backend/app/integrations/gemini_client.py b/backend/app/integrations/gemini_client.py index 98d70e3..838077c 100644 --- a/backend/app/integrations/gemini_client.py +++ b/backend/app/integrations/gemini_client.py @@ -98,7 +98,39 @@ async def generate_quiz_questions(files: list[dict], n_questions: int = 5) -> tu return _parse_quiz_payload(_strip_code_fence(response.text)) -async def grade_answers(questions: list[dict], answers: list[dict]) -> dict: +async def generate_followup_question(question: dict, answer: str) -> str: + """ + One sharp follow-up that pushes on the candidate's own wording. + + The point is not to ask something harder - it is to ask something that is only + answerable by whoever actually meant what they wrote. A pasted answer has no + author behind it to defend it. + """ + prompt = f"""A developer was asked this about a project they claim to have built: + +QUESTION: {question.get("question", "")} + +THEIR ANSWER: {answer} + +Write ONE short follow-up question that quotes or directly references specific wording +from THEIR ANSWER and pushes on it. Ask what that specific claim implies, what happens +at its edges, or what the consequence is if it fails. + +Example shape: "You said X handles the empty case by Y - what does the caller see if Y +throws?" + +It must be unanswerable by someone who did not mean what they wrote. Do not ask a +general question about the project. Do not ask them to quote code. + +Return ONLY the question text. No prose, no JSON, no quotes around it. +""" + response = await _model().generate_content_async(prompt) + return response.text.strip().strip('"') + + +async def grade_answers( + questions: list[dict], answers: list[dict], followup: dict | None = None +) -> dict: qa_pairs = "\n\n".join( "Q: {q}\nA: {a}".format( q=q["question"], @@ -107,6 +139,24 @@ async def grade_answers(questions: list[dict], answers: list[dict]) -> dict: for q in questions ) + followup_block = "" + if followup and followup.get("question"): + followup_block = f""" + +FOLLOW-UP ROUND - this is the strongest signal you have. +After answering, the candidate was pushed on their own wording and replied under time +pressure with no chance to prepare: + + Follow-up asked : {followup.get("question")} + They replied : {followup.get("answer") or "(no answer)"} + +Weigh this heavily. If they cannot defend, explain, or even engage with wording they +themselves used, treat the original answer it came from as very likely not their own +work and score that answer down hard, regardless of how polished it looked. If they +defend it coherently, that corroborates the original answer and it should score at +least as well as it otherwise would. +""" + prompt = f"""You are a hackathon judge scoring a developer's answers about a project they claim to have built. @@ -129,6 +179,7 @@ async def grade_answers(questions: list[dict], answers: list[dict]) -> dict: {{"overall_score": 0-100, "breakdown": [{{"question": "...", "score": 0-10, "note": "..."}}]}} {qa_pairs} +{followup_block} """ response = await _model().generate_content_async(prompt) return json.loads(_strip_code_fence(response.text)) diff --git a/backend/app/models/quiz.py b/backend/app/models/quiz.py index 180c7ea..1e17bfa 100644 --- a/backend/app/models/quiz.py +++ b/backend/app/models/quiz.py @@ -16,7 +16,17 @@ "tier": "trivial" | "moderate" | "complex" | "unknown", "reasoning": str }, - "status": "generated" | "graded", + "answers": [ + {"question_id": str, "answer": str, "seconds_left": float | None} + ] | None, + "followup": { + "id": str, + "question": str, + "targets_question_id": str, + "answer": str | None, + "seconds_left": float | None + } | None, + "status": "generated" | "awaiting_followup" | "graded", "result": { "overall_score": float, "breakdown": [{"question": str, "score": int, "note": str}] diff --git a/backend/app/repositories/quiz_repository.py b/backend/app/repositories/quiz_repository.py index 3f40806..4d14d95 100644 --- a/backend/app/repositories/quiz_repository.py +++ b/backend/app/repositories/quiz_repository.py @@ -18,8 +18,15 @@ async def get_attempt(quiz_id: str) -> Optional[dict]: return await collection.find_one({"_id": quiz_id}) -async def update_result(quiz_id: str, result: dict) -> None: +async def update_followup(quiz_id: str, answers: list[dict], followup: dict) -> None: await collection.update_one( {"_id": quiz_id}, - {"$set": {"status": "graded", "result": result}}, + {"$set": {"status": "awaiting_followup", "answers": answers, "followup": followup}}, ) + + +async def update_result(quiz_id: str, result: dict, followup: dict | None = None) -> None: + changes = {"status": "graded", "result": result} + if followup is not None: + changes["followup"] = followup + await collection.update_one({"_id": quiz_id}, {"$set": changes}) diff --git a/backend/app/schemas/quiz.py b/backend/app/schemas/quiz.py index c60fd40..a21fa09 100644 --- a/backend/app/schemas/quiz.py +++ b/backend/app/schemas/quiz.py @@ -3,6 +3,10 @@ from pydantic import BaseModel +# Seconds allowed per question, including the follow-up. The client enforces the +# countdown; the server treats it as the reference point for judging typing pace. +TIME_LIMIT_SECONDS = 75 + class QuizGenerateRequest(BaseModel): repo_url: str @@ -26,11 +30,15 @@ class QuizGenerateResponse(BaseModel): repo_url: str questions: List[QuizQuestion] complexity: ComplexityInfo + time_limit_seconds: int = TIME_LIMIT_SECONDS class QuizAnswer(BaseModel): question_id: str answer: str + # Countdown remaining when the answer was committed. A long answer submitted with + # most of the clock still left is the signal that it was not typed from scratch. + seconds_left: Optional[float] = None class QuizSubmitRequest(BaseModel): @@ -38,7 +46,27 @@ class QuizSubmitRequest(BaseModel): answers: List[QuizAnswer] +class FollowUpQuestion(BaseModel): + id: str + question: str + targets_question_id: str + + class QuizSubmitResponse(BaseModel): + """Submitting answers no longer grades — it opens the follow-up round.""" + + quiz_id: str + followup: FollowUpQuestion + time_limit_seconds: int = TIME_LIMIT_SECONDS + + +class FollowUpRequest(BaseModel): + quiz_id: str + answer: str + seconds_left: Optional[float] = None + + +class QuizResultResponse(BaseModel): quiz_id: str score: float breakdown: dict diff --git a/backend/app/services/quiz_service.py b/backend/app/services/quiz_service.py index f548d1d..ab2548e 100644 --- a/backend/app/services/quiz_service.py +++ b/backend/app/services/quiz_service.py @@ -7,6 +7,7 @@ from app.integrations import gemini_client, github_client from app.repositories import quiz_repository +from app.schemas.quiz import TIME_LIMIT_SECONDS async def create_quiz(repo_url: str, user_id: str | None) -> dict: @@ -33,14 +34,88 @@ async def create_quiz(repo_url: str, user_id: str | None) -> dict: "repo_url": repo_url, "questions": questions, "complexity": complexity, + "time_limit_seconds": TIME_LIMIT_SECONDS, } -async def grade_quiz(quiz_id: str, answers: list[dict]) -> dict: +def pick_suspect_answer(answers: list[dict], time_limit: int = TIME_LIMIT_SECONDS) -> dict | None: + """ + Choose the answer least likely to have been typed by its author. + + Typing pace is the signal. A long, polished answer committed with most of the + clock still unspent was not composed in the box — nobody writes 900 considered + characters in twelve seconds. Weighting rate by length keeps a fast one-liner + from outranking a suspiciously fast essay. + + Falls back to the first non-empty answer, then to the first answer, so the + follow-up round always happens even with no timing data at all. + """ + answered = [a for a in answers if (a.get("answer") or "").strip()] + if not answered: + return answers[0] if answers else None + + timed = [a for a in answered if a.get("seconds_left") is not None] + if not timed: + return answered[0] + + def suspicion(a: dict) -> float: + length = len((a.get("answer") or "").strip()) + elapsed = max(time_limit - float(a["seconds_left"]), 1.0) + return (length / elapsed) * length # fast AND long + + return max(timed, key=suspicion) + + +async def start_followup(quiz_id: str, answers: list[dict]) -> dict: + """ + Record the answers and open the follow-up round. + + Deliberately does not grade yet — grading before the follow-up would let a + candidate bank a score and abandon the round they cannot pass. + """ attempt = await quiz_repository.get_attempt(quiz_id) if not attempt: raise LookupError("quiz_not_found") - result = await gemini_client.grade_answers(attempt["questions"], answers) - await quiz_repository.update_result(quiz_id, result) + suspect = pick_suspect_answer(answers) + if suspect is None: + raise ValueError("no_answers") + + target = next( + (q for q in attempt["questions"] if q["id"] == suspect.get("question_id")), + attempt["questions"][0], + ) + question_text = await gemini_client.generate_followup_question( + target, suspect.get("answer") or "" + ) + + followup = { + "id": str(uuid.uuid4()), + "question": question_text, + "targets_question_id": target["id"], + "answer": None, + } + await quiz_repository.update_followup(quiz_id, answers, followup) + + return { + "quiz_id": quiz_id, + "followup": followup, + "time_limit_seconds": TIME_LIMIT_SECONDS, + } + + +async def grade_quiz(quiz_id: str, followup_answer: str, seconds_left: float | None = None) -> dict: + """Final grading — original answers plus the follow-up defence.""" + attempt = await quiz_repository.get_attempt(quiz_id) + if not attempt: + raise LookupError("quiz_not_found") + + followup = dict(attempt.get("followup") or {}) + followup["answer"] = followup_answer + followup["seconds_left"] = seconds_left + + result = await gemini_client.grade_answers( + attempt["questions"], attempt.get("answers") or [], followup=followup + ) + await quiz_repository.update_result(quiz_id, result, followup) return result diff --git a/frontend/src/features/quiz/QuizPage.jsx b/frontend/src/features/quiz/QuizPage.jsx index 8d58953..0b3fb9e 100644 --- a/frontend/src/features/quiz/QuizPage.jsx +++ b/frontend/src/features/quiz/QuizPage.jsx @@ -1,21 +1,40 @@ -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; import RepoInput from "./components/RepoInput"; import QuestionCard from "./components/QuestionCard"; import ScoreResult from "./components/ScoreResult"; -import { generateQuiz, submitQuiz } from "./api"; +import { generateQuiz, submitQuiz, submitFollowUp } from "./api"; export default function QuizPage() { const [repoUrl, setRepoUrl] = useState(""); const [quiz, setQuiz] = useState(null); const [answers, setAnswers] = useState({}); + const [followup, setFollowup] = useState(null); + const [followupAnswer, setFollowupAnswer] = useState(""); const [result, setResult] = useState(null); const [loading, setLoading] = useState(false); const [error, setError] = useState(""); + const [expired, setExpired] = useState(() => new Set()); + + // Live countdown per question, kept in a ref so ticking never re-renders the page. + const timeLeft = useRef({}); + // Guards against the auto-submit firing twice (expiry racing a manual click). + const sent = useRef({ answers: false, followup: false }); + + const limit = quiz?.time_limit_seconds ?? 75; + + function markExpired(id) { + setExpired((prev) => (prev.has(id) ? prev : new Set(prev).add(id))); + } async function handleGenerate() { setLoading(true); setError(""); setResult(null); + setFollowup(null); + setFollowupAnswer(""); + setExpired(new Set()); + timeLeft.current = {}; + sent.current = { answers: false, followup: false }; try { const data = await generateQuiz(repoUrl); setQuiz(data); @@ -28,22 +47,56 @@ export default function QuizPage() { } async function handleSubmit() { + if (sent.current.answers) return; + sent.current.answers = true; setLoading(true); setError(""); try { - const payload = Object.entries(answers).map(([question_id, answer]) => ({ - question_id, - answer, + // Every question is sent, answered or not — a blank answer is itself a result. + const payload = quiz.questions.map((q) => ({ + question_id: q.id, + answer: answers[q.id] || "", + seconds_left: timeLeft.current[q.id] ?? null, })); - const data = await submitQuiz(quiz.quiz_id, payload); - setResult(data); + setFollowup(await submitQuiz(quiz.quiz_id, payload)); + } catch (e) { + setError(e.message); + sent.current.answers = false; + } finally { + setLoading(false); + } + } + + async function handleFollowUp() { + if (sent.current.followup) return; + sent.current.followup = true; + setLoading(true); + setError(""); + try { + const id = followup.followup.id; + setResult( + await submitFollowUp(quiz.quiz_id, followupAnswer, timeLeft.current[id] ?? null) + ); } catch (e) { setError(e.message); + sent.current.followup = false; } finally { setLoading(false); } } + // When every question's clock has run out, submit whatever is there. + useEffect(() => { + if (quiz && !followup && !result && expired.size >= quiz.questions.length) { + handleSubmit(); + } + }, [expired, quiz, followup, result]); + + const followupExpired = followup && expired.has(followup.followup.id); + useEffect(() => { + if (followupExpired && !result) handleFollowUp(); + }, [followupExpired, result]); + return (

Repo Quiz

@@ -53,18 +106,45 @@ export default function QuizPage() { {error &&

{error}

} - {quiz && !result && ( + {quiz && !followup && !result && (
+

+ {limit}s per question · typing only, paste is disabled · answers lock when the + timer runs out +

{quiz.questions.map((q) => ( setAnswers({ ...answers, [id]: val })} + timeLimit={limit} + onTick={(id, s) => (timeLeft.current[id] = s)} + onExpire={markExpired} + onAnswerChange={(id, val) => setAnswers((prev) => ({ ...prev, [id]: val }))} /> ))} +
+ )} + + {followup && !result && ( +
+

+ One follow-up on what you just wrote. Same {followup.time_limit_seconds}s, same + rules. +

+ (timeLeft.current[id] = s)} + onExpire={markExpired} + onAnswerChange={(_, val) => setFollowupAnswer(val)} + /> +
)} diff --git a/frontend/src/features/quiz/api.js b/frontend/src/features/quiz/api.js index c64b5cf..56f0e79 100644 --- a/frontend/src/features/quiz/api.js +++ b/frontend/src/features/quiz/api.js @@ -7,9 +7,18 @@ export function generateQuiz(repoUrl) { }); } +// Records answers and opens the follow-up round — does not return a score. export function submitQuiz(quizId, answers) { return request("/quiz/submit", { method: "POST", body: JSON.stringify({ quiz_id: quizId, answers }), }); } + +// Final grading: original answers plus the follow-up defence. +export function submitFollowUp(quizId, answer, secondsLeft) { + return request("/quiz/followup", { + method: "POST", + body: JSON.stringify({ quiz_id: quizId, answer, seconds_left: secondsLeft }), + }); +} diff --git a/frontend/src/features/quiz/components/QuestionCard.jsx b/frontend/src/features/quiz/components/QuestionCard.jsx index f653c49..de932ad 100644 --- a/frontend/src/features/quiz/components/QuestionCard.jsx +++ b/frontend/src/features/quiz/components/QuestionCard.jsx @@ -1,14 +1,77 @@ -export default function QuestionCard({ question, answer, onAnswerChange }) { +import { useEffect, useRef, useState } from "react"; + +/** + * One timed question. + * + * The countdown starts when the card mounts and is derived from wall-clock time + * rather than accumulated ticks, so a backgrounded tab cannot buy extra seconds by + * having its timers throttled. On expiry the answer is committed as-is — blank + * included — and the box locks. + * + * Paste is blocked so the answer has to be typed. This is a speed bump, not a + * security control: it stops the casual paste from another tab, and the real + * signal is the typing pace reported via onTick. + */ +export default function QuestionCard({ + question, + answer, + onAnswerChange, + onExpire, + onTick, + timeLimit = 75, +}) { + const [left, setLeft] = useState(timeLimit); + const cbs = useRef({ onExpire, onTick }); + cbs.current = { onExpire, onTick }; + + useEffect(() => { + const startedAt = Date.now(); + let done = false; + const tick = () => { + const remaining = Math.max( + 0, + timeLimit - Math.floor((Date.now() - startedAt) / 1000) + ); + setLeft(remaining); + cbs.current.onTick?.(question.id, remaining); + if (remaining === 0 && !done) { + done = true; + clearInterval(handle); + cbs.current.onExpire?.(question.id); + } + }; + const handle = setInterval(tick, 250); + tick(); + return () => clearInterval(handle); + }, [question.id, timeLimit]); + + const locked = left === 0; + const mins = Math.floor(left / 60); + const secs = String(left % 60).padStart(2, "0"); + + const block = (e) => e.preventDefault(); + return ( -
+

{question.question} {question.file_reference && ({question.file_reference})}

+ +
+ {question.category && {question.category}} + + {locked ? "time up" : `${mins}:${secs}`} + +
+