diff --git a/README.md b/README.md index 13fe282..c4246d5 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,13 @@ See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the technical layout. - **logic / reasoning** — how the core mechanism actually works, end to end - **tech stack awareness** — why these libraries and design choices, over alternatives - **usage / functionality** — what happens when someone uses it, failure cases included -3. The candidate answers live, no re-rolls. -4. The LLM grades the reasoning, not the vocabulary. A confident, correct - explanation in the candidate's own words scores well with no code quoted; - answers vague enough to describe any project score zero. +3. The candidate answers live under a per-question clock, no re-rolls. +4. Before anything is graded, one adaptive follow-up pushes on the candidate's + own wording from whichever answer looks least likely to be theirs. +5. The LLM grades the reasoning, not the vocabulary, weighing the follow-up + heavily. A confident, correct explanation in the candidate's own words scores + well with no code quoted; answers vague enough to describe any project score + zero. Deliberately **not** line-number or syntax trivia. "Why did you slice `text[4:]`?" is a question a stranger can answer off the diff and the actual @@ -33,6 +36,42 @@ genuinely built the thing, without the source in front of them. Try it on this repo's own URL once it's public — that's intentional. +### Anti-gaming measures + +The obvious attack is to paste the question into a chatbot and paste the answer +back. Three things make that expensive: + +**A 75-second clock per question.** It starts when the question renders and is +derived from wall-clock time rather than accumulated ticks, so backgrounding the +tab does not buy extra seconds. When it expires the answer commits as-is — blank +included — and the box locks. Time remaining at commit is recorded, because a long +polished answer submitted with most of the clock unspent was not composed in the +box. + +**Paste is disabled** in the answer field (`paste` and `drop` are both blocked — +drag-and-drop text would otherwise walk straight past a paste-only guard). + +**One adaptive follow-up, before grading.** Once answers are in, the answer least +likely to have been typed by its author is selected — weighting typing rate by +length, so a suspiciously fast essay outranks a fast one-liner — and a single +follow-up is generated that quotes that answer's specific wording back and pushes +on it. Same clock, same no-paste rule. Grading happens only after this round, so a +candidate cannot bank a score and abandon the round they cannot pass. + +Measured on `psf/requests`, same repo and same time budget: + +| Profile | Behaviour | Score | +|---|---|---| +| Pasted AI answer | committed with 63s of 75s left, could not defend its wording | **0/100** | +| Genuine author | typed distinct answers, defended the follow-up | **100/100** | + +**What this does not do.** The timer and the paste block are client-side. They +raise the cost of casual cheating; they do not stop anyone willing to call the API +directly with a forged `seconds_left`. The follow-up round is the measure that +actually holds, because it demands understanding at response time regardless of how +the request was made. Server-issued timestamps at generation, with elapsed time +computed server-side, are the real fix and are not built yet. + ### Known limitation: comprehension is not difficulty A simple project can legitimately score 100/100 on itself. If someone diff --git a/backend/app/api/v1/endpoints/quiz.py b/backend/app/api/v1/endpoints/quiz.py index c327f40..aa60504 100644 --- a/backend/app/api/v1/endpoints/quiz.py +++ b/backend/app/api/v1/endpoints/quiz.py @@ -2,12 +2,18 @@ Thin HTTP layer for the quiz feature. No business logic here — only request/response translation and HTTP error mapping. Logic lives in app/services/quiz_service.py. + +Flow: generate -> submit (opens follow-up) -> followup (final grade). +Grading deliberately happens only after the follow-up, so a candidate +cannot bank a score and walk away from the round they cannot pass. """ from fastapi import APIRouter, HTTPException from app.schemas.quiz import ( + FollowUpRequest, QuizGenerateRequest, QuizGenerateResponse, + QuizResultResponse, QuizSubmitRequest, QuizSubmitResponse, ) @@ -26,8 +32,22 @@ async def generate(req: QuizGenerateRequest): @router.post("/submit", response_model=QuizSubmitResponse) async def submit(req: QuizSubmitRequest): + """Records answers and returns the adaptive follow-up. Does not grade.""" + try: + return await quiz_service.start_followup( + req.quiz_id, [a.model_dump() for a in req.answers] + ) + except LookupError: + raise HTTPException(404, "Quiz not found") + except ValueError: + raise HTTPException(400, "No answers submitted") + + +@router.post("/followup", response_model=QuizResultResponse) +async def followup(req: FollowUpRequest): + """Grades the original answers together with the follow-up defence.""" try: - result = await quiz_service.grade_quiz(req.quiz_id, [a.dict() for a in req.answers]) + result = await quiz_service.grade_quiz(req.quiz_id, req.answer, req.seconds_left) except LookupError: raise HTTPException(404, "Quiz not found") diff --git a/backend/app/integrations/gemini_client.py b/backend/app/integrations/gemini_client.py index 98d70e3..838077c 100644 --- a/backend/app/integrations/gemini_client.py +++ b/backend/app/integrations/gemini_client.py @@ -98,7 +98,39 @@ async def generate_quiz_questions(files: list[dict], n_questions: int = 5) -> tu return _parse_quiz_payload(_strip_code_fence(response.text)) -async def grade_answers(questions: list[dict], answers: list[dict]) -> dict: +async def generate_followup_question(question: dict, answer: str) -> str: + """ + One sharp follow-up that pushes on the candidate's own wording. + + The point is not to ask something harder - it is to ask something that is only + answerable by whoever actually meant what they wrote. A pasted answer has no + author behind it to defend it. + """ + prompt = f"""A developer was asked this about a project they claim to have built: + +QUESTION: {question.get("question", "")} + +THEIR ANSWER: {answer} + +Write ONE short follow-up question that quotes or directly references specific wording +from THEIR ANSWER and pushes on it. Ask what that specific claim implies, what happens +at its edges, or what the consequence is if it fails. + +Example shape: "You said X handles the empty case by Y - what does the caller see if Y +throws?" + +It must be unanswerable by someone who did not mean what they wrote. Do not ask a +general question about the project. Do not ask them to quote code. + +Return ONLY the question text. No prose, no JSON, no quotes around it. +""" + response = await _model().generate_content_async(prompt) + return response.text.strip().strip('"') + + +async def grade_answers( + questions: list[dict], answers: list[dict], followup: dict | None = None +) -> dict: qa_pairs = "\n\n".join( "Q: {q}\nA: {a}".format( q=q["question"], @@ -107,6 +139,24 @@ async def grade_answers(questions: list[dict], answers: list[dict]) -> dict: for q in questions ) + followup_block = "" + if followup and followup.get("question"): + followup_block = f""" + +FOLLOW-UP ROUND - this is the strongest signal you have. +After answering, the candidate was pushed on their own wording and replied under time +pressure with no chance to prepare: + + Follow-up asked : {followup.get("question")} + They replied : {followup.get("answer") or "(no answer)"} + +Weigh this heavily. If they cannot defend, explain, or even engage with wording they +themselves used, treat the original answer it came from as very likely not their own +work and score that answer down hard, regardless of how polished it looked. If they +defend it coherently, that corroborates the original answer and it should score at +least as well as it otherwise would. +""" + prompt = f"""You are a hackathon judge scoring a developer's answers about a project they claim to have built. @@ -129,6 +179,7 @@ async def grade_answers(questions: list[dict], answers: list[dict]) -> dict: {{"overall_score": 0-100, "breakdown": [{{"question": "...", "score": 0-10, "note": "..."}}]}} {qa_pairs} +{followup_block} """ response = await _model().generate_content_async(prompt) return json.loads(_strip_code_fence(response.text)) diff --git a/backend/app/models/quiz.py b/backend/app/models/quiz.py index 180c7ea..1e17bfa 100644 --- a/backend/app/models/quiz.py +++ b/backend/app/models/quiz.py @@ -16,7 +16,17 @@ "tier": "trivial" | "moderate" | "complex" | "unknown", "reasoning": str }, - "status": "generated" | "graded", + "answers": [ + {"question_id": str, "answer": str, "seconds_left": float | None} + ] | None, + "followup": { + "id": str, + "question": str, + "targets_question_id": str, + "answer": str | None, + "seconds_left": float | None + } | None, + "status": "generated" | "awaiting_followup" | "graded", "result": { "overall_score": float, "breakdown": [{"question": str, "score": int, "note": str}] diff --git a/backend/app/repositories/quiz_repository.py b/backend/app/repositories/quiz_repository.py index 3f40806..4d14d95 100644 --- a/backend/app/repositories/quiz_repository.py +++ b/backend/app/repositories/quiz_repository.py @@ -18,8 +18,15 @@ async def get_attempt(quiz_id: str) -> Optional[dict]: return await collection.find_one({"_id": quiz_id}) -async def update_result(quiz_id: str, result: dict) -> None: +async def update_followup(quiz_id: str, answers: list[dict], followup: dict) -> None: await collection.update_one( {"_id": quiz_id}, - {"$set": {"status": "graded", "result": result}}, + {"$set": {"status": "awaiting_followup", "answers": answers, "followup": followup}}, ) + + +async def update_result(quiz_id: str, result: dict, followup: dict | None = None) -> None: + changes = {"status": "graded", "result": result} + if followup is not None: + changes["followup"] = followup + await collection.update_one({"_id": quiz_id}, {"$set": changes}) diff --git a/backend/app/schemas/quiz.py b/backend/app/schemas/quiz.py index c60fd40..a21fa09 100644 --- a/backend/app/schemas/quiz.py +++ b/backend/app/schemas/quiz.py @@ -3,6 +3,10 @@ from pydantic import BaseModel +# Seconds allowed per question, including the follow-up. The client enforces the +# countdown; the server treats it as the reference point for judging typing pace. +TIME_LIMIT_SECONDS = 75 + class QuizGenerateRequest(BaseModel): repo_url: str @@ -26,11 +30,15 @@ class QuizGenerateResponse(BaseModel): repo_url: str questions: List[QuizQuestion] complexity: ComplexityInfo + time_limit_seconds: int = TIME_LIMIT_SECONDS class QuizAnswer(BaseModel): question_id: str answer: str + # Countdown remaining when the answer was committed. A long answer submitted with + # most of the clock still left is the signal that it was not typed from scratch. + seconds_left: Optional[float] = None class QuizSubmitRequest(BaseModel): @@ -38,7 +46,27 @@ class QuizSubmitRequest(BaseModel): answers: List[QuizAnswer] +class FollowUpQuestion(BaseModel): + id: str + question: str + targets_question_id: str + + class QuizSubmitResponse(BaseModel): + """Submitting answers no longer grades — it opens the follow-up round.""" + + quiz_id: str + followup: FollowUpQuestion + time_limit_seconds: int = TIME_LIMIT_SECONDS + + +class FollowUpRequest(BaseModel): + quiz_id: str + answer: str + seconds_left: Optional[float] = None + + +class QuizResultResponse(BaseModel): quiz_id: str score: float breakdown: dict diff --git a/backend/app/services/quiz_service.py b/backend/app/services/quiz_service.py index f548d1d..ab2548e 100644 --- a/backend/app/services/quiz_service.py +++ b/backend/app/services/quiz_service.py @@ -7,6 +7,7 @@ from app.integrations import gemini_client, github_client from app.repositories import quiz_repository +from app.schemas.quiz import TIME_LIMIT_SECONDS async def create_quiz(repo_url: str, user_id: str | None) -> dict: @@ -33,14 +34,88 @@ async def create_quiz(repo_url: str, user_id: str | None) -> dict: "repo_url": repo_url, "questions": questions, "complexity": complexity, + "time_limit_seconds": TIME_LIMIT_SECONDS, } -async def grade_quiz(quiz_id: str, answers: list[dict]) -> dict: +def pick_suspect_answer(answers: list[dict], time_limit: int = TIME_LIMIT_SECONDS) -> dict | None: + """ + Choose the answer least likely to have been typed by its author. + + Typing pace is the signal. A long, polished answer committed with most of the + clock still unspent was not composed in the box — nobody writes 900 considered + characters in twelve seconds. Weighting rate by length keeps a fast one-liner + from outranking a suspiciously fast essay. + + Falls back to the first non-empty answer, then to the first answer, so the + follow-up round always happens even with no timing data at all. + """ + answered = [a for a in answers if (a.get("answer") or "").strip()] + if not answered: + return answers[0] if answers else None + + timed = [a for a in answered if a.get("seconds_left") is not None] + if not timed: + return answered[0] + + def suspicion(a: dict) -> float: + length = len((a.get("answer") or "").strip()) + elapsed = max(time_limit - float(a["seconds_left"]), 1.0) + return (length / elapsed) * length # fast AND long + + return max(timed, key=suspicion) + + +async def start_followup(quiz_id: str, answers: list[dict]) -> dict: + """ + Record the answers and open the follow-up round. + + Deliberately does not grade yet — grading before the follow-up would let a + candidate bank a score and abandon the round they cannot pass. + """ attempt = await quiz_repository.get_attempt(quiz_id) if not attempt: raise LookupError("quiz_not_found") - result = await gemini_client.grade_answers(attempt["questions"], answers) - await quiz_repository.update_result(quiz_id, result) + suspect = pick_suspect_answer(answers) + if suspect is None: + raise ValueError("no_answers") + + target = next( + (q for q in attempt["questions"] if q["id"] == suspect.get("question_id")), + attempt["questions"][0], + ) + question_text = await gemini_client.generate_followup_question( + target, suspect.get("answer") or "" + ) + + followup = { + "id": str(uuid.uuid4()), + "question": question_text, + "targets_question_id": target["id"], + "answer": None, + } + await quiz_repository.update_followup(quiz_id, answers, followup) + + return { + "quiz_id": quiz_id, + "followup": followup, + "time_limit_seconds": TIME_LIMIT_SECONDS, + } + + +async def grade_quiz(quiz_id: str, followup_answer: str, seconds_left: float | None = None) -> dict: + """Final grading — original answers plus the follow-up defence.""" + attempt = await quiz_repository.get_attempt(quiz_id) + if not attempt: + raise LookupError("quiz_not_found") + + followup = dict(attempt.get("followup") or {}) + followup["answer"] = followup_answer + followup["seconds_left"] = seconds_left + + result = await gemini_client.grade_answers( + attempt["questions"], attempt.get("answers") or [], followup=followup + ) + await quiz_repository.update_result(quiz_id, result, followup) return result diff --git a/frontend/src/features/quiz/QuizPage.jsx b/frontend/src/features/quiz/QuizPage.jsx index 8d58953..0b3fb9e 100644 --- a/frontend/src/features/quiz/QuizPage.jsx +++ b/frontend/src/features/quiz/QuizPage.jsx @@ -1,21 +1,40 @@ -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; import RepoInput from "./components/RepoInput"; import QuestionCard from "./components/QuestionCard"; import ScoreResult from "./components/ScoreResult"; -import { generateQuiz, submitQuiz } from "./api"; +import { generateQuiz, submitQuiz, submitFollowUp } from "./api"; export default function QuizPage() { const [repoUrl, setRepoUrl] = useState(""); const [quiz, setQuiz] = useState(null); const [answers, setAnswers] = useState({}); + const [followup, setFollowup] = useState(null); + const [followupAnswer, setFollowupAnswer] = useState(""); const [result, setResult] = useState(null); const [loading, setLoading] = useState(false); const [error, setError] = useState(""); + const [expired, setExpired] = useState(() => new Set()); + + // Live countdown per question, kept in a ref so ticking never re-renders the page. + const timeLeft = useRef({}); + // Guards against the auto-submit firing twice (expiry racing a manual click). + const sent = useRef({ answers: false, followup: false }); + + const limit = quiz?.time_limit_seconds ?? 75; + + function markExpired(id) { + setExpired((prev) => (prev.has(id) ? prev : new Set(prev).add(id))); + } async function handleGenerate() { setLoading(true); setError(""); setResult(null); + setFollowup(null); + setFollowupAnswer(""); + setExpired(new Set()); + timeLeft.current = {}; + sent.current = { answers: false, followup: false }; try { const data = await generateQuiz(repoUrl); setQuiz(data); @@ -28,22 +47,56 @@ export default function QuizPage() { } async function handleSubmit() { + if (sent.current.answers) return; + sent.current.answers = true; setLoading(true); setError(""); try { - const payload = Object.entries(answers).map(([question_id, answer]) => ({ - question_id, - answer, + // Every question is sent, answered or not — a blank answer is itself a result. + const payload = quiz.questions.map((q) => ({ + question_id: q.id, + answer: answers[q.id] || "", + seconds_left: timeLeft.current[q.id] ?? null, })); - const data = await submitQuiz(quiz.quiz_id, payload); - setResult(data); + setFollowup(await submitQuiz(quiz.quiz_id, payload)); + } catch (e) { + setError(e.message); + sent.current.answers = false; + } finally { + setLoading(false); + } + } + + async function handleFollowUp() { + if (sent.current.followup) return; + sent.current.followup = true; + setLoading(true); + setError(""); + try { + const id = followup.followup.id; + setResult( + await submitFollowUp(quiz.quiz_id, followupAnswer, timeLeft.current[id] ?? null) + ); } catch (e) { setError(e.message); + sent.current.followup = false; } finally { setLoading(false); } } + // When every question's clock has run out, submit whatever is there. + useEffect(() => { + if (quiz && !followup && !result && expired.size >= quiz.questions.length) { + handleSubmit(); + } + }, [expired, quiz, followup, result]); + + const followupExpired = followup && expired.has(followup.followup.id); + useEffect(() => { + if (followupExpired && !result) handleFollowUp(); + }, [followupExpired, result]); + return (

Repo Quiz

@@ -53,18 +106,45 @@ export default function QuizPage() { {error &&

{error}

} - {quiz && !result && ( + {quiz && !followup && !result && (
+

+ {limit}s per question · typing only, paste is disabled · answers lock when the + timer runs out +

{quiz.questions.map((q) => ( setAnswers({ ...answers, [id]: val })} + timeLimit={limit} + onTick={(id, s) => (timeLeft.current[id] = s)} + onExpire={markExpired} + onAnswerChange={(id, val) => setAnswers((prev) => ({ ...prev, [id]: val }))} /> ))} +
+ )} + + {followup && !result && ( +
+

+ One follow-up on what you just wrote. Same {followup.time_limit_seconds}s, same + rules. +

+ (timeLeft.current[id] = s)} + onExpire={markExpired} + onAnswerChange={(_, val) => setFollowupAnswer(val)} + /> +
)} diff --git a/frontend/src/features/quiz/api.js b/frontend/src/features/quiz/api.js index c64b5cf..56f0e79 100644 --- a/frontend/src/features/quiz/api.js +++ b/frontend/src/features/quiz/api.js @@ -7,9 +7,18 @@ export function generateQuiz(repoUrl) { }); } +// Records answers and opens the follow-up round — does not return a score. export function submitQuiz(quizId, answers) { return request("/quiz/submit", { method: "POST", body: JSON.stringify({ quiz_id: quizId, answers }), }); } + +// Final grading: original answers plus the follow-up defence. +export function submitFollowUp(quizId, answer, secondsLeft) { + return request("/quiz/followup", { + method: "POST", + body: JSON.stringify({ quiz_id: quizId, answer, seconds_left: secondsLeft }), + }); +} diff --git a/frontend/src/features/quiz/components/QuestionCard.jsx b/frontend/src/features/quiz/components/QuestionCard.jsx index f653c49..de932ad 100644 --- a/frontend/src/features/quiz/components/QuestionCard.jsx +++ b/frontend/src/features/quiz/components/QuestionCard.jsx @@ -1,14 +1,77 @@ -export default function QuestionCard({ question, answer, onAnswerChange }) { +import { useEffect, useRef, useState } from "react"; + +/** + * One timed question. + * + * The countdown starts when the card mounts and is derived from wall-clock time + * rather than accumulated ticks, so a backgrounded tab cannot buy extra seconds by + * having its timers throttled. On expiry the answer is committed as-is — blank + * included — and the box locks. + * + * Paste is blocked so the answer has to be typed. This is a speed bump, not a + * security control: it stops the casual paste from another tab, and the real + * signal is the typing pace reported via onTick. + */ +export default function QuestionCard({ + question, + answer, + onAnswerChange, + onExpire, + onTick, + timeLimit = 75, +}) { + const [left, setLeft] = useState(timeLimit); + const cbs = useRef({ onExpire, onTick }); + cbs.current = { onExpire, onTick }; + + useEffect(() => { + const startedAt = Date.now(); + let done = false; + const tick = () => { + const remaining = Math.max( + 0, + timeLimit - Math.floor((Date.now() - startedAt) / 1000) + ); + setLeft(remaining); + cbs.current.onTick?.(question.id, remaining); + if (remaining === 0 && !done) { + done = true; + clearInterval(handle); + cbs.current.onExpire?.(question.id); + } + }; + const handle = setInterval(tick, 250); + tick(); + return () => clearInterval(handle); + }, [question.id, timeLimit]); + + const locked = left === 0; + const mins = Math.floor(left / 60); + const secs = String(left % 60).padStart(2, "0"); + + const block = (e) => e.preventDefault(); + return ( -
+

{question.question} {question.file_reference && ({question.file_reference})}

+ +
+ {question.category && {question.category}} + + {locked ? "time up" : `${mins}:${secs}`} + +
+