From d309e659b68e0523f2f1d0d36ca967b3bb118aeb Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 17 Jul 2026 13:43:47 +0100 Subject: [PATCH 1/4] Refactored evaluation logic for improved readability and error handling --- app/evaluation.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/app/evaluation.py b/app/evaluation.py index d0629d7..e518a05 100644 --- a/app/evaluation.py +++ b/app/evaluation.py @@ -24,28 +24,29 @@ def evaluation_function(response, answer, params) -> dict: rtol = params.get("rtol", 0) atol = params.get("atol", 0) - is_correct = None - real_diff = None - if not (isinstance(answer, int) or isinstance(answer, float)): raise Exception("Answer must be a number.") - real_diff = None allowed_diff = atol + rtol * abs(answer) allowed_diff += spacing(answer) - is_correct = False - feedback = "" + if not (isinstance(response, int) or isinstance(response, float)): - feedback = "Please enter a number." - else: - real_diff = abs(response - answer) - allowed_diff = atol + rtol * abs(answer) - allowed_diff += spacing(answer) - is_correct = bool(real_diff <= allowed_diff) + return { + "is_correct": False, + "real_diff": None, + "allowed_diff": allowed_diff, + "feedback": "Please enter a number.", + } + + + real_diff = abs(response - answer) + allowed_diff = atol + rtol * abs(answer) + allowed_diff += spacing(answer) + is_correct = bool(real_diff <= allowed_diff) return { "is_correct": is_correct, "real_diff": real_diff, "allowed_diff": allowed_diff, - "feedback": feedback, + "feedback": "", } From d8ea34a2592382fd53829dfe2eb6e31ce46f9ee1 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 17 Jul 2026 13:45:46 +0100 Subject: [PATCH 2/4] Added support for using relative and absolute tolerance parameters --- app/evaluation.py | 8 ++++---- app/evaluation_test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/app/evaluation.py b/app/evaluation.py index e518a05..1119b6d 100644 --- a/app/evaluation.py +++ b/app/evaluation.py @@ -21,13 +21,13 @@ def evaluation_function(response, answer, params) -> dict: to output the grading response. """ - rtol = params.get("rtol", 0) - atol = params.get("atol", 0) + relative_tolerance = params.get("relative_tolerance", params.get("rtol", 0)) + absolute_tolerance = params.get("absolute_tolerance", params.get("atol", 0)) if not (isinstance(answer, int) or isinstance(answer, float)): raise Exception("Answer must be a number.") - allowed_diff = atol + rtol * abs(answer) + allowed_diff = absolute_tolerance + relative_tolerance * abs(answer) allowed_diff += spacing(answer) if not (isinstance(response, int) or isinstance(response, float)): @@ -40,7 +40,7 @@ def evaluation_function(response, answer, params) -> dict: real_diff = abs(response - answer) - allowed_diff = atol + rtol * abs(answer) + allowed_diff = absolute_tolerance + relative_tolerance * abs(answer) allowed_diff += spacing(answer) is_correct = bool(real_diff <= allowed_diff) diff --git a/app/evaluation_test.py b/app/evaluation_test.py index a792d54..fda062e 100644 --- a/app/evaluation_test.py +++ b/app/evaluation_test.py @@ -244,5 +244,35 @@ def test_response_is_not_number(self): self.assertEqual(response.get("is_correct"), False) self.assertEqual("Please enter a number." in response.get("feedback"), True) + def test_full_param_correct(self): + body = { + "response": 1e6, + "answer": 1e7, + "params": { + "relative_tolerance": 1e-1, + "absolute_tolerance": 8.2e6 + }, + } + + response = evaluation_function(body['response'], body['answer'], + body.get('params', {})) + + self.assertEqual(response.get("is_correct"), True) + + def test_full_param_incorrect(self): + body = { + "response": 1e6, + "answer": 2e7, + "params": { + "relative_tolerance": 1e-1, + "absolute_tolerance": 8.2e6 + }, + } + + response = evaluation_function(body['response'], body['answer'], + body.get('params', {})) + + self.assertEqual(response.get("is_correct"), False) + if __name__ == "__main__": unittest.main() From 63e8d78215f244492fded736c1820a585813ff3d Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 17 Jul 2026 18:27:20 +0100 Subject: [PATCH 3/4] Renamed tolerance parameters for clarity across documentation and examples --- app/docs/dev.md | 14 +++++++------- app/docs/user.md | 16 ++++++++-------- readme.md | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/docs/dev.md b/app/docs/dev.md index e43156b..dbb1f6e 100644 --- a/app/docs/dev.md +++ b/app/docs/dev.md @@ -2,10 +2,10 @@ This simple evaluation function checks if the supplied response is within a tolerance range defined in `params`. Works exactly like the [numpy.isclose](https://numpy.org/doc/stable/reference/generated/numpy.isclose.html#numpy.isclose) function. -Valid params include `atol` and `rtol`, which can be used in combination, or alone. As the comparison made is the following: +Valid params include `absolute_tolerance` and `relative_tolerance`, which can be used in combination, or alone. As the comparison made is the following: ```python -is_correct = abs(res - ans) <= (atol + rtol*abs(ans)) +is_correct = abs(res - ans) <= (absolute_tolerance + relative_tolerance*abs(ans)) ``` ## Inputs @@ -15,17 +15,17 @@ is_correct = abs(res - ans) <= (atol + rtol*abs(ans)) "response": "", "answer": "", "params": { - "atol": "", - "rtol": "" + "absolute_tolerance": "", + "relative_tolerance": "" } } ``` -### `atol` +### `absolute_tolerance` Absolute tolerance parameter -### `rtol` +### `relative_tolerance` Relative tolerance parameter @@ -42,7 +42,7 @@ Relative tolerance parameter Real difference between the given answer and response ### `allowed_diff` -Allowed difference between answer and response, calculated using the supplied `atol` and `rtol` parameters +Allowed difference between answer and response, calculated using the supplied `absolute_tolerance` and `relative_tolerance` parameters ## Examples \ No newline at end of file diff --git a/app/docs/user.md b/app/docs/user.md index 438113f..81372f3 100644 --- a/app/docs/user.md +++ b/app/docs/user.md @@ -5,20 +5,20 @@ Use this evaluation function to check if a student's response is within a tolera A response is accepted if: ``` -|response - answer| ≤ atol + rtol × |answer| +|response - answer| ≤ absolute_tolerance + relative_tolerance × |answer| ``` -The left-hand side is the absolute difference between the student's response and the correct answer. The right-hand side is the total allowed difference, made up of a fixed part (`atol`) and a part that scales with the size of the answer (`rtol × |answer|`). A response is marked correct whenever the actual difference does not exceed the allowed difference. +The left-hand side is the absolute difference between the student's response and the correct answer. The right-hand side is the total allowed difference, made up of a fixed part (`absolute_tolerance`) and a part that scales with the size of the answer (`relative_tolerance × |answer|`). A response is marked correct whenever the actual difference does not exceed the allowed difference. ## Parameters Both parameters default to `0` (exact match required) and can be used individually or together. -### `atol` — Absolute tolerance +### `absolute_tolerance` — Absolute tolerance Specifies a fixed margin around the answer, regardless of its magnitude. Use this when you know the acceptable error in the same units as the answer. -### `rtol` — Relative tolerance +### `relative_tolerance` — Relative tolerance Specifies an acceptable error as a fraction of the answer's magnitude. Use this when the answer is very large or very small and a percentage-based margin makes more sense than a fixed one. @@ -31,7 +31,7 @@ No params needed. The student must enter exactly `42` (floating-point precision ### Absolute tolerance ```json -{ "atol": 0.05 } +{ "absolute_tolerance": 0.05 } ``` With answer `9.81`, accepts any response in the range **9.76 – 9.86**. Good for physical measurements where the acceptable error is known in the same units. @@ -39,7 +39,7 @@ With answer `9.81`, accepts any response in the range **9.76 – 9.86**. Good fo ### Relative tolerance ```json -{ "rtol": 0.01 } +{ "relative_tolerance": 0.01 } ``` With answer `6.674e-11`, accepts any response within **1%** of the answer. Good for very large or very small values where a fixed margin would be impractical. @@ -47,10 +47,10 @@ With answer `6.674e-11`, accepts any response within **1%** of the answer. Good ### Combined tolerances ```json -{ "atol": 0.01, "rtol": 0.005 } +{ "absolute_tolerance": 0.01, "relative_tolerance": 0.005 } ``` -Both tolerances contribute: with answer `9.81`, the allowed difference is `0.01 + 0.005 × 9.81 ≈ 0.059`. Useful when you want a minimum floor (`atol`) plus a proportional allowance (`rtol`). +Both tolerances contribute: with answer `9.81`, the allowed difference is `0.01 + 0.005 × 9.81 ≈ 0.059`. Useful when you want a minimum floor (`absolute_tolerance`) plus a proportional allowance (`relative_tolerance`). ## Notes diff --git a/readme.md b/readme.md index 7750fb5..6d08dbb 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # IsSimilar -This function checks whether a student's numeric response is within an acceptable tolerance of the correct answer, using absolute (`atol`) and relative (`rtol`) tolerance parameters. The comparison follows the formula: `|response - answer| ≤ atol + rtol × |answer|`. By default both tolerances are 0, requiring an exact match (within floating-point precision). +This function checks whether a student's numeric response is within an acceptable tolerance of the correct answer, using absolute (`absolute_tolerance`) and relative (`relative_tolerance`) tolerance parameters. The comparison follows the formula: `|response - answer| ≤ absolute_tolerance + relative_tolerance × |answer|`. By default both tolerances are 0, requiring an exact match (within floating-point precision). For more information, look at the docs in `app/docs/`. From 86be9869258b2ae3384a175adcdadc86da639fb1 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 17 Jul 2026 18:28:40 +0100 Subject: [PATCH 4/4] Removed unused tolerance calculation logic from evaluation module --- app/evaluation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/evaluation.py b/app/evaluation.py index 1119b6d..871f314 100644 --- a/app/evaluation.py +++ b/app/evaluation.py @@ -40,8 +40,6 @@ def evaluation_function(response, answer, params) -> dict: real_diff = abs(response - answer) - allowed_diff = absolute_tolerance + relative_tolerance * abs(answer) - allowed_diff += spacing(answer) is_correct = bool(real_diff <= allowed_diff) return {