From 35be9185db8220c1ab257a6e24f048048ab43201 Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Tue, 11 Aug 2026 13:34:28 +1200 Subject: [PATCH 1/3] Fix gtagging --- docs/assets/javascripts/chat.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/assets/javascripts/chat.js b/docs/assets/javascripts/chat.js index 418ce2b77..f79ecd3bf 100644 --- a/docs/assets/javascripts/chat.js +++ b/docs/assets/javascripts/chat.js @@ -9,6 +9,9 @@ let sending = false; let codeBlockSeq = 0; + window.dataLayer = window.dataLayer || []; + function gtag() { window.dataLayer.push(arguments); } + const ESCAPE_MAP = { "&": "&", "<": "<", @@ -260,9 +263,10 @@ { role: "user", content: question }, { role: "assistant", content: answer } ); - if (window.dataLayer) window.dataLayer.push(["event", "rag_chat", { question: question, answer: answer }]); history.splice(0, Math.max(0, history.length - 12)); + gtag("event", "rag_chat", { question: question, answer: answer }); + } catch (err) { botMsg.innerHTML = `

Error: ${escapeHtml(err.message)}

`; } finally { From 106d4dba626c0bddea970e1aa8a36f4bf06d34c7 Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Wed, 12 Aug 2026 13:41:21 +1200 Subject: [PATCH 2/3] first. (bad) --- checks/run_test_build.py | 47 ++++++++++++++++++++++---- docs/FORMAT.md | 7 +++- macro_hooks.py | 27 +++++++++++++++ mkdocs_hooks.py | 72 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 8 deletions(-) diff --git a/checks/run_test_build.py b/checks/run_test_build.py index 9cf81a800..9df4d8220 100755 --- a/checks/run_test_build.py +++ b/checks/run_test_build.py @@ -4,17 +4,35 @@ from mkdocs.commands import build, serve from mkdocs.config.base import Config, load_config import logging +import os import sys import re +import tempfile import time +import requests -""" + +""" This works but is a bit messy """ msg_count = {"DEBUG": 0, "NOTICE": 0, "WARNING": 0, "ERROR": 0} +MODULES_LIST_URL = "https://raw.githubusercontent.com/nesi/modules-list/main/module-list.json" + + +def fetch_module_list(): + """Fetch the latest module-list.json contents, or None if unavailable.""" + try: + response = requests.get(MODULES_LIST_URL, timeout=10) + response.raise_for_status() + return response.content + except requests.RequestException as e: + print(f"::WARNING file={__file__},title=module_list_fetch_failed,col=0,endColumn=0,line=0::" + f"Could not fetch latest module-list.json ({e}); using committed copy instead.") + return None + def parse_macro(record): @@ -48,12 +66,6 @@ def parse_macro(record): return True -def count_msg(record): - msg_count[record.levelname] += 1 - - return True - - if __name__ == '__main__': # Github uses 'NOTICE' rather than 'INFO' # This should overwrite existing INFO level. @@ -66,6 +78,17 @@ def count_msg(record): sh.setFormatter(logging.Formatter( '::%(levelname)s file=%(filename)s,title=%(name)s,col=0,endColumn=0,line=%(lineno)s::%(message)s')) log.addHandler(sh) + + module_list = fetch_module_list() + tmp_module_list_path = None + if module_list is not None: + # mkdocs_hooks.py / macro_hooks.py read MODULE_LIST_PATH at import time, + # so it must be set before load_config() pulls those in. + fd, tmp_module_list_path = tempfile.mkstemp(suffix=".json", prefix="module-list-") + with os.fdopen(fd, "wb") as f: + f.write(module_list) + os.environ["MODULE_LIST_PATH"] = tmp_module_list_path + config = load_config(config_file_path="./mkdocs.yml") config.plugins.on_startup(command='build', dirty=True) try: @@ -75,6 +98,16 @@ def count_msg(record): sys.exit(1) finally: config.plugins.on_shutdown() + if tmp_module_list_path: + os.remove(tmp_module_list_path) + + if module_list is not None: + # Overwrite the stale copy mkdocs just copied from docs/assets/module-list.json, + # so the client-side app filter (supportedApplications.js) also sees fresh data. + site_module_list_path = os.path.join(config.site_dir, "assets", "module-list.json") + os.makedirs(os.path.dirname(site_module_list_path), exist_ok=True) + with open(site_module_list_path, "wb") as f: + f.write(module_list) time.sleep(5) # exit(100 < msg_count["NOTICE"] + (30 * msg_count["WARNING"] + (100 * msg_count["ERROR"]))) diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 530297c4d..499822f29 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -530,12 +530,17 @@ The macro plugin allows the use of 'includes', here is an example. {% endraw %} ``` +```{% raw %}{% include %}{% endraw %}``` fails soft in this project: if the included template errors while +rendering (e.g. an app is missing an expected field in module-list.json), that +include renders as nothing instead of taking out the whole page. No special +syntax needed - this is patched into both Jinja environments in `mkdocs_hooks.py`. + There are a few includes you may want to use. | Path | content | usage | | ---- | ------- | ----- | | ```{% raw %}{% include "partials/support_request.html" %}{% endraw %}``` | ```Contact our Support Team``` | Anywhere the user is told to contact support. | -| ```{% raw %}{% include "partials/appHeader.html" %}{% endraw %}``` | Info block | At the top of documents about particular software (TODO: elaborate) | +| ```{% raw %}{% include "partials/app_header.html" %}{% endraw %}``` | Info block | At the top of documents about particular software (TODO: elaborate) | | ```{% raw %}{% include "partials/app/app_network_licence.html" %}{% endraw %}``` | List of network licences | When dynamic licence info is required (used in `appHeader.html`) | | ```{% raw %}{% include "partials/app/app_version.html" %}{% endraw %}``` | List of versions and a 'module load' code-block. | When dynamic version info is required | diff --git a/macro_hooks.py b/macro_hooks.py index 364ceefd2..5c231ad01 100644 --- a/macro_hooks.py +++ b/macro_hooks.py @@ -5,6 +5,7 @@ If this is confusing, ask Cal to explain. """ +import importlib.util import os import json @@ -12,6 +13,25 @@ tag_index_path = os.getenv("TAG_INDEX_PATH", "docs/assets/tag-index.json") +def _load_mkdocs_hooks(): + """Load mkdocs_hooks.py by file path rather than `import mkdocs_hooks`. + + mkdocs and mkdocs-macros load their own hook module through different + mechanisms (mkdocs briefly patches sys.path and restores it; mkdocs-macros + execs this file directly by path), so a plain cross-file import can't + reliably resolve `mkdocs_hooks` regardless of how *this* file was loaded. + Resolving relative to our own (always-correct) __file__ sidesteps that. + """ + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mkdocs_hooks.py") + spec = importlib.util.spec_from_file_location("mkdocs_hooks", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +configure_safe_includes = _load_mkdocs_hooks().configure_safe_includes + + class CaseInsensitiveDict(dict): """Dict wrapper allowing `applications[app_name]` lookups regardless of case.""" @@ -59,3 +79,10 @@ def pages_with_tag(tag): {"title": e["title"], "path": os.path.relpath(e["path"], current_dir)} for e in entries ] + + +def on_pre_page_macros(env): + # env.env (the actual Jinja Environment) doesn't exist yet at define_env() + # time - on_config() creates it afterwards - so patch it here instead. + # configure_safe_includes() is idempotent, so re-running per page is cheap. + configure_safe_includes(env.env) diff --git a/mkdocs_hooks.py b/mkdocs_hooks.py index dd04856f0..55cc3c425 100644 --- a/mkdocs_hooks.py +++ b/mkdocs_hooks.py @@ -13,9 +13,80 @@ import re import yaml +from jinja2.compiler import CodeGenerator +from jinja2.exceptions import TemplateNotFound module_list_path = os.getenv("MODULE_LIST_PATH", "docs/assets/module-list.json") + +class SafeIncludeCodeGenerator(CodeGenerator): + """Makes every `{% include %}` in this environment render as nothing + instead of raising, if the included template errors while rendering. + + Partials driven by module-list.json data (versions, network licences, + ...) can fail on one app's unexpected/missing data. Without this, that + error propagates and takes out the *whole* page (mkdocs-macros' error + block) or the *whole* build (the supported-apps index renders every app + inline via the theme's own Jinja env, outside mkdocs-macros' handling). + + `{% extends %}` and `{% import %}` are untouched (separate visit_* + methods) - only `{% include %}` gets this treatment. + + Set as `environment.code_generator_class` (an official Jinja extension + point - see Environment.code_generator_class), it replaces the include's + normal streaming codegen with a call to `environment.safe_include()`, + which renders the target as one buffered string (so a mid-render failure + can't leak partial output) and swallows/logs any exception. + """ + + def visit_Include(self, node, frame): + self.writeline("yield environment.safe_include(", node) + self.visit(node.template, frame) + if node.with_context: + self.write(f", {{**context.get_all(), **{self.dump_local_context(frame)}}}") + else: + self.write(", {}") + self.write(f", {node.ignore_missing!r})") + + +def make_safe_include(jinja_env): + """Build the `environment.safe_include()` called by the codegen above. + + A closure (not a method) bound to one specific environment via + `env.safe_include = make_safe_include(env)`, since generated template + code looks it up as a plain attribute (`environment.safe_include(...)`) + rather than through the descriptor protocol. + """ + def safe_include(name, context_dict, ignore_missing=False): + try: + template = (jinja_env.select_template(name) + if isinstance(name, (list, tuple)) + else jinja_env.get_template(name)) + except TemplateNotFound: + if ignore_missing: + return "" + raise + try: + return template.render(context_dict) + except Exception as e: + print(f"::WARNING file={name},title=include_failed,col=0,endColumn=0,line=0::{e}") + return "" + return safe_include + + +def configure_safe_includes(jinja_env): + """Make every `{% include %}` rendered by `jinja_env` fail soft. + + Idempotent - safe to call more than once on the same environment (e.g. + macro_hooks.py's on_pre_page_macros(), which fires per page). + """ + if getattr(jinja_env, "_safe_includes_configured", False): + return + jinja_env.code_generator_class = SafeIncludeCodeGenerator + jinja_env.safe_include = make_safe_include(jinja_env) + jinja_env._safe_includes_configured = True + + _FRONT_MATTER = re.compile(r"\A---\s*\n(.*?)\n---\s*\n", re.DOTALL) @@ -90,6 +161,7 @@ def on_env(env, config, files, **kwargs): for app in applications.values() for domain in app.get("domains", []) }) + configure_safe_includes(env) def lint(*args, **kwargs): From d1bb1cbe905e9b364a81551483f44d7e721cc09a Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Wed, 12 Aug 2026 14:11:04 +1200 Subject: [PATCH 3/3] first --- docs/Batch_Computing/Job_Limits.md | 4 ++-- .../FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md | 2 +- docs/Storage/Data_Recovery.md | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/Batch_Computing/Job_Limits.md b/docs/Batch_Computing/Job_Limits.md index e97b959d8..ce02a3100 100644 --- a/docs/Batch_Computing/Job_Limits.md +++ b/docs/Batch_Computing/Job_Limits.md @@ -16,8 +16,8 @@ These are open for review if you find any of them unreasonable or inefficient. ![job limits](../assets/images/job_limits.png){ align=right width=75% } - 10 nodes -- 21 node-days -- 21 days +- 21 days walltime +- 21 node-days (walltime x nodes)
diff --git a/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md b/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md index 6e4cffe6d..13a4fac27 100644 --- a/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md +++ b/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md @@ -32,7 +32,7 @@ This option might be less convenient as the exporter saves the python file to your local computer, meaning you will have to drag it back into the file explorer in Jupyter from your downloads folder. -This script can then be run as a regular python script as described in +This file can then be run as a regular python script inside a Slurm job as described in our [Python](../../Software/Available_Applications/Python.md) documentation. diff --git a/docs/Storage/Data_Recovery.md b/docs/Storage/Data_Recovery.md index 328050c9f..d6c9ad897 100644 --- a/docs/Storage/Data_Recovery.md +++ b/docs/Storage/Data_Recovery.md @@ -5,9 +5,11 @@ tags: - storage --- +Deleted files may be retrieved from a snapshot. + Snapshots are read only copies of the filesystems at a point in time. They are taken daily (or weekly for `/nesi/nobackup`) and retained for -at least seven days. +at least seven days. Files from your home directory can be found in `/home/.snapshots/` followed by a snapshot timestamp and