From 35be9185db8220c1ab257a6e24f048048ab43201 Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Tue, 11 Aug 2026 13:34:28 +1200 Subject: [PATCH 1/6] Fix gtagging --- docs/assets/javascripts/chat.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/assets/javascripts/chat.js b/docs/assets/javascripts/chat.js index 418ce2b77..f79ecd3bf 100644 --- a/docs/assets/javascripts/chat.js +++ b/docs/assets/javascripts/chat.js @@ -9,6 +9,9 @@ let sending = false; let codeBlockSeq = 0; + window.dataLayer = window.dataLayer || []; + function gtag() { window.dataLayer.push(arguments); } + const ESCAPE_MAP = { "&": "&", "<": "<", @@ -260,9 +263,10 @@ { role: "user", content: question }, { role: "assistant", content: answer } ); - if (window.dataLayer) window.dataLayer.push(["event", "rag_chat", { question: question, answer: answer }]); history.splice(0, Math.max(0, history.length - 12)); + gtag("event", "rag_chat", { question: question, answer: answer }); + } catch (err) { botMsg.innerHTML = `

Error: ${escapeHtml(err.message)}

`; } finally { From 106d4dba626c0bddea970e1aa8a36f4bf06d34c7 Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Wed, 12 Aug 2026 13:41:21 +1200 Subject: [PATCH 2/6] first. (bad) --- checks/run_test_build.py | 47 ++++++++++++++++++++++---- docs/FORMAT.md | 7 +++- macro_hooks.py | 27 +++++++++++++++ mkdocs_hooks.py | 72 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 8 deletions(-) diff --git a/checks/run_test_build.py b/checks/run_test_build.py index 9cf81a800..9df4d8220 100755 --- a/checks/run_test_build.py +++ b/checks/run_test_build.py @@ -4,17 +4,35 @@ from mkdocs.commands import build, serve from mkdocs.config.base import Config, load_config import logging +import os import sys import re +import tempfile import time +import requests -""" + +""" This works but is a bit messy """ msg_count = {"DEBUG": 0, "NOTICE": 0, "WARNING": 0, "ERROR": 0} +MODULES_LIST_URL = "https://raw.githubusercontent.com/nesi/modules-list/main/module-list.json" + + +def fetch_module_list(): + """Fetch the latest module-list.json contents, or None if unavailable.""" + try: + response = requests.get(MODULES_LIST_URL, timeout=10) + response.raise_for_status() + return response.content + except requests.RequestException as e: + print(f"::WARNING file={__file__},title=module_list_fetch_failed,col=0,endColumn=0,line=0::" + f"Could not fetch latest module-list.json ({e}); using committed copy instead.") + return None + def parse_macro(record): @@ -48,12 +66,6 @@ def parse_macro(record): return True -def count_msg(record): - msg_count[record.levelname] += 1 - - return True - - if __name__ == '__main__': # Github uses 'NOTICE' rather than 'INFO' # This should overwrite existing INFO level. @@ -66,6 +78,17 @@ def count_msg(record): sh.setFormatter(logging.Formatter( '::%(levelname)s file=%(filename)s,title=%(name)s,col=0,endColumn=0,line=%(lineno)s::%(message)s')) log.addHandler(sh) + + module_list = fetch_module_list() + tmp_module_list_path = None + if module_list is not None: + # mkdocs_hooks.py / macro_hooks.py read MODULE_LIST_PATH at import time, + # so it must be set before load_config() pulls those in. + fd, tmp_module_list_path = tempfile.mkstemp(suffix=".json", prefix="module-list-") + with os.fdopen(fd, "wb") as f: + f.write(module_list) + os.environ["MODULE_LIST_PATH"] = tmp_module_list_path + config = load_config(config_file_path="./mkdocs.yml") config.plugins.on_startup(command='build', dirty=True) try: @@ -75,6 +98,16 @@ def count_msg(record): sys.exit(1) finally: config.plugins.on_shutdown() + if tmp_module_list_path: + os.remove(tmp_module_list_path) + + if module_list is not None: + # Overwrite the stale copy mkdocs just copied from docs/assets/module-list.json, + # so the client-side app filter (supportedApplications.js) also sees fresh data. + site_module_list_path = os.path.join(config.site_dir, "assets", "module-list.json") + os.makedirs(os.path.dirname(site_module_list_path), exist_ok=True) + with open(site_module_list_path, "wb") as f: + f.write(module_list) time.sleep(5) # exit(100 < msg_count["NOTICE"] + (30 * msg_count["WARNING"] + (100 * msg_count["ERROR"]))) diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 530297c4d..499822f29 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -530,12 +530,17 @@ The macro plugin allows the use of 'includes', here is an example. {% endraw %} ``` +```{% raw %}{% include %}{% endraw %}``` fails soft in this project: if the included template errors while +rendering (e.g. an app is missing an expected field in module-list.json), that +include renders as nothing instead of taking out the whole page. No special +syntax needed - this is patched into both Jinja environments in `mkdocs_hooks.py`. + There are a few includes you may want to use. | Path | content | usage | | ---- | ------- | ----- | | ```{% raw %}{% include "partials/support_request.html" %}{% endraw %}``` | ```Contact our Support Team``` | Anywhere the user is told to contact support. | -| ```{% raw %}{% include "partials/appHeader.html" %}{% endraw %}``` | Info block | At the top of documents about particular software (TODO: elaborate) | +| ```{% raw %}{% include "partials/app_header.html" %}{% endraw %}``` | Info block | At the top of documents about particular software (TODO: elaborate) | | ```{% raw %}{% include "partials/app/app_network_licence.html" %}{% endraw %}``` | List of network licences | When dynamic licence info is required (used in `appHeader.html`) | | ```{% raw %}{% include "partials/app/app_version.html" %}{% endraw %}``` | List of versions and a 'module load' code-block. | When dynamic version info is required | diff --git a/macro_hooks.py b/macro_hooks.py index 364ceefd2..5c231ad01 100644 --- a/macro_hooks.py +++ b/macro_hooks.py @@ -5,6 +5,7 @@ If this is confusing, ask Cal to explain. """ +import importlib.util import os import json @@ -12,6 +13,25 @@ tag_index_path = os.getenv("TAG_INDEX_PATH", "docs/assets/tag-index.json") +def _load_mkdocs_hooks(): + """Load mkdocs_hooks.py by file path rather than `import mkdocs_hooks`. + + mkdocs and mkdocs-macros load their own hook module through different + mechanisms (mkdocs briefly patches sys.path and restores it; mkdocs-macros + execs this file directly by path), so a plain cross-file import can't + reliably resolve `mkdocs_hooks` regardless of how *this* file was loaded. + Resolving relative to our own (always-correct) __file__ sidesteps that. + """ + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mkdocs_hooks.py") + spec = importlib.util.spec_from_file_location("mkdocs_hooks", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +configure_safe_includes = _load_mkdocs_hooks().configure_safe_includes + + class CaseInsensitiveDict(dict): """Dict wrapper allowing `applications[app_name]` lookups regardless of case.""" @@ -59,3 +79,10 @@ def pages_with_tag(tag): {"title": e["title"], "path": os.path.relpath(e["path"], current_dir)} for e in entries ] + + +def on_pre_page_macros(env): + # env.env (the actual Jinja Environment) doesn't exist yet at define_env() + # time - on_config() creates it afterwards - so patch it here instead. + # configure_safe_includes() is idempotent, so re-running per page is cheap. + configure_safe_includes(env.env) diff --git a/mkdocs_hooks.py b/mkdocs_hooks.py index dd04856f0..55cc3c425 100644 --- a/mkdocs_hooks.py +++ b/mkdocs_hooks.py @@ -13,9 +13,80 @@ import re import yaml +from jinja2.compiler import CodeGenerator +from jinja2.exceptions import TemplateNotFound module_list_path = os.getenv("MODULE_LIST_PATH", "docs/assets/module-list.json") + +class SafeIncludeCodeGenerator(CodeGenerator): + """Makes every `{% include %}` in this environment render as nothing + instead of raising, if the included template errors while rendering. + + Partials driven by module-list.json data (versions, network licences, + ...) can fail on one app's unexpected/missing data. Without this, that + error propagates and takes out the *whole* page (mkdocs-macros' error + block) or the *whole* build (the supported-apps index renders every app + inline via the theme's own Jinja env, outside mkdocs-macros' handling). + + `{% extends %}` and `{% import %}` are untouched (separate visit_* + methods) - only `{% include %}` gets this treatment. + + Set as `environment.code_generator_class` (an official Jinja extension + point - see Environment.code_generator_class), it replaces the include's + normal streaming codegen with a call to `environment.safe_include()`, + which renders the target as one buffered string (so a mid-render failure + can't leak partial output) and swallows/logs any exception. + """ + + def visit_Include(self, node, frame): + self.writeline("yield environment.safe_include(", node) + self.visit(node.template, frame) + if node.with_context: + self.write(f", {{**context.get_all(), **{self.dump_local_context(frame)}}}") + else: + self.write(", {}") + self.write(f", {node.ignore_missing!r})") + + +def make_safe_include(jinja_env): + """Build the `environment.safe_include()` called by the codegen above. + + A closure (not a method) bound to one specific environment via + `env.safe_include = make_safe_include(env)`, since generated template + code looks it up as a plain attribute (`environment.safe_include(...)`) + rather than through the descriptor protocol. + """ + def safe_include(name, context_dict, ignore_missing=False): + try: + template = (jinja_env.select_template(name) + if isinstance(name, (list, tuple)) + else jinja_env.get_template(name)) + except TemplateNotFound: + if ignore_missing: + return "" + raise + try: + return template.render(context_dict) + except Exception as e: + print(f"::WARNING file={name},title=include_failed,col=0,endColumn=0,line=0::{e}") + return "" + return safe_include + + +def configure_safe_includes(jinja_env): + """Make every `{% include %}` rendered by `jinja_env` fail soft. + + Idempotent - safe to call more than once on the same environment (e.g. + macro_hooks.py's on_pre_page_macros(), which fires per page). + """ + if getattr(jinja_env, "_safe_includes_configured", False): + return + jinja_env.code_generator_class = SafeIncludeCodeGenerator + jinja_env.safe_include = make_safe_include(jinja_env) + jinja_env._safe_includes_configured = True + + _FRONT_MATTER = re.compile(r"\A---\s*\n(.*?)\n---\s*\n", re.DOTALL) @@ -90,6 +161,7 @@ def on_env(env, config, files, **kwargs): for app in applications.values() for domain in app.get("domains", []) }) + configure_safe_includes(env) def lint(*args, **kwargs): From d1bb1cbe905e9b364a81551483f44d7e721cc09a Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Wed, 12 Aug 2026 14:11:04 +1200 Subject: [PATCH 3/6] first --- docs/Batch_Computing/Job_Limits.md | 4 ++-- .../FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md | 2 +- docs/Storage/Data_Recovery.md | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/Batch_Computing/Job_Limits.md b/docs/Batch_Computing/Job_Limits.md index e97b959d8..ce02a3100 100644 --- a/docs/Batch_Computing/Job_Limits.md +++ b/docs/Batch_Computing/Job_Limits.md @@ -16,8 +16,8 @@ These are open for review if you find any of them unreasonable or inefficient. ![job limits](../assets/images/job_limits.png){ align=right width=75% } - 10 nodes -- 21 node-days -- 21 days +- 21 days walltime +- 21 node-days (walltime x nodes)
diff --git a/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md b/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md index 6e4cffe6d..13a4fac27 100644 --- a/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md +++ b/docs/Getting_Started/FAQs/How_Do_I_Run_My_Python_Notebook_Through_SLURM.md @@ -32,7 +32,7 @@ This option might be less convenient as the exporter saves the python file to your local computer, meaning you will have to drag it back into the file explorer in Jupyter from your downloads folder. -This script can then be run as a regular python script as described in +This file can then be run as a regular python script inside a Slurm job as described in our [Python](../../Software/Available_Applications/Python.md) documentation. diff --git a/docs/Storage/Data_Recovery.md b/docs/Storage/Data_Recovery.md index 328050c9f..d6c9ad897 100644 --- a/docs/Storage/Data_Recovery.md +++ b/docs/Storage/Data_Recovery.md @@ -5,9 +5,11 @@ tags: - storage --- +Deleted files may be retrieved from a snapshot. + Snapshots are read only copies of the filesystems at a point in time. They are taken daily (or weekly for `/nesi/nobackup`) and retained for -at least seven days. +at least seven days. Files from your home directory can be found in `/home/.snapshots/` followed by a snapshot timestamp and From df610e31c3cded1de672988e8ebb0925e1877687 Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Wed, 12 Aug 2026 14:26:58 +1200 Subject: [PATCH 4/6] remove 40gb --- docs/Batch_Computing/Hardware.md | 12 ++++++------ docs/Batch_Computing/Using_GPUs.md | 29 ++++++++++++++++------------- docs/Storage/Models.md | 15 +++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/docs/Batch_Computing/Hardware.md b/docs/Batch_Computing/Hardware.md index ab4af905e..e4abb9aea 100644 --- a/docs/Batch_Computing/Hardware.md +++ b/docs/Batch_Computing/Hardware.md @@ -53,10 +53,10 @@ You will always get the amount of memory you requested, even if running on a nod - 44 - + 716GB (2GB / Core) - 2 x NVIDIA A100 + 2 x NVIDIA RTX 6000 4 @@ -93,17 +93,17 @@ For information about how to request these GPUs in a Slurm job, see [Using GPUs] Nodes - NVIDIA A100 - + NVIDIA A100 + 80GB 4 Milan 4 - 40GB + 96GB 2 - Genoa + Genoa 4 diff --git a/docs/Batch_Computing/Using_GPUs.md b/docs/Batch_Computing/Using_GPUs.md index f28f7f752..bc9b93bdd 100644 --- a/docs/Batch_Computing/Using_GPUs.md +++ b/docs/Batch_Computing/Using_GPUs.md @@ -48,16 +48,18 @@ where `` is the type of gpu you want to use (either 'h100', 'a100', or Slurm Header - NVIDIA A100 - + NVIDIA A100 + 80GB 4 - 
#SBATCH --partition=milan
#SBATCH --gpus-per-node=a100:1
+ 
#SBATCH --gpus-per-node=a100:1
- - 40GB + + RTX 6000 + + 96GB 2 - 
#SBATCH --partition=genoa
#SBATCH --gpus-per-node=a100:1
+ 
#SBATCH --gpus-per-node=rtx6000:1
NVIDIA H100 @@ -80,7 +82,7 @@ You can also use the `--gpus-per-node`option in with the `srun` and `salloc` commands. For example: ``` sh -srun --job-name "InteractiveGPU" --gpus-per-node L4:1 --partition genoa --cpus-per-task 8 --mem 2GB --time 00:30:00 --pty bash +srun --job-name "InteractiveGPU" --gpus-per-node L4:1 --cpus-per-task 8 --mem 2GB --time 00:30:00 --pty bash ``` will request and then start a bash session with access to a L4 GPU, for a @@ -159,7 +161,6 @@ GPU: #SBATCH --job-name GPUJob # job name (shows up in the queue) #SBATCH --account nesi99991 # Your account #SBATCH --time 00-00:10:00 # Walltime (DD-HH:MM:SS) -#SBATCH --partition genoa # This means the job will land on A100 with 40GB VRAM #SBATCH --gpus-per-node A100:1 # GPU resources required per node #SBATCH --cpus-per-task 2 # number of CPUs per task (1 by default) #SBATCH --mem 512MB # amount of memory per node (1 by default) @@ -300,11 +301,13 @@ The following flow diagram explains the steps you should take to test which GPU When running a 15-minute test job, add the following settings in your Slurm submission script: ```sl -#SBATCH --time=00:15:00 -#SBATCH --gpu-per-node=:1 -#SBATCH --qos=debug -#SBATCH --profile=task # Only for testing -#SBATCH --acctg-freq=1 # Only for testing +#!/bin/bash -e + +#SBATCH --time 00:15:00 +#SBATCH --gpu-per-node :1 +#SBATCH --qos debug +#SBATCH --profile task # Only for testing +#SBATCH --acctg-freq 1 # Only for testing ``` To record the GPU utilisation and GPU memory, see [Measuring GPU efficiency after a job has finished](./Using_GPUs.md#measuring-gpu-efficiency-after-a-job-has-finished) for more information. diff --git a/docs/Storage/Models.md b/docs/Storage/Models.md index 6e2f497d9..60266f0ef 100644 --- a/docs/Storage/Models.md +++ b/docs/Storage/Models.md @@ -31,8 +31,7 @@ If you need a model that is not listed here, please {% include "partials/support 
/opt/nesi/model/gguf/llama3.1/llama3.1-70b.gguf
-
#SBATCH --partition=milan
-#SBATCH --gpus-per-node=a100:1
+
#SBATCH --gpus-per-node=a100:1
DeepSeek-R1 @@ -42,13 +41,11 @@ If you need a model that is not listed here, please {% include "partials/support 
/opt/nesi/model/gguf/deepseek-r1/deepseek-r1-32b.gguf
-
#SBATCH --partition=genoa
-#SBATCH --gpus-per-node=a100:1
+
#SBATCH --gpus-per-node=a100:1
/opt/nesi/model/gguf/deepseek-r1/deepseek-r1-70b.gguf
-
#SBATCH --partition=milan
-#SBATCH --gpus-per-node=a100:1
+
#SBATCH --gpus-per-node=a100:1
Qwen3 @@ -58,8 +55,7 @@ If you need a model that is not listed here, please {% include "partials/support 
/opt/nesi/model/gguf/qwen3/qwen3-32b.gguf
-
#SBATCH --partition=genoa
-#SBATCH --gpus-per-node=a100:1
+
#SBATCH --gpus-per-node=a100:1
Qwen2.5 @@ -75,8 +71,7 @@ If you need a model that is not listed here, please {% include "partials/support Gemma 3 Gemma 
/opt/nesi/model/gguf/gemma3/gemma3-27b.gguf
-
#SBATCH --partition=genoa
-#SBATCH --gpus-per-node=a100:1
+
#SBATCH --gpus-per-node=a100:1
From ffcb3de6cbac254bd1f0649a67f87e61488d5395 Mon Sep 17 00:00:00 2001 From: nesi-mkdocs-bot Date: Wed, 12 Aug 2026 14:30:11 +1200 Subject: [PATCH 5/6] oopsie --- checks/run_test_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/checks/run_test_build.py b/checks/run_test_build.py index 9df4d8220..b0b700e03 100755 --- a/checks/run_test_build.py +++ b/checks/run_test_build.py @@ -74,7 +74,6 @@ def parse_macro(record): log.setLevel(logging.INFO) sh = logging.StreamHandler(sys.stdout) sh.addFilter(parse_macro) - sh.addFilter(count_msg) sh.setFormatter(logging.Formatter( '::%(levelname)s file=%(filename)s,title=%(name)s,col=0,endColumn=0,line=%(lineno)s::%(message)s')) log.addHandler(sh) From b7c06f2a1b8d343fbd20e41365db086f19e706f4 Mon Sep 17 00:00:00 2001 From: Andre Geldenhuis Date: Thu, 13 Aug 2026 11:05:03 +1200 Subject: [PATCH 6/6] Update Hardware.md (#1376) Update GPU info, update some threads to cores, adjust core/mem ratios. Use installed memory for now, though Peter suggests using what sinfo reports and rounding it. Signed-off-by: Andre Geldenhuis --- docs/Batch_Computing/Hardware.md | 67 +++++++++++++++++++------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/docs/Batch_Computing/Hardware.md b/docs/Batch_Computing/Hardware.md index b1b6664e3..9277c9e8b 100644 --- a/docs/Batch_Computing/Hardware.md +++ b/docs/Batch_Computing/Hardware.md @@ -22,51 +22,55 @@ You will always get the amount of memory you requested, even if running on a nod - + - - + + - + - - + + + + + + - - - - + + + + - - - + + + - - + + - + @@ -74,24 +78,29 @@ You will always get the amount of memory you requested, even if running on a nod - - - - + + - + - + + - +
ArchitectureCoreCores Memory GPU Nodes
2 x AMD Milan 7713 CPU
└ 8 x Chiplets
    └ 8 x Cores		1262 x AMD Milan 7713 CPU
└ 8 x Chiplets
    └ 8 x Cores		128 512GB (4GB / Core) -5455
1024GB(8GB / Core)1024GB(8GB / Core) - 8
1 x AMD Milan 7713P CPU
└ 8 x Chiplets
    └ 8 x Cores		64512GB(8GB / Core) 4 x NVIDIA HGX A100 4
2 x AMD Genoa 9634 CPU
└ 12 x Chiplets
    └ 7 x Cores		166358GB(1GB / Core)2 x AMD Genoa 9634 CPU
└ 12 x Chiplets
    └ 7 x Cores		168384GB(2GB / Core) - 44
716GB(2GB / Core)2 x NVIDIA RTX 6000768GB(4GB / Core)2 x NVIDIA RTX PRO 6000 4
1432GB(4GB / Core)1536GB(8GB / Core) - 8
2 x NVIDIA H1002 x NVIDIA H100 NVL 4
4
Intel Xeon Gold802 x Intel Xeon Gold 6230 CPU
    (Cascade Lake)		40 1.5TB(18GB / Core)(38GB / Core) - 2
1764 x Intel Xeon Gold 6238M CPU
    (Cascade Lake)		88 6TB(346GB / Core)(69GB / Core) - 1
- + +!!! note "Memory figures" + Memory shown is the amount physically installed. A small amount is reserved for the operating system, + so the memory actually available to jobs is a few percent lower — for example a 512GB Milan node offers + 480GB to Slurm. A job requesting exactly the full per-core ratio across every core of a node will + therefore not fit. Run `sinfo -o '%n %m'` for the exact schedulable figures. + !!! warning "hugemem" Jobs will not automatically land on the Intel 'hugemem' nodes. You must specifically request `--partition hugemem`. The CPU architecture is different enough from the milan and genoa nodes, you will probably have to recompile your software. @@ -115,7 +124,7 @@ For information about how to request these GPUs in a Slurm job, see [Using GPUs] Nodes - NVIDIA A100 + NVIDIA A100 SXM4 80GB 4 @@ -123,15 +132,17 @@ For information about how to request these GPUs in a Slurm job, see [Using GPUs] 4 + NVIDIA RTX PRO 6000 + 96GB 2 - Genoa + Genoa 4 - NVIDIA H100 + NVIDIA H100 NVL - 96GB + 94GB 2 Genoa 4