From 8e702193b527c6fc87e658114315e52b2b218ed8 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Wed, 19 Aug 2026 10:03:57 +0200 Subject: [PATCH 01/54] add Scheduler ABC base methods --- src/osekit/utils/job.py | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/osekit/utils/job.py b/src/osekit/utils/job.py index edb78931f..03f569329 100644 --- a/src/osekit/utils/job.py +++ b/src/osekit/utils/job.py @@ -8,6 +8,7 @@ from __future__ import annotations import subprocess +from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Literal @@ -589,3 +590,56 @@ def submit_pbs( depend_on = dependencies[job.name] job.submit_pbs(dependency=depend_on) + + +class Scheduler(ABC): + """Abstract class representing a job scheduler.""" + + @abstractmethod + def write(self, job: Job, path: Path) -> None: + """Write a job script to file. + + Parameters + ---------- + job: Job + Job of which to write the script. + path: Path + Path of the file in which the job script is written. + + """ + ... + + @abstractmethod + def submit( + self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None + ) -> None: + """Submit the job to the scheduler. + + Parameters + ---------- + job: Job + Job to submit to the scheduler. + dependency: Job | list[Job] | str | None + Job dependency. Can be: + - A ``Job`` instance: will wait for that job to complete successfully + - A ``list[Job]``: will wait for all jobs to complete successfully + - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification + - ``None``: no dependency + + """ + + @abstractmethod + def update_info(self, job: Job) -> None: + """Request info about the job and update it.""" + ... + + @abstractmethod + def update_status(self, job: Job) -> None: + """Request info about the job and update its status. + + Returns + ------- + JobStatus: + The updated status of the job. + + """ From e3cdd948841152bfb14add9c5ba588b2d722ad61 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Wed, 19 Aug 2026 10:31:57 +0200 Subject: [PATCH 02/54] move job module to job package --- docs/source/job.rst | 2 +- docs/source/jobs.rst | 10 +++++----- src/osekit/job/__init__.py | 0 src/osekit/{utils => job}/job.py | 0 src/osekit/public/project.py | 2 +- tests/test_export_transform.py | 2 +- tests/test_job.py | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 src/osekit/job/__init__.py rename src/osekit/{utils => job}/job.py (100%) diff --git a/docs/source/job.rst b/docs/source/job.rst index 9cd6c3854..fe22917f9 100644 --- a/docs/source/job.rst +++ b/docs/source/job.rst @@ -3,5 +3,5 @@ Job --- -.. automodule:: osekit.utils.job +.. automodule:: osekit.job.job :members: JobConfig, JobBuilder, Job diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index ded25ee6f..2f1c3ddb0 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -7,17 +7,17 @@ through the PBS queuing system. This feature has mainly be thought for the Public API, but it can nonetheless be used for any Core API operation. -The job module is located at :mod:`osekit.utils.job`. +The job module is located at :mod:`osekit.job.job`. Public API ^^^^^^^^^^ -Running Public API Analyses through PBS jobs only requires adding a :class:`osekit.utils.job.JobBuilder` +Running Public API Analyses through PBS jobs only requires adding a :class:`osekit.job.job.JobBuilder` instance to the :attr:`osekit.public.project.Project.job_builder` attribute: .. code-block:: python - from osekit.utils.job import JobConfig, JobBuilder + from osekit.job.job import JobConfig, JobBuilder from osekit.public.project import Project project = Project(...) # See the Project documentation @@ -46,7 +46,7 @@ instance to the :attr:`osekit.public.project.Project.job_builder` attribute: Core API ^^^^^^^^ -Exporting Core API datasets with jobs is doable by explicitly instantiating a :class:`osekit.utils.job.Job` object. +Exporting Core API datasets with jobs is doable by explicitly instantiating a :class:`osekit.job.job.Job` object. The export parameters are specified in the ``script_args`` parameter of the ``Job`` constructor, and follow the console arguments of the :mod:`osekit.public.export` script. @@ -64,7 +64,7 @@ and follow the console arguments of the :mod:`osekit.public.export` script. # Some Public API imports are required from osekit.public.transform import OutputType - from osekit.utils.job import Job, JobConfig + from osekit.job.job import Job, JobConfig ads = AudioDataset(...) # See the AudioDataset doc sds = SpectroDataset(...) # See the SpectroDataset doc diff --git a/src/osekit/job/__init__.py b/src/osekit/job/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/osekit/utils/job.py b/src/osekit/job/job.py similarity index 100% rename from src/osekit/utils/job.py rename to src/osekit/job/job.py diff --git a/src/osekit/public/project.py b/src/osekit/public/project.py index 6a899a850..5361aa748 100644 --- a/src/osekit/public/project.py +++ b/src/osekit/public/project.py @@ -39,7 +39,7 @@ from pandas import Timestamp from osekit.core.audio_file import AudioFile - from osekit.utils.job import JobBuilder + from osekit.job.job import JobBuilder class Project: diff --git a/tests/test_export_transform.py b/tests/test_export_transform.py index f2de2a370..86e53f223 100644 --- a/tests/test_export_transform.py +++ b/tests/test_export_transform.py @@ -10,7 +10,7 @@ from osekit.core.audio_dataset import AudioDataset from osekit.public import export_transform from osekit.public.export_transform import create_parser -from osekit.utils.job import Job +from osekit.job.job import Job def test_parser_factory() -> None: diff --git a/tests/test_job.py b/tests/test_job.py index 766336631..611453383 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -8,8 +8,8 @@ import pytest from pandas import Timedelta -import osekit.utils.job as job_module -from osekit.utils.job import Job, JobBuilder, JobConfig, JobStatus +import osekit.job.job as job_module +from osekit.job.job import Job, JobBuilder, JobConfig, JobStatus @pytest.mark.parametrize( From 7fc25538b308487a0ee658a3ec9e20d17921568d Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Wed, 19 Aug 2026 11:11:03 +0200 Subject: [PATCH 03/54] move job classes in specific modules --- src/osekit/job/builder.py | 79 +++++++++++++++++ src/osekit/job/config.py | 36 ++++++++ src/osekit/job/job.py | 164 +----------------------------------- src/osekit/job/scheduler.py | 57 +++++++++++++ tests/test_job.py | 9 +- 5 files changed, 179 insertions(+), 166 deletions(-) create mode 100644 src/osekit/job/builder.py create mode 100644 src/osekit/job/config.py create mode 100644 src/osekit/job/scheduler.py diff --git a/src/osekit/job/builder.py b/src/osekit/job/builder.py new file mode 100644 index 000000000..d6b982c70 --- /dev/null +++ b/src/osekit/job/builder.py @@ -0,0 +1,79 @@ +from pathlib import Path + +from osekit.job.config import JobConfig +from osekit.job.job import Job, JobStatus + + +class JobBuilder: + """Class that should be attached to a Public API ``Project`` for working with jobs. + + If a ``Project`` has a ``JobBuilder``, it will use it to run transforms through jobs. + + """ + + def __init__(self, config: JobConfig = JobConfig) -> None: + """Initialize a ``JobBuilder`` instance. + + Parameters + ---------- + config: JobConfig + Config of the jobs built by this job builder. + + """ + self.config = config + self.jobs = [] + + def create_job( + self, + script_path: Path, + script_args: dict | None = None, + name: str = "osekit_transform", + output_folder: Path | None = None, + ) -> None: + """Create a new ``Job`` instance. + + Parameters + ---------- + script_path: Path + Path to the script file the job must run. + script_args: dict | None + Additional arguments to pass to the script file. + name: str + Name of the job. + output_folder: Path | None + Folder in which the output files (``.out`` and ``.err``) will be written. + + """ + job = Job( + script_path=script_path, + script_args=script_args, + name=name, + output_folder=output_folder, + config=self.config, + ) + job.write_pbs(output_folder / f"{name}.pbs") + self.jobs.append(job) + + def submit_pbs( + self, + dependencies: dict[str, Job | list[Job]] | None = None, + ) -> None: + """Submit all prepared jobs to the ``pbs`` queueing system. + + Parameters + ---------- + dependencies: dict[str, Job | list[Job]] | None + Optional dictionary mapping job names to their dependencies. + Example: ``{"job2": job1, "job3": [job1, job2]}`` + + """ + for job in self.jobs: + if job.update_status() is not JobStatus.PREPARED: + continue + + # Check if this job has dependencies + depend_on = None + if dependencies and job.name in dependencies: + depend_on = dependencies[job.name] + + job.submit_pbs(dependency=depend_on) diff --git a/src/osekit/job/config.py b/src/osekit/job/config.py new file mode 100644 index 000000000..8ed0182db --- /dev/null +++ b/src/osekit/job/config.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass +from typing import Literal + +from pandas import Timedelta + + +@dataclass +class JobConfig: + """Config used for creating a job. + + Parameters + ---------- + nb_nodes: int + Number of nodes on which the job runs. + ncpus: int + Number of total cores used per node. + ngpus: int | None + Number of total GPU used per node. + mem: str + Maximum amount of physical memory used by the job. + walltime: str | Timedelta + Maximum amount of real time during which the job can be running. + venv_name: str + Name (or path) of the conda virtual environment in which the job is running. + queue: Literal["omp", "mpi"] + Queue in which the job will be submitted. + + """ + + nb_nodes: int = 1 + ncpus: int = 2 + ngpus: int | None = None + mem: str = "8gb" + walltime: str | Timedelta = "01:00:00" + venv_name: str = "osekit" + queue: Literal["omp", "mpi"] = "omp" diff --git a/src/osekit/job/job.py b/src/osekit/job/job.py index 03f569329..9023fc025 100644 --- a/src/osekit/job/job.py +++ b/src/osekit/job/job.py @@ -8,13 +8,13 @@ from __future__ import annotations import subprocess -from abc import ABC, abstractmethod -from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Literal from pandas import Timedelta +from osekit.job.config import JobConfig + if TYPE_CHECKING: from pathlib import Path @@ -37,38 +37,6 @@ class JobStatus(Enum): COMPLETED = 5 -@dataclass -class JobConfig: - """Config used for creating a job. - - Parameters - ---------- - nb_nodes: int - Number of nodes on which the job runs. - ncpus: int - Number of total cores used per node. - ngpus: int | None - Number of total GPU used per node. - mem: str - Maximum amount of physical memory used by the job. - walltime: str | Timedelta - Maximum amount of real time during which the job can be running. - venv_name: str - Name (or path) of the conda virtual environment in which the job is running. - queue: Literal["omp", "mpi"] - Queue in which the job will be submitted. - - """ - - nb_nodes: int = 1 - ncpus: int = 2 - ngpus: int | None = None - mem: str = "8gb" - walltime: str | Timedelta = "01:00:00" - venv_name: str = "osekit" - queue: Literal["omp", "mpi"] = "omp" - - class Job: """Job that concerns a specific transform.""" @@ -515,131 +483,3 @@ def update_status(self) -> JobStatus: if self.job_info["job_state"] in job_state: self.status = job_state[self.job_info["job_state"]] return self.status - - -class JobBuilder: - """Class that should be attached to a Public API ``Project`` for working with jobs. - - If a ``Project`` has a ``JobBuilder``, it will use it to run transforms through jobs. - - """ - - def __init__(self, config: JobConfig = JobConfig) -> None: - """Initialize a ``JobBuilder`` instance. - - Parameters - ---------- - config: JobConfig - Config of the jobs built by this job builder. - - """ - self.config = config - self.jobs = [] - - def create_job( - self, - script_path: Path, - script_args: dict | None = None, - name: str = "osekit_transform", - output_folder: Path | None = None, - ) -> None: - """Create a new ``Job`` instance. - - Parameters - ---------- - script_path: Path - Path to the script file the job must run. - script_args: dict | None - Additional arguments to pass to the script file. - name: str - Name of the job. - output_folder: Path | None - Folder in which the output files (``.out`` and ``.err``) will be written. - - """ - job = Job( - script_path=script_path, - script_args=script_args, - name=name, - output_folder=output_folder, - config=self.config, - ) - job.write_pbs(output_folder / f"{name}.pbs") - self.jobs.append(job) - - def submit_pbs( - self, - dependencies: dict[str, Job | list[Job]] | None = None, - ) -> None: - """Submit all prepared jobs to the ``pbs`` queueing system. - - Parameters - ---------- - dependencies: dict[str, Job | list[Job]] | None - Optional dictionary mapping job names to their dependencies. - Example: ``{"job2": job1, "job3": [job1, job2]}`` - - """ - for job in self.jobs: - if job.update_status() is not JobStatus.PREPARED: - continue - - # Check if this job has dependencies - depend_on = None - if dependencies and job.name in dependencies: - depend_on = dependencies[job.name] - - job.submit_pbs(dependency=depend_on) - - -class Scheduler(ABC): - """Abstract class representing a job scheduler.""" - - @abstractmethod - def write(self, job: Job, path: Path) -> None: - """Write a job script to file. - - Parameters - ---------- - job: Job - Job of which to write the script. - path: Path - Path of the file in which the job script is written. - - """ - ... - - @abstractmethod - def submit( - self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None - ) -> None: - """Submit the job to the scheduler. - - Parameters - ---------- - job: Job - Job to submit to the scheduler. - dependency: Job | list[Job] | str | None - Job dependency. Can be: - - A ``Job`` instance: will wait for that job to complete successfully - - A ``list[Job]``: will wait for all jobs to complete successfully - - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification - - ``None``: no dependency - - """ - - @abstractmethod - def update_info(self, job: Job) -> None: - """Request info about the job and update it.""" - ... - - @abstractmethod - def update_status(self, job: Job) -> None: - """Request info about the job and update its status. - - Returns - ------- - JobStatus: - The updated status of the job. - - """ diff --git a/src/osekit/job/scheduler.py b/src/osekit/job/scheduler.py new file mode 100644 index 000000000..5d159b5ec --- /dev/null +++ b/src/osekit/job/scheduler.py @@ -0,0 +1,57 @@ +from abc import ABC, abstractmethod +from pathlib import Path + +from osekit.job.job import Job + + +class Scheduler(ABC): + """Abstract class representing a job scheduler.""" + + @abstractmethod + def write(self, job: Job, path: Path) -> None: + """Write a job script to file. + + Parameters + ---------- + job: Job + Job of which to write the script. + path: Path + Path of the file in which the job script is written. + + """ + ... + + @abstractmethod + def submit( + self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None + ) -> None: + """Submit the job to the scheduler. + + Parameters + ---------- + job: Job + Job to submit to the scheduler. + dependency: Job | list[Job] | str | None + Job dependency. Can be: + - A ``Job`` instance: will wait for that job to complete successfully + - A ``list[Job]``: will wait for all jobs to complete successfully + - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification + - ``None``: no dependency + + """ + + @abstractmethod + def update_info(self, job: Job) -> None: + """Request info about the job and update it.""" + ... + + @abstractmethod + def update_status(self, job: Job) -> None: + """Request info about the job and update its status. + + Returns + ------- + JobStatus: + The updated status of the job. + + """ diff --git a/tests/test_job.py b/tests/test_job.py index 611453383..767332776 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -8,8 +8,9 @@ import pytest from pandas import Timedelta -import osekit.job.job as job_module -from osekit.job.job import Job, JobBuilder, JobConfig, JobStatus +from osekit.job.builder import JobBuilder +from osekit.job.config import JobConfig +from osekit.job.job import Job, JobStatus @pytest.mark.parametrize( @@ -374,7 +375,7 @@ def write_pbs(self, path: Path) -> None: self.path = path self.status = JobStatus.PREPARED - monkeypatch.setattr(job_module, "Job", DummyJob) + monkeypatch.setattr("osekit.job.builder.Job", DummyJob) job_config = JobConfig( nb_nodes=2, @@ -459,7 +460,7 @@ def submit_pbs(self, dependency=None) -> None: def update_status(self) -> JobStatus: return self.status - monkeypatch.setattr(job_module, "Job", DummyJob) + monkeypatch.setattr("osekit.job.job.Job", DummyJob) jobs = [ DummyJob(name="unprepared", status=JobStatus.UNPREPARED), From d1ee9da0c93bd1e76d1dc65de931d2e1456e1098 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Wed, 19 Aug 2026 18:06:30 +0200 Subject: [PATCH 04/54] adapt old PBS support to new scheduler package --- src/osekit/job/builder.py | 23 +- src/osekit/job/config.py | 4 - src/osekit/job/job.py | 257 +---------------- src/osekit/job/scheduler/__init__.py | 0 src/osekit/job/scheduler/pbs.py | 258 ++++++++++++++++++ src/osekit/job/scheduler/scheduler.py | 96 +++++++ .../job/{scheduler.py => scheduler/slurm.py} | 18 +- tests/test_export_transform.py | 2 +- tests/test_job.py | 126 +++++---- 9 files changed, 459 insertions(+), 325 deletions(-) create mode 100644 src/osekit/job/scheduler/__init__.py create mode 100644 src/osekit/job/scheduler/pbs.py create mode 100644 src/osekit/job/scheduler/scheduler.py rename src/osekit/job/{scheduler.py => scheduler/slurm.py} (84%) diff --git a/src/osekit/job/builder.py b/src/osekit/job/builder.py index d6b982c70..ed8ea65c5 100644 --- a/src/osekit/job/builder.py +++ b/src/osekit/job/builder.py @@ -2,6 +2,8 @@ from osekit.job.config import JobConfig from osekit.job.job import Job, JobStatus +from osekit.job.scheduler.pbs import Pbs +from osekit.job.scheduler.scheduler import Scheduler class JobBuilder: @@ -11,16 +13,21 @@ class JobBuilder: """ - def __init__(self, config: JobConfig = JobConfig) -> None: + def __init__( + self, config: JobConfig | None = None, scheduler: Scheduler | None = None + ) -> None: """Initialize a ``JobBuilder`` instance. Parameters ---------- config: JobConfig Config of the jobs built by this job builder. + scheduler: Scheduler + Scheduler used to format, write and submit jobs. """ - self.config = config + self.config = config or JobConfig() + self.scheduler = scheduler or Pbs() self.jobs = [] def create_job( @@ -51,14 +58,16 @@ def create_job( output_folder=output_folder, config=self.config, ) - job.write_pbs(output_folder / f"{name}.pbs") + self.scheduler.write( + job=job, path=output_folder / f"{name}.{self.scheduler.JOB_FILE_EXTENSION}" + ) self.jobs.append(job) - def submit_pbs( + def submit( self, dependencies: dict[str, Job | list[Job]] | None = None, ) -> None: - """Submit all prepared jobs to the ``pbs`` queueing system. + """Submit all prepared jobs to the scheduler system. Parameters ---------- @@ -68,7 +77,7 @@ def submit_pbs( """ for job in self.jobs: - if job.update_status() is not JobStatus.PREPARED: + if self.scheduler.update_status(job=job) is not JobStatus.PREPARED: continue # Check if this job has dependencies @@ -76,4 +85,4 @@ def submit_pbs( if dependencies and job.name in dependencies: depend_on = dependencies[job.name] - job.submit_pbs(dependency=depend_on) + self.scheduler.submit(job=job, dependency=depend_on) diff --git a/src/osekit/job/config.py b/src/osekit/job/config.py index 8ed0182db..94981fc36 100644 --- a/src/osekit/job/config.py +++ b/src/osekit/job/config.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Literal from pandas import Timedelta @@ -22,8 +21,6 @@ class JobConfig: Maximum amount of real time during which the job can be running. venv_name: str Name (or path) of the conda virtual environment in which the job is running. - queue: Literal["omp", "mpi"] - Queue in which the job will be submitted. """ @@ -33,4 +30,3 @@ class JobConfig: mem: str = "8gb" walltime: str | Timedelta = "01:00:00" venv_name: str = "osekit" - queue: Literal["omp", "mpi"] = "omp" diff --git a/src/osekit/job/job.py b/src/osekit/job/job.py index 9023fc025..a2176a2ad 100644 --- a/src/osekit/job/job.py +++ b/src/osekit/job/job.py @@ -7,9 +7,8 @@ from __future__ import annotations -import subprocess from enum import Enum -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING from pandas import Timedelta @@ -73,7 +72,6 @@ def __init__( self.mem = config.mem self.walltime = config.walltime self.venv_name = config.venv_name - self.queue = config.queue self.name = name self.output_folder = output_folder self._status = JobStatus.UNPREPARED @@ -118,7 +116,7 @@ def ncpus(self, ncpus: int) -> None: self._ncpus = ncpus @property - def ngpus(self) -> int: + def ngpus(self) -> int | None: """Number of total GPU used per node.""" return self._ngpus @@ -163,20 +161,6 @@ def venv_name(self) -> str: def venv_name(self, venv_name: str) -> None: self._venv_name = venv_name - @property - def venv_activate_script(self) -> str: - """Bash script used for activating the conda virtual environment.""" - return f". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate {self.venv_name}" - - @property - def queue(self) -> Literal["omp", "mpi"]: - """Queue in which the job will be submitted.""" - return self._queue - - @queue.setter - def queue(self, queue: Literal["omp", "mpi"]) -> None: - self._queue = queue - @property def name(self) -> str: """Name of the job.""" @@ -222,17 +206,17 @@ def output_folder(self, output_folder: Path | None) -> None: self._output_folder = output_folder @property - def job_id(self) -> str: + def job_id(self) -> str | None: """Job ID.""" return self._id @job_id.setter - def job_id(self, job_id: str) -> None: + def job_id(self, job_id: str | None) -> None: self._id = job_id @property def job_info(self) -> dict | None: - """Information about the job as returned by a qstat request.""" + """Information about the job.""" return self._info @job_info.setter @@ -245,7 +229,7 @@ def progress(self) -> None: return self._status = JobStatus(self._status.value + 1) - def _build_arg_string(self) -> str: + def get_arg_string(self) -> str: """Build a string representation of the job's arguments.""" arg_list = [] for key, value in self.script_args.items(): @@ -254,232 +238,3 @@ def _build_arg_string(self) -> str: else: arg_list.append(f"--{key} {value}") return " ".join(arg_list) - - def write_pbs(self, path: Path) -> None: - """Write a ``pbs`` file matching the job. - - Parameters - ---------- - path: Path - Path of the ``pbs`` file to write. - - """ - preamble = "#!/bin/bash" - - select_parts = { - "select": self.nb_nodes, - "ncpus": self.ncpus, - "mem": self.mem, - } - if self.ngpus is not None: - select_parts["ngpus"] = self.ngpus - select_str = ":".join(f"{k}={v}" for k, v in select_parts.items()) - - request = { - "-N": self.name, - "-q": self.queue, - "-l": [ - select_str, - f"walltime={self.walltime_str}", - ], - "-o": f"{self.output_folder}/{self.name}.out" - if self.output_folder - else None, - "-e": f"{self.output_folder}/{self.name}.err" - if self.output_folder - else None, - } - request_str = "\n".join( - f"#PBS {key} {value}" - if type(value) is not list - else "\n".join(f"#PBS {key} {value_part}" for value_part in value) - for key, value in request.items() - if value - ) - - script = f"python {self.script_path} {self._build_arg_string()}" - - pbs = f"{preamble}\n{request_str}\n{self.venv_activate_script}\n{script}" - with path.open("w") as file: - file.write(pbs) - - self.path = path - self.progress() - - def submit_pbs( - self, - dependency: Job | list[Job] | str | list[str] | None = None, - ) -> None: - """Submit the ``pbs`` file of the job to a ``pbs`` queueing system. - - Parameters - ---------- - dependency: Job | list[Job] | str | None - Job dependency. Can be: - - A ``Job`` instance: will wait for that job to complete successfully - - A ``list[Job]``: will wait for all jobs to complete successfully - - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification - - ``None``: no dependency - - """ - if self.update_status() is not JobStatus.PREPARED: - msg = "Job should be written before being submitted." - raise ValueError(msg) - - cmd = ["qsub"] - - if dependency is not None: - dependency_str = self._build_dependency_string(dependency) - if dependency_str: - cmd.extend(["-W", f"depend={dependency_str}"]) - - cmd.append(str(self.path)) - - try: - request = subprocess.run( - cmd, - capture_output=True, - text=True, - check=False, - ) - except subprocess.CalledProcessError as e: - msg = f"Submission failed with exit code {e.returncode}" - raise RuntimeError(msg) from e - - self.job_id = request.stdout.split(".", maxsplit=1)[0].strip() - self.update_status() - - _VALID_DEPENDENCY_TYPES = {"afterok", "afterany", "afternotok", "after"} - - @staticmethod - def _validate_dependency_type(dependency_type: str) -> None: - if dependency_type not in Job._VALID_DEPENDENCY_TYPES: - msg = ( - f"Unsupported dependency type '{dependency_type}'. " - f"Expected one of {sorted(Job._VALID_DEPENDENCY_TYPES)}." - ) - raise ValueError(msg) - - @staticmethod - def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: - job_ids = [dep.job_id if isinstance(dep, Job) else dep for dep in dependency] - job_id_length = 7 - for job_id in job_ids: - if not job_id.isdigit() or len(job_id) != job_id_length: - msg = ( - f"Invalid job ID '{job_id}'. " - f"Job IDs must be {job_id_length} digits long." - ) - raise ValueError(msg) - return job_ids - - @staticmethod - def _build_dependency_string( - dependency: str | Job | list[str] | list[Job], - dependency_type: str = "afterok", - ) -> str: - """Build a PBS dependency string. - - Parameters - ---------- - dependency: Job | str - ``Job`` or job ID to depend on. - dependency_type: str - Type of dependency (``afterok``, ``afterany``, ``afternotok``, ``after``). - - Returns - ------- - str - PBS dependency string. - - Examples - -------- - >>> Job._build_dependency_string("1234567") - 'afterok:1234567' - >>> Job._build_dependency_string(["1234567", "4567891"]) - 'afterok:1234567:4567891' - >>> Job._build_dependency_string("7894561", dependency_type="afterany") - 'afterany:7894651' - - """ - dependency = dependency if isinstance(dependency, list) else [dependency] - id_str = Job._validate_dependency(dependency) - Job._validate_dependency_type(dependency_type) - - if unsubmitted_job := next( - ( - j - for j in dependency - if isinstance(j, Job) and j.status.value < JobStatus.QUEUED.value - ), - None, - ): - msg = f"Job '{unsubmitted_job.name}' has not been submitted yet." - raise ValueError(msg) - - return f"{dependency_type}:{':'.join(id_str)}" - - def update_info(self) -> None: - """Request info about the job and update it.""" - if self.job_id is None: - return - - try: - request = subprocess.run( - ["qstat", "-f", self.job_id], - capture_output=True, - text=True, - check=False, - ) - stdout = request.stdout - except subprocess.CalledProcessError as e: - msg = f"Qstat failed with exit code {e.returncode}" - raise RuntimeError(msg) from e - - if not stdout: - err = request.stderr - if "Job has finished" in err: - self.status = JobStatus.COMPLETED - self.job_info["job_state"] = "C" - if "Unknown Job Id" in err: - msg = f"Unknown Job Id {self.job_id}" - raise ValueError(msg) - return - - info = {} - for line in stdout.splitlines(): - if "=" not in line: - continue - key, value = line.split("=", 1) - info[key.strip()] = value.strip() - self.job_info = info - - def update_status(self) -> JobStatus: - """Request info about the job and update its status. - - Returns - ------- - JobStatus: - The updated status of the job. - - """ - if self.job_id is None: - self.status = ( - JobStatus.PREPARED - if self.path and self.path.exists() - else JobStatus.UNPREPARED - ) - return self.status - - self.update_info() - - if self.status == JobStatus.COMPLETED: - return self.status - - job_state = { - "Q": JobStatus.QUEUED, - "R": JobStatus.RUNNING, - } - if self.job_info["job_state"] in job_state: - self.status = job_state[self.job_info["job_state"]] - return self.status diff --git a/src/osekit/job/scheduler/__init__.py b/src/osekit/job/scheduler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py new file mode 100644 index 000000000..b8ef76a59 --- /dev/null +++ b/src/osekit/job/scheduler/pbs.py @@ -0,0 +1,258 @@ +import subprocess +from pathlib import Path +from typing import Literal + +from osekit.job.job import Job, JobStatus +from osekit.job.scheduler.scheduler import Scheduler + + +class Pbs(Scheduler): + """Abstract class representing a job scheduler.""" + + _VALID_DEPENDENCY_TYPES = frozenset({"afterok", "afterany", "afternotok", "after"}) + JOB_FILE_EXTENSION = "pbs" + + def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: + """Initialize the PBS scheduler.""" + self.queue = queue + + @property + def queue(self) -> str: + """Queue in which the job will be submitted.""" + return self._queue + + @queue.setter + def queue(self, queue: Literal["omp", "mpi"]) -> None: + self._queue = queue + + def write(self, job: Job, path: Path) -> None: + """Write a job script to file. + + Parameters + ---------- + job: Job + Job of which to write the script. + path: Path + Path of the file in which the job script is written. + + """ + preamble = "#!/bin/bash" + + select_parts = { + "select": job.nb_nodes, + "ncpus": job.ncpus, + "mem": job.mem, + } + if job.ngpus is not None: + select_parts["ngpus"] = job.ngpus + select_str = ":".join(f"{k}={v}" for k, v in select_parts.items()) + + request = { + "-N": job.name, + "-q": self.queue, + "-l": [ + select_str, + f"walltime={job.walltime_str}", + ], + "-o": f"{job.output_folder}/{job.name}.out" if job.output_folder else None, + "-e": f"{job.output_folder}/{job.name}.err" if job.output_folder else None, + } + request_str = "\n".join( + f"#PBS {key} {value}" + if type(value) is not list + else "\n".join(f"#PBS {key} {value_part}" for value_part in value) + for key, value in request.items() + if value + ) + + script = f"python {job.script_path} {job.get_arg_string()}" + + pbs = f"{preamble}\n{request_str}\n{self._build_venv_string(job=job)}\n{script}" + with path.open("w") as file: + file.write(pbs) + + job.path = path + job.progress() + + def submit( + self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None + ) -> None: + """Submit the job to the scheduler. + + Parameters + ---------- + job: Job + Job to submit to the scheduler. + dependency: Job | list[Job] | str | None + Job dependency. Can be: + - A ``Job`` instance: will wait for that job to complete successfully + - A ``list[Job]``: will wait for all jobs to complete successfully + - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification + - ``None``: no dependency + + """ + if self.update_status(job=job) is not JobStatus.PREPARED: + msg = "Job should be written before being submitted." + raise ValueError(msg) + + cmd = ["qsub"] + + if dependency is not None: + dependency_str = self._build_dependency_string(dependency) + if dependency_str: + cmd.extend(["-W", f"depend={dependency_str}"]) + + cmd.append(str(job.path)) + + try: + request = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + ) + except subprocess.CalledProcessError as e: + msg = f"Submission failed with exit code {e.returncode}" + raise RuntimeError(msg) from e + + job.job_id = request.stdout.split(".", maxsplit=1)[0].strip() + self.update_status(job=job) + + def update_info(self, job: Job) -> None: + """Request info about the job and update it.""" + if job.job_id is None: + return + + try: + request = subprocess.run( + ["qstat", "-f", str(job.job_id)], + capture_output=True, + text=True, + check=False, + ) + stdout = request.stdout + except subprocess.CalledProcessError as e: + msg = f"Qstat failed with exit code {e.returncode}" + raise RuntimeError(msg) from e + + if not stdout: + err = request.stderr + if "Job has finished" in err: + job.status = JobStatus.COMPLETED + job.job_info["job_state"] = "C" + if "Unknown Job Id" in err: + msg = f"Unknown Job Id {job.job_id}" + raise ValueError(msg) + return + + info = {} + for line in stdout.splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + info[key.strip()] = value.strip() + job.job_info = info + + def update_status(self, job: Job) -> JobStatus: + """Request info about the job and update its status. + + Returns + ------- + JobStatus: + The updated status of the job. + + """ + if job.job_id is None: + job.status = ( + JobStatus.PREPARED + if job.path and job.path.exists() + else JobStatus.UNPREPARED + ) + return job.status + + self.update_info(job=job) + + if job.status == JobStatus.COMPLETED: + return job.status + + job_state = { + "Q": JobStatus.QUEUED, + "R": JobStatus.RUNNING, + } + if job.job_info["job_state"] in job_state: + job.status = job_state[job.job_info["job_state"]] + return job.status + + @staticmethod + def _build_venv_string(job: Job) -> str: + """Bash script used for activating the conda virtual environment.""" + return f". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate {job.venv_name}" + + @classmethod + def _validate_dependency_type(cls, dependency_type: str) -> None: + if dependency_type not in cls._VALID_DEPENDENCY_TYPES: + msg = ( + f"Unsupported dependency type '{dependency_type}'. " + f"Expected one of {cls._VALID_DEPENDENCY_TYPES}." + ) + raise ValueError(msg) + + @staticmethod + def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: + job_ids = [dep.job_id if isinstance(dep, Job) else dep for dep in dependency] + job_id_length = 7 + for job_id in job_ids: + if not job_id.isdigit() or len(job_id) != job_id_length: + msg = ( + f"Invalid job ID '{job_id}'. " + f"Job IDs must be {job_id_length} digits long." + ) + raise ValueError(msg) + return job_ids + + @classmethod + def _build_dependency_string( + cls, + dependency: str | Job | list[str] | list[Job], + dependency_type: str = "afterok", + ) -> str: + """Build a PBS dependency string. + + Parameters + ---------- + dependency: Job | str + ``Job`` or job ID to depend on. + dependency_type: str + Type of dependency (``afterok``, ``afterany``, ``afternotok``, ``after``). + + Returns + ------- + str + PBS dependency string. + + Examples + -------- + >>> Pbs._build_dependency_string("1234567") + 'afterok:1234567' + >>> Pbs._build_dependency_string(["1234567", "4567891"]) + 'afterok:1234567:4567891' + >>> Pbs._build_dependency_string("7894561", dependency_type="afterany") + 'afterany:7894651' + + """ + dependency = dependency if isinstance(dependency, list) else [dependency] + id_str = cls._validate_dependency(dependency=dependency) + cls._validate_dependency_type(dependency_type=dependency_type) + + if unsubmitted_job := next( + ( + j + for j in dependency + if isinstance(j, Job) and j.status.value < JobStatus.QUEUED.value + ), + None, + ): + msg = f"Job '{unsubmitted_job.name}' has not been submitted yet." + raise ValueError(msg) + + return f"{dependency_type}:{':'.join(id_str)}" diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py new file mode 100644 index 000000000..b963c65af --- /dev/null +++ b/src/osekit/job/scheduler/scheduler.py @@ -0,0 +1,96 @@ +from abc import ABC, abstractmethod +from pathlib import Path + +from osekit.job.job import Job, JobStatus + + +class Scheduler(ABC): + """Abstract class representing a job scheduler.""" + + JOB_FILE_EXTENSION = "job" + + @abstractmethod + def write(self, job: Job, path: Path) -> None: + """Write a job script to file. + + Parameters + ---------- + job: Job + Job of which to write the script. + path: Path + Path of the file in which the job script is written. + + """ + ... + + @abstractmethod + def submit( + self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None + ) -> None: + """Submit the job to the scheduler. + + Parameters + ---------- + job: Job + Job to submit to the scheduler. + dependency: Job | list[Job] | str | None + Job dependency. Can be: + - A ``Job`` instance: will wait for that job to complete successfully + - A ``list[Job]``: will wait for all jobs to complete successfully + - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification + - ``None``: no dependency + + """ + ... + + @abstractmethod + def update_info(self, job: Job) -> None: + """Request info about the job and update it.""" + ... + + @abstractmethod + def update_status(self, job: Job) -> JobStatus: + """Request info about the job and update its status. + + Returns + ------- + JobStatus: + The updated status of the job. + + """ + ... + + @staticmethod + @abstractmethod + def _build_venv_string(job: Job) -> str: ... + + @classmethod + @abstractmethod + def _validate_dependency_type(cls, dependency_type: str) -> None: ... + + @staticmethod + @abstractmethod + def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: ... + + @classmethod + @abstractmethod + def _build_dependency_string( + cls, + dependency: str | Job | list[str] | list[Job], + dependency_type: str = "", + ) -> str: + """Build a job dependency string. + + Parameters + ---------- + dependency: Job | str + ``Job`` or job ID to depend on. + dependency_type: str + Type of dependency. + + Returns + ------- + str + Job dependency string. + """ + ... diff --git a/src/osekit/job/scheduler.py b/src/osekit/job/scheduler/slurm.py similarity index 84% rename from src/osekit/job/scheduler.py rename to src/osekit/job/scheduler/slurm.py index 5d159b5ec..31d8a2b5f 100644 --- a/src/osekit/job/scheduler.py +++ b/src/osekit/job/scheduler/slurm.py @@ -1,13 +1,12 @@ -from abc import ABC, abstractmethod from pathlib import Path -from osekit.job.job import Job +from osekit.job.job import Job, JobStatus +from osekit.job.scheduler.scheduler import Scheduler -class Scheduler(ABC): +class Slurm(Scheduler): """Abstract class representing a job scheduler.""" - @abstractmethod def write(self, job: Job, path: Path) -> None: """Write a job script to file. @@ -19,9 +18,8 @@ def write(self, job: Job, path: Path) -> None: Path of the file in which the job script is written. """ - ... + pass - @abstractmethod def submit( self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None ) -> None: @@ -39,14 +37,13 @@ def submit( - ``None``: no dependency """ + pass - @abstractmethod def update_info(self, job: Job) -> None: """Request info about the job and update it.""" - ... + pass - @abstractmethod - def update_status(self, job: Job) -> None: + def update_status(self, job: Job) -> JobStatus: """Request info about the job and update its status. Returns @@ -55,3 +52,4 @@ def update_status(self, job: Job) -> None: The updated status of the job. """ + pass diff --git a/tests/test_export_transform.py b/tests/test_export_transform.py index 86e53f223..0f979408b 100644 --- a/tests/test_export_transform.py +++ b/tests/test_export_transform.py @@ -98,7 +98,7 @@ def script_arguments() -> dict: def test_specified_arguments(script_arguments: dict) -> None: parser = create_parser() - parsed_str = Job(Path(), script_arguments)._build_arg_string() + parsed_str = Job(Path(), script_arguments).get_arg_string() args = parser.parse_args(shlex.split(parsed_str)) diff --git a/tests/test_job.py b/tests/test_job.py index 767332776..e71c0e700 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -11,6 +11,8 @@ from osekit.job.builder import JobBuilder from osekit.job.config import JobConfig from osekit.job.job import Job, JobStatus +from osekit.job.scheduler.pbs import Pbs +from osekit.job.scheduler.scheduler import Scheduler @pytest.mark.parametrize( @@ -47,7 +49,7 @@ def test_job_progress(initial_status: JobStatus, expected_status: JobStatus) -> assert job.status == expected_status -def test_properties_and_venv_activation() -> None: +def test_properties() -> None: script = Path("myscript.py") nb_nodes = 2 ncpus = 28 @@ -57,7 +59,6 @@ def test_properties_and_venv_activation() -> None: mem="16gb", walltime=Timedelta(hours=2), venv_name="merriweather", - queue="mpi", ) job = Job( script_path=script, @@ -76,15 +77,8 @@ def test_properties_and_venv_activation() -> None: assert job.walltime == Timedelta(hours=2) assert job.venv_name == "merriweather" assert job.name == "post_pavillion" - assert job.queue == "mpi" assert job.output_folder == Path("output") - # venv activation - expected = ( - ". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate merriweather" - ) - assert job.venv_activate_script == expected - def test_progress_transitions() -> None: job = Job(Path("strawberry.py")) @@ -122,7 +116,9 @@ def test_write_pbs(tmp_path: Path) -> None: output_folder=output_dir, ) pbs_path = tmp_path / "lafayette.pbs" - job.write_pbs(pbs_path) + + pbs_scheduler = Pbs(queue="omp") + pbs_scheduler.write(job=job, path=pbs_path) content = pbs_path.read_text().splitlines() assert content[0] == "#!/bin/bash" @@ -168,7 +164,8 @@ def test_write_pbs_job_with_gpu(tmp_path: Path) -> None: output_folder=output_dir, ) pbs_path = tmp_path / "patch.pbs" - job.write_pbs(pbs_path) + pbs_scheduler = Pbs(queue="omp") + pbs_scheduler.write(job=job, path=pbs_path) content = pbs_path.read_text().splitlines() assert any("select=1:ncpus=2:mem=8gb:ngpus=2" in line for line in content) @@ -179,11 +176,12 @@ def test_write_pbs_job_with_gpu(tmp_path: Path) -> None: def test_submit_pbs_without_write_raises() -> None: job = Job(Path("script.py")) + pbs = Pbs(queue="omp") with pytest.raises( ValueError, match=r"Job should be written before being submitted.", ): - job.submit_pbs() + pbs.submit(job=job) def test_submit_pbs_success(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -192,7 +190,8 @@ def test_submit_pbs_success(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> outdir = tmp_path job = Job(script, name="amobishoproden", output_folder=outdir) pbs_path = tmp_path / "amobishoproden.pbs" - job.write_pbs(pbs_path) + pbs = Pbs() + pbs.write(job=job, path=pbs_path) class Dummy: def __init__(self) -> None: @@ -208,14 +207,14 @@ def __init__(self) -> None: updated_jobs = [] - def mock_update_status(self: Job) -> JobStatus: + def mock_update_status(self, job: Job) -> JobStatus: updated_jobs.append(job) return JobStatus.PREPARED - monkeypatch.setattr(Job, "update_status", mock_update_status) + monkeypatch.setattr(Pbs, "update_status", mock_update_status) assert job.status == JobStatus.PREPARED - job.submit_pbs() + pbs.submit(job=job) assert job.job_id == "35173" assert np.array_equal( @@ -230,7 +229,8 @@ def test_submit_pbs_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N outdir = tmp_path job = Job(script, name="amobishoproden", output_folder=outdir) pbs_path = tmp_path / "amobishoproden.pbs" - job.write_pbs(pbs_path) + pbs_scheduler = Pbs(queue="omp") + pbs_scheduler.write(job=job, path=pbs_path) class Dummy: def __init__(self) -> None: @@ -245,15 +245,16 @@ def __init__(self) -> None: assert job.status == JobStatus.PREPARED with pytest.raises(RuntimeError, match="Submission failed with exit code 5"): - job.submit_pbs() + pbs_scheduler.submit(job=job) assert job.status == JobStatus.PREPARED def test_update_info_no_job_id() -> None: job = Job(Path("pixies.py")) + pbs_scheduler = Pbs() job.job_id = None - job.update_info() + pbs_scheduler.update_info(job=job) assert job.job_info is None @@ -272,7 +273,8 @@ class Dummy: "run", lambda *args, **kwargs: Dummy(), ) - job.update_info() + scheduler = Pbs() + scheduler.update_info(job=job) assert job.job_info == {"frankie": "cosmos", "avey": "tare", "attic": "abasement"} @@ -291,7 +293,8 @@ class Dummy: lambda *args, **kwargs: Dummy(), ) - job.update_info() + scheduler = Pbs() + scheduler.update_info(job=job) assert job.status == JobStatus.COMPLETED assert job.job_info["job_state"] == "C" @@ -310,8 +313,9 @@ class Dummy: lambda *args, **kwargs: Dummy(), ) + scheduler = Pbs() with pytest.raises(ValueError, match="Unknown Job Id 17112014"): - job.update_info() + scheduler.update_info(job=job) def test_update_info_error(monkeypatch: pytest.MonkeyPatch) -> None: @@ -328,40 +332,43 @@ def __init__(self) -> None: lambda *args, **kwargs: Dummy(), ) + scheduler = Pbs() with pytest.raises(RuntimeError, match="Qstat failed with exit code 5"): - job.update_info() + scheduler.update_info(job=job) def test_update_status(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("porticoquartet.py")) job.path = tmp_path / "pompidou.pbs" - assert job.update_status() == JobStatus.UNPREPARED + scheduler = Pbs() + + assert scheduler.update_status(job=job) == JobStatus.UNPREPARED job.path.write_text("prickly pear") - assert job.update_status() == JobStatus.PREPARED + assert scheduler.update_status(job=job) == JobStatus.PREPARED monkeypatch.setattr( - job, + scheduler, "update_info", - lambda: None, + lambda job: None, ) job.job_info = {"job_state": "Q"} job.job_id = "5129195" - assert job.update_status() == JobStatus.QUEUED + assert scheduler.update_status(job=job) == JobStatus.QUEUED assert job.status == JobStatus.QUEUED job.job_info = {"job_state": "R"} - assert job.update_status() == JobStatus.RUNNING + assert scheduler.update_status(job=job) == JobStatus.RUNNING assert job.status == JobStatus.RUNNING job.status = JobStatus.COMPLETED - assert job.update_status() == JobStatus.COMPLETED + assert scheduler.update_status(job=job) == JobStatus.COMPLETED assert job.status == JobStatus.COMPLETED -def test_job_builder_write(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_pbs_job_builder_write(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: called = {} class DummyJob: @@ -370,12 +377,13 @@ def __init__(self, **kwargs: dict) -> None: self.path = None self.status = JobStatus.UNPREPARED - def write_pbs(self, path: Path) -> None: - called["write_pbs"] = path - self.path = path - self.status = JobStatus.PREPARED + def mock_write(self: Pbs, job: Job, path: Path) -> None: + called["write_pbs"] = path + self.path = path + job.status = JobStatus.PREPARED monkeypatch.setattr("osekit.job.builder.Job", DummyJob) + monkeypatch.setattr(Pbs, "write", mock_write) job_config = JobConfig( nb_nodes=2, @@ -383,10 +391,9 @@ def write_pbs(self, path: Path) -> None: mem="60gb", walltime=Timedelta(hours=2), venv_name="abyssinie", - queue="mpi", ) - job_builder = JobBuilder(config=job_config) + job_builder = JobBuilder(scheduler=Pbs(), config=job_config) assert job_builder.jobs == [] @@ -441,7 +448,7 @@ def test_build_arg_string_booleans(tmp_path: Path): ) job = next(iter(job_builder.jobs)) - arg_str = job._build_arg_string() + arg_str = job.get_arg_string() assert arg_str == "--no-danser --avec --le 0.3 --vent test" @@ -454,13 +461,17 @@ def __init__(self, name: str, status: JobStatus) -> None: self.name = name self.status = status - def submit_pbs(self, dependency=None) -> None: - submitted_jobs.append((self.name, dependency)) + def mock_submit( + self: Scheduler, job: Job, dependency: Job | str | None = None + ) -> None: + submitted_jobs.append((job.name, dependency)) - def update_status(self) -> JobStatus: - return self.status + def mock_update_status(self: Scheduler, job: Job) -> JobStatus: + return job.status monkeypatch.setattr("osekit.job.job.Job", DummyJob) + monkeypatch.setattr(Pbs, "submit", mock_submit) + monkeypatch.setattr(Pbs, "update_status", mock_update_status) jobs = [ DummyJob(name="unprepared", status=JobStatus.UNPREPARED), @@ -475,7 +486,7 @@ def update_status(self) -> JobStatus: dependencies = {"prepared": jobs[0]} - job_builder.submit_pbs(dependencies=dependencies) + job_builder.submit(dependencies=dependencies) assert submitted_jobs == [("prepared", jobs[0])] @@ -602,20 +613,21 @@ def update_status(self) -> JobStatus: ), ], ) -def test_build_dependency_string_with_string_input( +def test_pbs_build_dependency_string_with_string_input( dependency: list[str] | list[Job], ids: list[str] | None, status: list[JobStatus], expected: str | None, ) -> None: - """Test building dependency string from string and Job inputs.""" + """Test building PBS dependency string from string and Job inputs.""" + scheduler = Pbs() for dep, id, st in zip(dependency, ids, status, strict=True): if isinstance(dep, Job): dep.status = st dep.job_id = id with expected as e: - assert Job._build_dependency_string(dependency) == e + assert scheduler._build_dependency_string(dependency=dependency) == e def test_submit_pbs_adds_dependency_flag( @@ -624,8 +636,9 @@ def test_submit_pbs_adds_dependency_flag( ) -> None: script = tmp_path / "script.py" script.write_text("") + scheduler = Pbs() job = Job(script, name="crazy_diamond", output_folder=tmp_path) - job.write_pbs(tmp_path / "wywh.pbs") + scheduler.write(job=job, path=tmp_path / "wywh.pbs") captured_cmd = {} @@ -637,10 +650,13 @@ def fake_run(cmd: list[str], *args: None, **kwargs: None) -> Dummy: captured_cmd["cmd"] = cmd return Dummy() + def mock_update_status(self: Pbs, job: Job) -> JobStatus: + return JobStatus.PREPARED + monkeypatch.setattr(subprocess, "run", fake_run) - monkeypatch.setattr(Job, "update_status", lambda _: JobStatus.PREPARED) + monkeypatch.setattr(Pbs, "update_status", mock_update_status) - job.submit_pbs(dependency="1234567") + scheduler.submit(job=job, dependency="1234567") assert "-W" in captured_cmd["cmd"] assert "depend=afterok:1234567" in captured_cmd["cmd"] @@ -657,19 +673,25 @@ def fake_run(cmd: list[str], *args: None, **kwargs: None) -> Dummy: "not_a_supported_type", pytest.raises( ValueError, - match=r"Unsupported dependency type 'not_a_supported_type'\. Expected one of \['after', 'afterany', 'afternotok', 'afterok'\]\.", + match=r"Unsupported dependency type 'not_a_supported_type'", ), id="invalid_dependency_type", ), ], ) -def test_build_dependency_string_with_different_types( +def test_pbs_build_dependency_string_with_different_types( dependency_type: str, expected: type[Exception], ) -> None: """Test building dependency strings with different dependency types.""" + scheduler = Pbs() with expected as e: - assert Job._build_dependency_string("1234567", dependency_type) == e + assert ( + scheduler._build_dependency_string( + dependency="1234567", dependency_type=dependency_type + ) + == e + ) @pytest.mark.parametrize( From 613f805730b7b975b2f8853b88ac38b58d8bd008 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 10:16:24 +0200 Subject: [PATCH 05/54] explicit JobBuilder docstring --- src/osekit/job/builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osekit/job/builder.py b/src/osekit/job/builder.py index ed8ea65c5..762afb1c7 100644 --- a/src/osekit/job/builder.py +++ b/src/osekit/job/builder.py @@ -9,7 +9,8 @@ class JobBuilder: """Class that should be attached to a Public API ``Project`` for working with jobs. - If a ``Project`` has a ``JobBuilder``, it will use it to run transforms through jobs. + If a ``Project`` has a ``JobBuilder``, it will run its transforms through jobs + using the specified scheduler. """ From 87643b2a5bd543c37df425aebeb3a2c64e4b9dce Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 10:20:10 +0200 Subject: [PATCH 06/54] explicit JobConfig docstring --- src/osekit/job/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osekit/job/config.py b/src/osekit/job/config.py index 94981fc36..e7552bfa6 100644 --- a/src/osekit/job/config.py +++ b/src/osekit/job/config.py @@ -5,7 +5,7 @@ @dataclass class JobConfig: - """Config used for creating a job. + """Configuration of the computing resources allowed for a job. Parameters ---------- From 434cf973e0bee6d6b722be2e5877aa4a6614c563 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 11:31:02 +0200 Subject: [PATCH 07/54] fix job_builder.submit() call in Project.run() --- src/osekit/public/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osekit/public/project.py b/src/osekit/public/project.py index 5361aa748..6bae219ca 100644 --- a/src/osekit/public/project.py +++ b/src/osekit/public/project.py @@ -634,7 +634,7 @@ def export( name=name + (f"_{index}" if len(batch_indexes) > 1 else ""), output_folder=self.folder / self.SUBFOLDERS["log"], ) - self.job_builder.submit_pbs() + self.job_builder.submit() def _add_spectro_dataset( self, From 18f189d340e0187afc7155eb8c86bd5db0fd2dad Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 11:51:17 +0200 Subject: [PATCH 08/54] adapt Public API job documentation --- docs/source/jobs.rst | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 2f1c3ddb0..6624bd52f 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -2,45 +2,59 @@ Working with jobs ----------------- **OSEkit** can be set to send transform instructions to be computed on a remote server -through the PBS queuing system. +through queuing systems. This feature has mainly be thought for the Public API, but it can nonetheless be used for any Core API operation. -The job module is located at :mod:`osekit.job.job`. +This is done thanks to the job package, located in :mod:`osekit.job`. Public API ^^^^^^^^^^ -Running Public API Analyses through PBS jobs only requires adding a :class:`osekit.job.job.JobBuilder` -instance to the :attr:`osekit.public.project.Project.job_builder` attribute: +Running Public API Analyses through jobs only requires adding a :class:`osekit.job.builder.JobBuilder` +instance to the :attr:`osekit.public.project.Project.job_builder` attribute. + +The :class:`osekit.job.builder.JobBuilder` + +Here is an example for running a transform on a PBS queue: .. code-block:: python - from osekit.job.job import JobConfig, JobBuilder + import os + + from pandas import Timedelta + + from osekit.job.builder import JobBuilder + from osekit.job.config import JobConfig + from osekit.job.scheduler.pbs import Pbs from osekit.public.project import Project - project = Project(...) # See the Project documentation + project = Project(...) # See the Project documentation job_config = JobConfig( - nb_nodes=1, # Number of nodes on which the job runs - ncpus=28, # Number of total cores used per node - ngpus=1, # Number of total GPU used per node - mem="60gb", # Maximum amount of physical memory used by the job - walltime=Timedelta(hours=5), # Maximum amount of real itime during which the job can be running - venv_name=os.environ["CONDA_DEFAULT_ENV"], # Works only for conda venvs - queue="omp" # Queue in which the job will be submitted + nb_nodes=1, # Number of nodes on which the job runs + ncpus=28, # Number of total cores used per node + ngpus=1, # Number of total GPU used per node + mem="60gb", # Maximum amount of physical memory used by the job + walltime=Timedelta( + hours=5 + ), # Maximum amount of real itime during which the job can be running + venv_name=os.environ["CONDA_DEFAULT_ENV"], # Works only for conda venvs ) + scheduler = Pbs(queue="omp") # Scheduler in which the job is submitted + project.job_builder = JobBuilder( config=job_config, + scheduler=scheduler, ) # Now the dataset has a non-None job_builder attribute, # running a transform will write a PBS file in the logs directory - # and submit it to the requested queue. + # and submit it through the selected scheduler. - project.run(...) # See the Transform documentation + project.run(...) # See the Transform documentation Core API From 0cf6c872f31935868abe3358006d8a07c815af34 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 12:26:41 +0200 Subject: [PATCH 09/54] adapt core job documentation --- docs/source/aplose.rst | 2 +- docs/source/jobs.rst | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/source/aplose.rst b/docs/source/aplose.rst index 646f60183..eba45bb84 100644 --- a/docs/source/aplose.rst +++ b/docs/source/aplose.rst @@ -38,7 +38,7 @@ The :class:`osekit.core.detection.Detection` class inherits from the :class:`ose Plotting a detection ^^^^^^^^^^^^^^^^^^^^ -Detection boxes can be plotted on spectrograms thanks to the :method:`osekit.core.detection.Detection.to_rectangle` method: +Detection boxes can be plotted on spectrograms thanks to the :meth:`osekit.core.detection.Detection.to_rectangle` method: .. code-block:: python diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 6624bd52f..2b5ef43c4 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -74,11 +74,13 @@ and follow the console arguments of the :mod:`osekit.public.export` script. from osekit.core.audio_dataset import AudioDataset from osekit.core.spectro_dataset import SpectroDataset + from osekit.job.config import JobConfig + from osekit.job.job import Job + from osekit.job.scheduler.pbs import Pbs from osekit.public import export_transform # Some Public API imports are required from osekit.public.transform import OutputType - from osekit.job.job import Job, JobConfig ads = AudioDataset(...) # See the AudioDataset doc sds = SpectroDataset(...) # See the SpectroDataset doc @@ -123,9 +125,11 @@ and follow the console arguments of the :mod:`osekit.public.export` script. mem="60gb", walltime=Timedelta(hours=1), venv_name=os.environ["CONDA_DEFAULT_ENV"], - queue="omp", ) + # Scheduler configuration + scheduler = Pbs(queue="omp") + job = Job( script_path=Path(export_transform.__file__), script_args=args, @@ -134,12 +138,12 @@ and follow the console arguments of the :mod:`osekit.public.export` script. output_folder=Path(...), # Path in which the .out and .err files are written ) - # Write the PBS file and submit the job - job.write_pbs(Path(...) / f"{job.name}.pbs") - job.submit_pbs() + # Write the job file and submit it through the scheduler + scheduler.write(job=job, path=Path(...) / f"{job.name}.pbs") + scheduler.submit(job=job) -You can then follow the status of the submitted job: +You can then follow the status of the submitted job through the scheduler: .. code-block:: python - job.update_status() + scheduler.update_status(job=job) From fff6b333defe62415c5670441c823a93b53f9329 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 12:31:07 +0200 Subject: [PATCH 10/54] add new job package API pages in doc --- docs/source/job.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/job.rst b/docs/source/job.rst index fe22917f9..6a01191bf 100644 --- a/docs/source/job.rst +++ b/docs/source/job.rst @@ -4,4 +4,22 @@ Job --- .. automodule:: osekit.job.job - :members: JobConfig, JobBuilder, Job + :members: Job + +Job Config +---------- + +.. automodule:: osekit.job.config + :members: JobConfig + +Job Scheduler +------------- + +.. automodule:: osekit.job.scheduler.pbs + :members: Pbs + +Job Builder +----------- + +.. automodule:: osekit.job.builder + :members: JobBuilder From 122c1e2cd7ecb91537bbeb23c845475f762b8160 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Thu, 20 Aug 2026 14:20:55 +0200 Subject: [PATCH 11/54] remove slurm module to keep it for another PR --- src/osekit/job/scheduler/slurm.py | 55 ------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 src/osekit/job/scheduler/slurm.py diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py deleted file mode 100644 index 31d8a2b5f..000000000 --- a/src/osekit/job/scheduler/slurm.py +++ /dev/null @@ -1,55 +0,0 @@ -from pathlib import Path - -from osekit.job.job import Job, JobStatus -from osekit.job.scheduler.scheduler import Scheduler - - -class Slurm(Scheduler): - """Abstract class representing a job scheduler.""" - - def write(self, job: Job, path: Path) -> None: - """Write a job script to file. - - Parameters - ---------- - job: Job - Job of which to write the script. - path: Path - Path of the file in which the job script is written. - - """ - pass - - def submit( - self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None - ) -> None: - """Submit the job to the scheduler. - - Parameters - ---------- - job: Job - Job to submit to the scheduler. - dependency: Job | list[Job] | str | None - Job dependency. Can be: - - A ``Job`` instance: will wait for that job to complete successfully - - A ``list[Job]``: will wait for all jobs to complete successfully - - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification - - ``None``: no dependency - - """ - pass - - def update_info(self, job: Job) -> None: - """Request info about the job and update it.""" - pass - - def update_status(self, job: Job) -> JobStatus: - """Request info about the job and update its status. - - Returns - ------- - JobStatus: - The updated status of the job. - - """ - pass From dface7d759aaf5f08ecf9c0022e143760ce7c1e2 Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Fri, 28 Aug 2026 12:03:03 +0200 Subject: [PATCH 12/54] move write logic up in scheduler class --- src/osekit/job/scheduler/pbs.py | 33 ++++++++++-------------- src/osekit/job/scheduler/scheduler.py | 37 +++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index b8ef76a59..94965c1fe 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -1,5 +1,4 @@ import subprocess -from pathlib import Path from typing import Literal from osekit.job.job import Job, JobStatus @@ -25,19 +24,22 @@ def queue(self) -> str: def queue(self, queue: Literal["omp", "mpi"]) -> None: self._queue = queue - def write(self, job: Job, path: Path) -> None: - """Write a job script to file. + def _build_job_specification(self, job: Job) -> str: + """Build the job specification string. Parameters ---------- job: Job - Job of which to write the script. - path: Path - Path of the file in which the job script is written. + The job for which to build the specifications. - """ - preamble = "#!/bin/bash" + Returns + ------- + str: + Job specification string. + It includes the name of the job, the requested resources, + output log directories, etc. + """ select_parts = { "select": job.nb_nodes, "ncpus": job.ncpus, @@ -57,7 +59,7 @@ def write(self, job: Job, path: Path) -> None: "-o": f"{job.output_folder}/{job.name}.out" if job.output_folder else None, "-e": f"{job.output_folder}/{job.name}.err" if job.output_folder else None, } - request_str = "\n".join( + return "\n".join( f"#PBS {key} {value}" if type(value) is not list else "\n".join(f"#PBS {key} {value_part}" for value_part in value) @@ -65,17 +67,10 @@ def write(self, job: Job, path: Path) -> None: if value ) - script = f"python {job.script_path} {job.get_arg_string()}" - - pbs = f"{preamble}\n{request_str}\n{self._build_venv_string(job=job)}\n{script}" - with path.open("w") as file: - file.write(pbs) - - job.path = path - job.progress() - def submit( - self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None + self, + job: Job, + dependency: Job | list[Job] | str | list[str] | None = None, ) -> None: """Submit the job to the scheduler. diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index b963c65af..f240d4fd2 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -9,7 +9,6 @@ class Scheduler(ABC): JOB_FILE_EXTENSION = "job" - @abstractmethod def write(self, job: Job, path: Path) -> None: """Write a job script to file. @@ -20,12 +19,45 @@ def write(self, job: Job, path: Path) -> None: path: Path Path of the file in which the job script is written. + """ + preamble = "#!/bin/bash" + + request_str = self._build_job_specification(job=job) + venv_str = self._build_venv_string(job=job) + python_script = f"python {job.script_path} {job.get_arg_string()}" + + script = f"{preamble}\n\n{request_str}\n\n{venv_str}\n\n{python_script}" + + with path.open("w") as file: + file.write(script) + + job.path = path + job.progress() + + @abstractmethod + def _build_job_specification(self, job: Job) -> str: + """Build the job specification string. + + Parameters + ---------- + job: Job + The job for which to build the specifications. + + Returns + ------- + str: + Job specification string. + It includes the name of the job, the requested resources, + output log directories, etc. + """ ... @abstractmethod def submit( - self, job: Job, dependency: Job | list[Job] | str | list[str] | None = None + self, + job: Job, + dependency: Job | list[Job] | str | list[str] | None = None, ) -> None: """Submit the job to the scheduler. @@ -92,5 +124,6 @@ def _build_dependency_string( ------- str Job dependency string. + """ ... From 99752d38f2f2809b7197279185db9ffe60110266 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:38:05 +0200 Subject: [PATCH 13/54] change / path separator to system dependant path separator --- src/osekit/job/scheduler/pbs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 94965c1fe..c34a9e391 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -56,8 +56,8 @@ def _build_job_specification(self, job: Job) -> str: select_str, f"walltime={job.walltime_str}", ], - "-o": f"{job.output_folder}/{job.name}.out" if job.output_folder else None, - "-e": f"{job.output_folder}/{job.name}.err" if job.output_folder else None, + "-o": f"{job.output_folder / job.name}.out" if job.output_folder else None, + "-e": f"{job.output_folder / job.name}.err" if job.output_folder else None, } return "\n".join( f"#PBS {key} {value}" From a29c09c2e1f9d6734a5b8303c5b3bf43c4ac982a Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:38:47 +0200 Subject: [PATCH 14/54] add Pbs._build_job_specifications() test --- tests/test_job.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 4e5107017..0617678fe 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -103,6 +103,34 @@ def test_walltime_str_and_setter() -> None: assert job.walltime_str == "13:08:09" +def test_pbs_build_job_specifications() -> None: + job = Job( + script_path=Path(), + config=JobConfig( + nb_nodes=2, + ncpus=3, + ngpus=1, + mem="16gb", + walltime=Timedelta(hours=2), + venv_name="cool_env", + ), + output_folder=Path(r"cool/folder"), + name="cool_job", + ) + + specifications = Pbs(queue="mpi")._build_job_specification(job=job).splitlines() + + for expected_specification in ( + "#PBS -N cool_job", + "#PBS -q mpi", + "#PBS -l select=2:ncpus=3:mem=16gb:ngpus=1", + "#PBS -l walltime=02:00:00", + f"#PBS -o {Path('cool/folder') / 'cool_job.out'}", + f"#PBS -e {Path('cool/folder') / 'cool_job.err'}", + ): + assert expected_specification in specifications + + def test_write_pbs(tmp_path: Path) -> None: script = tmp_path / "shpouik_shpouik.py" script.write_text("print('edgar')") @@ -462,7 +490,9 @@ def __init__(self, name: str, status: JobStatus) -> None: self.status = status def mock_submit( - self: Scheduler, job: Job, dependency: Job | str | None = None + self: Scheduler, + job: Job, + dependency: Job | str | None = None, ) -> None: submitted_jobs.append((job.name, dependency)) @@ -688,7 +718,8 @@ def test_pbs_build_dependency_string_with_different_types( with expected as e: assert ( scheduler._build_dependency_string( - dependency="1234567", dependency_type=dependency_type + dependency="1234567", + dependency_type=dependency_type, ) == e ) From b6327a5400e4e5de0d568e5534234d8c1c1890dd Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:12:20 +0200 Subject: [PATCH 15/54] remove windows-like path separator in write_pbs test --- tests/test_job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 0617678fe..117d2daca 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -155,11 +155,11 @@ def test_write_pbs(tmp_path: Path) -> None: assert any("select=1:ncpus=2:mem=8gb" in line for line in content) assert any("walltime=01:00:00" in line for line in content) assert any( - line.startswith(f"#PBS -o {job.output_folder}/{job.name}.out") + line.startswith(f"#PBS -o {job.output_folder / job.name}.out") for line in content ) assert any( - line.startswith(f"#PBS -e {job.output_folder}/{job.name}.err") + line.startswith(f"#PBS -e {job.output_folder / job.name}.err") for line in content ) From e1c0463e9a380776cfe3c4865bc8232cb97e7c8a Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:43:32 +0200 Subject: [PATCH 16/54] fix JobBuilder import --- src/osekit/public/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osekit/public/project.py b/src/osekit/public/project.py index 648559fc9..aef0db689 100644 --- a/src/osekit/public/project.py +++ b/src/osekit/public/project.py @@ -38,7 +38,7 @@ from pandas import Timestamp from osekit.core.audio_file import AudioFile - from osekit.job.job import JobBuilder + from osekit.job.builder import JobBuilder class Project: @@ -634,7 +634,7 @@ def export( nb_jobs=nb_jobs, ) - self.job_builder.submit_pbs() + self.job_builder.submit() @staticmethod def get_json_paths( From f0b43846deb7ef719806c572e024fe9bd8b86e46 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:10:22 +0200 Subject: [PATCH 17/54] Dependency dict (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * change dependencies management * add full examples coverage in doctest * update dependency parameter in test_submit_pbs function * add test function for build dependencies string from string ids in pbs scheduler * add test function for build dependencies string from jobs in pbs scheduler * add test function for _validate_dependency_type call in PBS build_dependencies_string * move dependencies job status validation to dedicated method * add test for job status validation call on pbs build dependencies string * add tests for Pbs._vaidate_dependenciesÃ_jobs_status * remove old all-in-one test function * remove redundant test * add missing valid dependency types * simplify PBS submit erorrs test * add test for Pbs.validate_dependency_type * add test for Pbs.validate_job_id * add docstring for Pbs._parse_job_ids() * delegate pbs validations to the scheduler * adapt JobBuilder.submit() test * remove tests on delegated validation functions * expose JobBuilder job list in a property * separate venv instruction string parts with a linebreak * adapt PBS write test to multiline conda env instruction --- src/osekit/job/builder.py | 34 +++- src/osekit/job/scheduler/pbs.py | 132 ++++++++----- src/osekit/job/scheduler/scheduler.py | 36 ++-- tests/test_job.py | 275 ++++++++++---------------- 4 files changed, 228 insertions(+), 249 deletions(-) diff --git a/src/osekit/job/builder.py b/src/osekit/job/builder.py index f33ebf536..2e083286e 100644 --- a/src/osekit/job/builder.py +++ b/src/osekit/job/builder.py @@ -34,6 +34,15 @@ def __init__( self.scheduler = scheduler or Pbs() self.jobs = [] + @property + def jobs(self) -> list[Job]: + """Return the jobs created by this job builder.""" + return self._jobs + + @jobs.setter + def jobs(self, jobs: list[Job]) -> None: + self._jobs = jobs + def create_jobs( self, nb_tasks: int, @@ -112,24 +121,33 @@ def create_job( def submit( self, - dependencies: dict[str, Job | list[Job]] | None = None, + dependencies: dict[Job, dict[str, str | Job | list[str | Job]]] | None = None, ) -> None: """Submit all prepared jobs to the scheduler system. Parameters ---------- - dependencies: dict[str, Job | list[Job]] | None - Optional dictionary mapping job names to their dependencies. - Example: ``{"job2": job1, "job3": [job1, job2]}`` + dependencies: dict[Job, dict[str, str | Job | list[str | Job]]] | None + Optional mapping of the jobs dependencies. + + For each key (which is a job), the value is a + dictionary explaining its dependencies. + + Such dictionary follows the following format: + The keys of the dictionary are the dependency types, + that are proper to the scheduler. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. + + If ``None``, the jobs are submitted without any dependency. """ + dependencies = dependencies or {} for job in self.jobs: if self.scheduler.update_status(job=job) is not JobStatus.PREPARED: continue # Check if this job has dependencies - depend_on = None - if dependencies and job.name in dependencies: - depend_on = dependencies[job.name] + job_dependencies = dependencies.get(job, None) - self.scheduler.submit(job=job, dependency=depend_on) + self.scheduler.submit(job=job, dependencies=job_dependencies) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index c34a9e391..b447fe037 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -6,9 +6,22 @@ class Pbs(Scheduler): - """Abstract class representing a job scheduler.""" - - _VALID_DEPENDENCY_TYPES = frozenset({"afterok", "afterany", "afternotok", "after"}) + """Abstract class representing a PBS job scheduler.""" + + _VALID_DEPENDENCY_TYPES = frozenset( + { + "after", + "afterok", + "afternotok", + "afterany", + "before", + "beforeok", + "beforenotok", + "beforeany", + "on", + "runone", + }, + ) JOB_FILE_EXTENSION = "pbs" def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: @@ -70,7 +83,7 @@ def _build_job_specification(self, job: Job) -> str: def submit( self, job: Job, - dependency: Job | list[Job] | str | list[str] | None = None, + dependencies: dict[str, Job | str | list[Job | str]] | None = None, ) -> None: """Submit the job to the scheduler. @@ -78,12 +91,14 @@ def submit( ---------- job: Job Job to submit to the scheduler. - dependency: Job | list[Job] | str | None - Job dependency. Can be: - - A ``Job`` instance: will wait for that job to complete successfully - - A ``list[Job]``: will wait for all jobs to complete successfully - - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification - - ``None``: no dependency + dependencies: dict[str, Job | str | list[Job|str]] + The dependencies of the submitted job. + The keys of the dictionary are the dependency types, + see https://help.altair.com/2022.1.0/PBS%20Professional/PBSReferenceGuide2022.1.pdf#page=151 + for the list of supported values. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. + If ``None``, the job is submitted without any dependency. """ if self.update_status(job=job) is not JobStatus.PREPARED: @@ -92,8 +107,8 @@ def submit( cmd = ["qsub"] - if dependency is not None: - dependency_str = self._build_dependency_string(dependency) + if dependencies is not None: + dependency_str = self._build_dependency_string(dependencies) if dependency_str: cmd.extend(["-W", f"depend={dependency_str}"]) @@ -181,44 +196,53 @@ def update_status(self, job: Job) -> JobStatus: @staticmethod def _build_venv_string(job: Job) -> str: """Bash script used for activating the conda virtual environment.""" - return f". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate {job.venv_name}" + return ( + f". /appli/anaconda/latest/etc/profile.d/conda.sh\n" + f"conda activate {job.venv_name}" + ) @classmethod def _validate_dependency_type(cls, dependency_type: str) -> None: if dependency_type not in cls._VALID_DEPENDENCY_TYPES: msg = ( - f"Unsupported dependency type '{dependency_type}'. " - f"Expected one of {cls._VALID_DEPENDENCY_TYPES}." + f"Unsupported dependency type '{dependency_type}'.\n" + f"Expected one of:\n\t{'\n\t'.join(sorted(cls._VALID_DEPENDENCY_TYPES))}." ) raise ValueError(msg) - @staticmethod - def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: - job_ids = [dep.job_id if isinstance(dep, Job) else dep for dep in dependency] - job_id_length = 7 - for job_id in job_ids: - if not job_id.isdigit() or len(job_id) != job_id_length: - msg = ( - f"Invalid job ID '{job_id}'. " - f"Job IDs must be {job_id_length} digits long." - ) - raise ValueError(msg) - return job_ids + @classmethod + def _parse_job_ids( + cls, + dependencies: dict[str, Job | str | list[Job | str]], + ) -> dict[str, list[str]]: + """Replace all ``Job`` instances by their ID string.""" + parsed_dependencies = {} + for key, value in dependencies.items(): + parsed_values = value if isinstance(value, list) else [value] + parsed_values = [ + parsed_value.job_id if isinstance(parsed_value, Job) else parsed_value + for parsed_value in parsed_values + ] + parsed_dependencies[key] = parsed_values + + return parsed_dependencies @classmethod def _build_dependency_string( cls, - dependency: str | Job | list[str] | list[Job], - dependency_type: str = "afterok", + dependencies: dict[str, Job | str | list[Job | str]], ) -> str: """Build a PBS dependency string. Parameters ---------- - dependency: Job | str - ``Job`` or job ID to depend on. - dependency_type: str - Type of dependency (``afterok``, ``afterany``, ``afternotok``, ``after``). + dependencies: dict[str, Job | str | list[Job|str]] + The dependencies of the submitted job. + The keys of the dictionary are the dependency types, + see https://help.altair.com/2022.1.0/PBS%20Professional/PBSReferenceGuide2022.1.pdf#page=151 + for the list of supported values. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. Returns ------- @@ -227,27 +251,31 @@ def _build_dependency_string( Examples -------- - >>> Pbs._build_dependency_string("1234567") + >>> Pbs._build_dependency_string({"afterok": "1234567"}) 'afterok:1234567' - >>> Pbs._build_dependency_string(["1234567", "4567891"]) + >>> Pbs._build_dependency_string({"afterok": ["1234567","4567891"]}) 'afterok:1234567:4567891' - >>> Pbs._build_dependency_string("7894561", dependency_type="afterany") - 'afterany:7894651' + >>> from pathlib import Path + >>> job = Job(Path()) + >>> job._id = "7894561" + >>> Pbs._build_dependency_string({"afterany":job}) + 'afterany:7894561' + >>> from pathlib import Path + >>> job1 = Job(Path()) + >>> job1._id = "7894561" + >>> job2 = Job(Path()) + >>> job2._id = "4839572" + >>> Pbs._build_dependency_string({"afterany":[job1,job2]}) + 'afterany:7894561:4839572' """ - dependency = dependency if isinstance(dependency, list) else [dependency] - id_str = cls._validate_dependency(dependency=dependency) - cls._validate_dependency_type(dependency_type=dependency_type) - - if unsubmitted_job := next( - ( - j - for j in dependency - if isinstance(j, Job) and j.status.value < JobStatus.QUEUED.value - ), - None, - ): - msg = f"Job '{unsubmitted_job.name}' has not been submitted yet." - raise ValueError(msg) + # Check that types are valid before submitting + for dependency_type in dependencies: + cls._validate_dependency_type(dependency_type=dependency_type) - return f"{dependency_type}:{':'.join(id_str)}" + id_str = cls._parse_job_ids(dependencies=dependencies) + + return ",".join( + f"{dependency_type}:{':'.join(ids)}" + for dependency_type, ids in id_str.items() + ) diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index f240d4fd2..f0e9808c6 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -57,7 +57,7 @@ def _build_job_specification(self, job: Job) -> str: def submit( self, job: Job, - dependency: Job | list[Job] | str | list[str] | None = None, + dependencies: dict[str, Job | str | list[Job | str]] | None = None, ) -> None: """Submit the job to the scheduler. @@ -65,12 +65,13 @@ def submit( ---------- job: Job Job to submit to the scheduler. - dependency: Job | list[Job] | str | None - Job dependency. Can be: - - A ``Job`` instance: will wait for that job to complete successfully - - A ``list[Job]``: will wait for all jobs to complete successfully - - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification - - ``None``: no dependency + dependencies: dict[str, Job | str | list[Job|str]] + The dependencies of the submitted job. + The keys of the dictionary are the dependency types, + that are proper to the scheduler. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. + If ``None``, the job is submitted without any dependency. """ ... @@ -100,25 +101,30 @@ def _build_venv_string(job: Job) -> str: ... @abstractmethod def _validate_dependency_type(cls, dependency_type: str) -> None: ... - @staticmethod + @classmethod @abstractmethod - def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: ... + def _parse_job_ids( + cls, + dependencies: dict[str, Job | str | list[Job | str]], + ) -> dict[str, list[str]]: ... @classmethod @abstractmethod def _build_dependency_string( cls, - dependency: str | Job | list[str] | list[Job], - dependency_type: str = "", + dependencies: dict[str, Job | str | list[Job | str]], ) -> str: """Build a job dependency string. Parameters ---------- - dependency: Job | str - ``Job`` or job ID to depend on. - dependency_type: str - Type of dependency. + dependencies: dict[str, Job | str | list[Job|str]] + The dependencies of the submitted job. + The keys of the dictionary are the dependency types, + that are proper to the scheduler. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. + If ``None``, the job is submitted without any dependency. Returns ------- diff --git a/tests/test_job.py b/tests/test_job.py index 117d2daca..a674193b9 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -1,7 +1,6 @@ from __future__ import annotations import subprocess -from contextlib import nullcontext from pathlib import Path import numpy as np @@ -163,10 +162,9 @@ def test_write_pbs(tmp_path: Path) -> None: for line in content ) - assert ( - ". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate osekit" - in content - ) + assert ". /appli/anaconda/latest/etc/profile.d/conda.sh" in content + assert "conda activate osekit" in content + last = content[-1] assert last.startswith(f"python {script}") assert "--vieille face" in last @@ -252,13 +250,9 @@ def mock_update_status(self, job: Job) -> JobStatus: def test_submit_pbs_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - script = tmp_path / "boc.py" - script.write_text("") - outdir = tmp_path - job = Job(script, name="amobishoproden", output_folder=outdir) - pbs_path = tmp_path / "amobishoproden.pbs" + job = Job(Path()) pbs_scheduler = Pbs(queue="omp") - pbs_scheduler.write(job=job, path=pbs_path) + job.status = JobStatus.PREPARED class Dummy: def __init__(self) -> None: @@ -271,7 +265,12 @@ def __init__(self) -> None: lambda *args, **kwargs: Dummy(), ) - assert job.status == JobStatus.PREPARED + def mock_update_status(self: Pbs, job: Job) -> JobStatus: + return JobStatus.PREPARED + + monkeypatch.setattr(Pbs, "update_status", mock_update_status) + + # Submit error should leave the job prepared: with pytest.raises(RuntimeError, match="Submission failed with exit code 5"): pbs_scheduler.submit(job=job) @@ -492,9 +491,9 @@ def __init__(self, name: str, status: JobStatus) -> None: def mock_submit( self: Scheduler, job: Job, - dependency: Job | str | None = None, + dependencies: Job | str | None = None, ) -> None: - submitted_jobs.append((job.name, dependency)) + submitted_jobs.append((job, dependencies)) def mock_update_status(self: Scheduler, job: Job) -> JobStatus: return job.status @@ -514,150 +513,111 @@ def mock_update_status(self: Scheduler, job: Job) -> JobStatus: job_builder = JobBuilder() job_builder.jobs = jobs - dependencies = {"prepared": jobs[0]} + unprepared_job = job_builder.jobs[0] + prepared_job = job_builder.jobs[1] + + dependencies = { + prepared_job: {"beforeok": unprepared_job}, + unprepared_job: {"afterany": prepared_job}, + } job_builder.submit(dependencies=dependencies) - assert submitted_jobs == [("prepared", jobs[0])] + # Only the prepared job should be submitted + assert len(submitted_jobs) == 1 + submitted_job = submitted_jobs[0] + assert submitted_job[0] == prepared_job + + # Only the submitted job dependencies should be injected + assert submitted_job[1] == dependencies[prepared_job] + + +def test_pbs_build_dependencies_string_validates_type( + monkeypatch: pytest.MonkeyPatch, +) -> None: + validate_calls = [] + + def mock_validate(dependency_type: str) -> None: + validate_calls.append(dependency_type) + + monkeypatch.setattr(Pbs, "_validate_dependency_type", mock_validate) + + dependencies = {"afterok": "1234567", "afterany": ["2345678", "3456789"]} + Pbs()._build_dependency_string( + dependencies=dependencies, + ) + + assert all(dependency_type in validate_calls for dependency_type in dependencies) + + +def test_pbs_validate_dependency_type() -> None: + pbs = Pbs() + + # Supported dependency type shouldn't raise + pbs._validate_dependency_type("afterok") + + # Unsupported dependency type should raise + with pytest.raises(ValueError) as e: + pbs._validate_dependency_type("afterdummy") + + assert e.match("Unsupported dependency type 'afterdummy'") + for supported in Pbs._VALID_DEPENDENCY_TYPES: + assert e.match(supported) @pytest.mark.parametrize( - ("dependency", "ids", "status", "expected"), + ("dependencies", "expected"), [ pytest.param( - ["1234567"], - [None], - [None], - nullcontext("afterok:1234567"), - id="single_job_id", + {"afterok": "1234567"}, + "afterok:1234567", + id="one_type_one_job", ), pytest.param( - ["1234567", "4567891", "7891234"], - [None] * 3, - [None] * 3, - nullcontext("afterok:1234567:4567891:7891234"), - id="multiple_job_ids", + {"afterok": ["1234567", "2345678"]}, + "afterok:1234567:2345678", + id="one_type_multiple_jobs", ), pytest.param( - ["123"], - [None], - [None], - pytest.raises( - ValueError, - match=r"Invalid job ID '123'\. Job IDs must be 7 digits long\.", - ), - id="invalid_job_id_too_short", + {"afterok": "1234567", "afterany": "2345678"}, + "afterok:1234567,afterany:2345678", + id="multiple_types_one_job", ), pytest.param( - [Job(script_path=Path("test.py"), name="job_1")], - ["12345678"], - [JobStatus.QUEUED], - pytest.raises( - ValueError, - match=r"Invalid job ID '12345678'\. Job IDs must be 7 digits long\.", - ), - id="invalid_job_id_too_long", - ), - pytest.param( - ["abcdefg"], - [None], - [None], - pytest.raises( - ValueError, - match=r"Invalid job ID 'abcdefg'\. Job IDs must be 7 digits long\.", - ), - id="invalid_job_id_non_numeric", - ), - pytest.param( - ["1234567", "not_a_job_id"], - [None] * 2, - [None] * 2, - pytest.raises( - ValueError, - match=r"Invalid job ID 'not_a_job_id'\. Job IDs must be 7 digits long\.", - ), - id="multiple_job_id_one_invalid", - ), - pytest.param( - [Job(script_path=Path("test.py"), name="job_1")], - ["1234567"], - [JobStatus.QUEUED], - nullcontext("afterok:1234567"), - id="single_job_instance", - ), - pytest.param( - [ - Job(script_path=Path("horse_with.py"), name="job_1"), - Job(script_path=Path("no_name.py"), name="job_2"), - ], - ["1234567", "4567891"], - [JobStatus.QUEUED, JobStatus.QUEUED], - nullcontext("afterok:1234567:4567891"), - id="multiple_job_instance", - ), - pytest.param( - [ - Job(script_path=Path("king_crimson.py"), name="job_1"), - Job(script_path=Path("crimson_king.py"), name="job_2"), - ], - ["1234567", "not_an_id"], - [JobStatus.QUEUED, JobStatus.QUEUED], - pytest.raises( - ValueError, - match=r"Invalid job ID 'not_an_id'\. Job IDs must be 7 digits long\.", - ), - id="multiple_job_instance_invalid_one", - ), - pytest.param( - [ - Job(script_path=Path("king_crimson.py"), name="job_1"), - "9876543", - ], - ["1234567", None], - [JobStatus.QUEUED, None], - nullcontext("afterok:1234567:9876543"), - id="job_and_string_input", - ), - pytest.param( - [Job(script_path=Path("test.py"), name="tornero")], - ["1234567"], - [JobStatus.UNPREPARED], - pytest.raises( - ValueError, - match="Job 'tornero' has not been submitted yet.", - ), - id="unprepared_job_instance", - ), - pytest.param( - [ - Job(script_path=Path("script.py"), name="dalida"), - Job(script_path=Path("script.py"), name="mourir_sur_scene"), - ], - ["1234567", "4567896"], - [JobStatus.QUEUED, JobStatus.PREPARED], - pytest.raises( - ValueError, - match="Job 'mourir_sur_scene' has not been submitted yet.", - ), - id="multiple_job_instance_one_not_submitted", + {"afterok": ["1234567", "2345678"], "afterany": ["3456789", "4567890"]}, + "afterok:1234567:2345678,afterany:3456789:4567890", + id="multiple_types_multiple_jobs", ), ], ) -def test_pbs_build_dependency_string_with_string_input( - dependency: list[str] | list[Job], - ids: list[str] | None, - status: list[JobStatus], - expected: str | None, +def test_pbs_build_dependencies_string( + dependencies: dict[str, str | list[str]], + expected: str, ) -> None: - """Test building PBS dependency string from string and Job inputs.""" - scheduler = Pbs() - for dep, id, st in zip(dependency, ids, status, strict=True): - if isinstance(dep, Job): - dep.status = st - dep.job_id = id - - with expected as e: - assert scheduler._build_dependency_string(dependency=dependency) == e + # %% Dependencies string from job IDs + assert Pbs()._build_dependency_string(dependencies=dependencies) == expected + + # %% Dependencies string from Job instances + def id_to_job(job_id: str | list[str]) -> Job | list[Job]: + """Convert a Job ID ``job_id`` to a Job object with an ID of ``job_id`` + + If ``job_id`` is a list, converts the list of job_ids to a list of jobs + with the given IDs.""" + if isinstance(job_id, str): + job = Job(Path()) + job._id = job_id + job.status = JobStatus.QUEUED + return job + output = [] + for j_id in job_id: + job = Job(Path()) + job._id = j_id + job.status = JobStatus.QUEUED + output.append(job) + return output + + dependencies = {key: id_to_job(value) for key, value in dependencies.items()} + assert Pbs()._build_dependency_string(dependencies=dependencies) == expected def test_submit_pbs_adds_dependency_flag( @@ -686,45 +646,12 @@ def mock_update_status(self: Pbs, job: Job) -> JobStatus: monkeypatch.setattr(subprocess, "run", fake_run) monkeypatch.setattr(Pbs, "update_status", mock_update_status) - scheduler.submit(job=job, dependency="1234567") + scheduler.submit(job=job, dependencies={"afterok": "1234567"}) assert "-W" in captured_cmd["cmd"] assert "depend=afterok:1234567" in captured_cmd["cmd"] -@pytest.mark.parametrize( - ("dependency_type", "expected"), - [ - pytest.param("afterok", nullcontext("afterok:1234567"), id="afterok"), - pytest.param("afterany", nullcontext("afterany:1234567"), id="afterany"), - pytest.param("afternotok", nullcontext("afternotok:1234567"), id="afternotok"), - pytest.param("after", nullcontext("after:1234567"), id="after"), - pytest.param( - "not_a_supported_type", - pytest.raises( - ValueError, - match=r"Unsupported dependency type 'not_a_supported_type'", - ), - id="invalid_dependency_type", - ), - ], -) -def test_pbs_build_dependency_string_with_different_types( - dependency_type: str, - expected: type[Exception], -) -> None: - """Test building dependency strings with different dependency types.""" - scheduler = Pbs() - with expected as e: - assert ( - scheduler._build_dependency_string( - dependency="1234567", - dependency_type=dependency_type, - ) - == e - ) - - @pytest.mark.parametrize( "walltime", [ From 776d19296f182088889b8a0dc791de5be2703c7c Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:36:14 +0200 Subject: [PATCH 18/54] move parse_job_ids() up from the pbs class to the scheduler ABC --- src/osekit/job/scheduler/pbs.py | 17 ----------------- src/osekit/job/scheduler/scheduler.py | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index b447fe037..6e4263cbe 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -210,23 +210,6 @@ def _validate_dependency_type(cls, dependency_type: str) -> None: ) raise ValueError(msg) - @classmethod - def _parse_job_ids( - cls, - dependencies: dict[str, Job | str | list[Job | str]], - ) -> dict[str, list[str]]: - """Replace all ``Job`` instances by their ID string.""" - parsed_dependencies = {} - for key, value in dependencies.items(): - parsed_values = value if isinstance(value, list) else [value] - parsed_values = [ - parsed_value.job_id if isinstance(parsed_value, Job) else parsed_value - for parsed_value in parsed_values - ] - parsed_dependencies[key] = parsed_values - - return parsed_dependencies - @classmethod def _build_dependency_string( cls, diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index f0e9808c6..4ee7d5226 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -101,12 +101,21 @@ def _build_venv_string(job: Job) -> str: ... @abstractmethod def _validate_dependency_type(cls, dependency_type: str) -> None: ... - @classmethod - @abstractmethod + @staticmethod def _parse_job_ids( - cls, dependencies: dict[str, Job | str | list[Job | str]], - ) -> dict[str, list[str]]: ... + ) -> dict[str, list[str]]: + """Replace all ``Job`` instances by their ID string.""" + parsed_dependencies = {} + for key, value in dependencies.items(): + parsed_values = value if isinstance(value, list) else [value] + parsed_values = [ + parsed_value.job_id if isinstance(parsed_value, Job) else parsed_value + for parsed_value in parsed_values + ] + parsed_dependencies[key] = parsed_values + + return parsed_dependencies @classmethod @abstractmethod From d1c7d6c86a7b97544a906973397fd72387ed2688 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:21:40 +0200 Subject: [PATCH 19/54] add pbs status request result file --- tests/_static/job_status_request_results/pbs.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/_static/job_status_request_results/pbs.txt diff --git a/tests/_static/job_status_request_results/pbs.txt b/tests/_static/job_status_request_results/pbs.txt new file mode 100644 index 000000000..c77af3532 --- /dev/null +++ b/tests/_static/job_status_request_results/pbs.txt @@ -0,0 +1,3 @@ +Job id Name User Time Use S Queue +---------------- ---------------- ---------------- -------- - ----- +7137005.a24films0 SwissArmyMan daniels 00:10:37 R jetski From 9c2809cb334cc09544b67959133856d9212a5c7e Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:25:42 +0200 Subject: [PATCH 20/54] change pbs job info parsing --- src/osekit/job/job.py | 6 +++--- src/osekit/job/scheduler/pbs.py | 28 +++++++++++++++++----------- tests/test_job.py | 29 ++++++++++++++++++----------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/osekit/job/job.py b/src/osekit/job/job.py index a2176a2ad..c03194893 100644 --- a/src/osekit/job/job.py +++ b/src/osekit/job/job.py @@ -65,7 +65,7 @@ def __init__( """ config = JobConfig() if config is None else config self.script_path = script_path - self.script_args = script_args if script_args else {} + self.script_args = script_args or {} self.nb_nodes = config.nb_nodes self.ncpus = config.ncpus self.ngpus = config.ngpus @@ -74,10 +74,10 @@ def __init__( self.venv_name = config.venv_name self.name = name self.output_folder = output_folder + self.job_info = {} self._status = JobStatus.UNPREPARED self._path = None self._id = None - self._info = None @property def script_path(self) -> Path: @@ -215,7 +215,7 @@ def job_id(self, job_id: str | None) -> None: self._id = job_id @property - def job_info(self) -> dict | None: + def job_info(self) -> dict: """Information about the job.""" return self._info diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 6e4263cbe..92859fde9 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -135,7 +135,7 @@ def update_info(self, job: Job) -> None: try: request = subprocess.run( - ["qstat", "-f", str(job.job_id)], + ["qstat", "-x", str(job.job_id)], capture_output=True, text=True, check=False, @@ -147,21 +147,27 @@ def update_info(self, job: Job) -> None: if not stdout: err = request.stderr - if "Job has finished" in err: - job.status = JobStatus.COMPLETED - job.job_info["job_state"] = "C" if "Unknown Job Id" in err: msg = f"Unknown Job Id {job.job_id}" raise ValueError(msg) return - info = {} - for line in stdout.splitlines(): - if "=" not in line: - continue - key, value = line.split("=", 1) - info[key.strip()] = value.strip() - job.job_info = info + self._parse_info_str(job=job, info=stdout) + + def _parse_info_str(self, job: Job, info: str) -> None: + """Parse the info from the requested qstat info string.""" + keys, _, values = info.splitlines() + + # Get keys order in the string + known_keys = ["Job id", "Name", "User", "Time Use", "S", "Queue"] + keys = sorted(known_keys, key=keys.index) + + # Get the associated values + kvp = dict(zip(keys, values.split(), strict=True)) + + job.job_info["user"] = kvp["User"] + job.job_info["time"] = kvp["Time Use"] + job.job_info["queue"] = kvp["Queue"] def update_status(self, job: Job) -> JobStatus: """Request info about the job and update its status. diff --git a/tests/test_job.py b/tests/test_job.py index a674193b9..f27cc7f33 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -277,7 +277,7 @@ def mock_update_status(self: Pbs, job: Job) -> JobStatus: assert job.status == JobStatus.PREPARED -def test_update_info_no_job_id() -> None: +def test_pbs_update_info_no_job_id() -> None: job = Job(Path("pixies.py")) pbs_scheduler = Pbs() job.job_id = None @@ -285,14 +285,14 @@ def test_update_info_no_job_id() -> None: assert job.job_info is None -def test_update_info_parse_stdout(monkeypatch: pytest.MonkeyPatch) -> None: +def test_pbs_update_info_parse_stdout(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("fontaines.py")) - job.job_id = "43" - job.status = JobStatus.RUNNING - raw = " frankie = cosmos \navey=tare\nattic= abasement\nthis will be ignored" + job.job_id = "7137005" class Dummy: - stdout = raw + stdout = ( + Path(__file__).parent / "_static/job_status_request_results/pbs.txt" + ).read_text() stderr = "" monkeypatch.setattr( @@ -302,10 +302,17 @@ class Dummy: ) scheduler = Pbs() scheduler.update_info(job=job) - assert job.job_info == {"frankie": "cosmos", "avey": "tare", "attic": "abasement"} + assert job.job_id == "7137005" + assert job.name == "SwissArmyMan" + assert job.status == JobStatus.RUNNING + assert job.job_info == { + "user": "daniels", + "time": "00:10:37", + "queue": "jetski", + } -def test_update_info_completed(monkeypatch: pytest.MonkeyPatch) -> None: +def test_pbs_update_info_completed(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("amok.py")) job.job_id = "25022013" job.job_info = {} @@ -326,7 +333,7 @@ class Dummy: assert job.job_info["job_state"] == "C" -def test_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: +def test_pbs_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" @@ -345,7 +352,7 @@ class Dummy: scheduler.update_info(job=job) -def test_update_info_error(monkeypatch: pytest.MonkeyPatch) -> None: +def test_pbs_update_info_error(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" @@ -364,7 +371,7 @@ def __init__(self) -> None: scheduler.update_info(job=job) -def test_update_status(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_pbs_update_status(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("porticoquartet.py")) job.path = tmp_path / "pompidou.pbs" From 3bc338987a1e9a3e723f9c0f0c435b6a821b6e94 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:24:15 +0200 Subject: [PATCH 21/54] remove job status progress() system --- src/osekit/job/job.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/osekit/job/job.py b/src/osekit/job/job.py index c03194893..e6aeb0d7b 100644 --- a/src/osekit/job/job.py +++ b/src/osekit/job/job.py @@ -33,7 +33,8 @@ class JobStatus(Enum): PREPARED = 2 QUEUED = 3 RUNNING = 4 - COMPLETED = 5 + SUSPENDED = 5 + COMPLETED = 6 class Job: @@ -74,7 +75,7 @@ def __init__( self.venv_name = config.venv_name self.name = name self.output_folder = output_folder - self.job_info = {} + self.info = {} self._status = JobStatus.UNPREPARED self._path = None self._id = None @@ -215,20 +216,14 @@ def job_id(self, job_id: str | None) -> None: self._id = job_id @property - def job_info(self) -> dict: + def info(self) -> dict: """Information about the job.""" return self._info - @job_info.setter - def job_info(self, info: dict) -> None: + @info.setter + def info(self, info: dict) -> None: self._info = info - def progress(self) -> None: - """Bring the job to the next state.""" - if self.status == JobStatus.COMPLETED: - return - self._status = JobStatus(self._status.value + 1) - def get_arg_string(self) -> str: """Build a string representation of the job's arguments.""" arg_list = [] From e8c4bd14be55e1ee0613e6bef9f6c236ec8771b3 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:25:16 +0200 Subject: [PATCH 22/54] adapt pbs job status parsing from qstat --- src/osekit/job/scheduler/pbs.py | 30 ++++++++++++++++++++------- src/osekit/job/scheduler/scheduler.py | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 92859fde9..a01e5cda1 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -1,4 +1,5 @@ import subprocess +import typing from typing import Literal from osekit.job.job import Job, JobStatus @@ -8,7 +9,7 @@ class Pbs(Scheduler): """Abstract class representing a PBS job scheduler.""" - _VALID_DEPENDENCY_TYPES = frozenset( + _VALID_DEPENDENCY_TYPES: typing.ClassVar = frozenset( { "after", "afterok", @@ -22,7 +23,16 @@ class Pbs(Scheduler): "runone", }, ) - JOB_FILE_EXTENSION = "pbs" + JOB_FILE_EXTENSION: typing.ClassVar = "pbs" + + JOB_STATUS_CODES: typing.ClassVar = { + "Q": JobStatus.QUEUED, + "R": JobStatus.RUNNING, + "S": JobStatus.SUSPENDED, + "H": JobStatus.SUSPENDED, + "E": JobStatus.COMPLETED, + "F": JobStatus.COMPLETED, + } def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: """Initialize the PBS scheduler.""" @@ -154,7 +164,8 @@ def update_info(self, job: Job) -> None: self._parse_info_str(job=job, info=stdout) - def _parse_info_str(self, job: Job, info: str) -> None: + @classmethod + def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested qstat info string.""" keys, _, values = info.splitlines() @@ -165,9 +176,12 @@ def _parse_info_str(self, job: Job, info: str) -> None: # Get the associated values kvp = dict(zip(keys, values.split(), strict=True)) - job.job_info["user"] = kvp["User"] - job.job_info["time"] = kvp["Time Use"] - job.job_info["queue"] = kvp["Queue"] + job.info["user"] = kvp["User"] + job.info["time"] = kvp["Time Use"] + job.info["queue"] = kvp["Queue"] + + if status := cls.JOB_STATUS_CODES.get(kvp["S"], False): + job.status = status def update_status(self, job: Job) -> JobStatus: """Request info about the job and update its status. @@ -195,8 +209,8 @@ def update_status(self, job: Job) -> JobStatus: "Q": JobStatus.QUEUED, "R": JobStatus.RUNNING, } - if job.job_info["job_state"] in job_state: - job.status = job_state[job.job_info["job_state"]] + if job.info["job_state"] in job_state: + job.status = job_state[job.info["job_state"]] return job.status @staticmethod diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index 4ee7d5226..5c0c3e91d 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -32,7 +32,7 @@ def write(self, job: Job, path: Path) -> None: file.write(script) job.path = path - job.progress() + job.status = JobStatus.PREPARED @abstractmethod def _build_job_specification(self, job: Job) -> str: From 44088456209cddbedd9ee70234e4c2d63962c6da Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:27:29 +0200 Subject: [PATCH 23/54] adapt tests to new job info fetch --- tests/test_job.py | 80 +++-------------------------------------------- 1 file changed, 5 insertions(+), 75 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index f27cc7f33..3fd4106bd 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -14,40 +14,6 @@ from osekit.job.scheduler.scheduler import Scheduler -@pytest.mark.parametrize( - ("initial_status", "expected_status"), - [ - pytest.param( - JobStatus.UNPREPARED, - JobStatus.PREPARED, - id="unprepared_becomes_prepared", - ), - pytest.param( - JobStatus.PREPARED, - JobStatus.QUEUED, - id="prepared_becomes_queued", - ), - pytest.param(JobStatus.QUEUED, JobStatus.RUNNING, id="queued_becomes_running"), - pytest.param( - JobStatus.RUNNING, - JobStatus.COMPLETED, - id="running_becomes_completed", - ), - pytest.param( - JobStatus.COMPLETED, - JobStatus.COMPLETED, - id="completed_remains_completed", - ), - ], -) -def test_job_progress(initial_status: JobStatus, expected_status: JobStatus) -> None: - job = Job(script_path=Path()) - assert job.status == JobStatus.UNPREPARED - job._status = initial_status - job.progress() - assert job.status == expected_status - - def test_properties() -> None: script = Path("myscript.py") nb_nodes = 2 @@ -79,21 +45,6 @@ def test_properties() -> None: assert job.output_folder == Path("output") -def test_progress_transitions() -> None: - job = Job(Path("strawberry.py")) - assert job.status == JobStatus.UNPREPARED - for expected in ( - JobStatus.PREPARED, - JobStatus.QUEUED, - JobStatus.RUNNING, - JobStatus.COMPLETED, - ): - job.progress() - assert job.status == expected - job.progress() - assert job.status == JobStatus.COMPLETED - - def test_walltime_str_and_setter() -> None: job = Job(Path("bossanova.py")) for walltime in ("13:08:09", Timedelta(hours=13, minutes=8, seconds=9)): @@ -282,11 +233,11 @@ def test_pbs_update_info_no_job_id() -> None: pbs_scheduler = Pbs() job.job_id = None pbs_scheduler.update_info(job=job) - assert job.job_info is None + assert not job.info def test_pbs_update_info_parse_stdout(monkeypatch: pytest.MonkeyPatch) -> None: - job = Job(Path("fontaines.py")) + job = Job(script_path=Path("fontaines.py"), name="SwissArmyMan") job.job_id = "7137005" class Dummy: @@ -305,34 +256,13 @@ class Dummy: assert job.job_id == "7137005" assert job.name == "SwissArmyMan" assert job.status == JobStatus.RUNNING - assert job.job_info == { + assert job.info == { "user": "daniels", "time": "00:10:37", "queue": "jetski", } -def test_pbs_update_info_completed(monkeypatch: pytest.MonkeyPatch) -> None: - job = Job(Path("amok.py")) - job.job_id = "25022013" - job.job_info = {} - - class Dummy: - stdout = "" - stderr = "Atoms\nJob has finished\nFor peace" - - monkeypatch.setattr( - subprocess, - "run", - lambda *args, **kwargs: Dummy(), - ) - - scheduler = Pbs() - scheduler.update_info(job=job) - assert job.status == JobStatus.COMPLETED - assert job.job_info["job_state"] == "C" - - def test_pbs_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" @@ -388,12 +318,12 @@ def test_pbs_update_status(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N lambda job: None, ) - job.job_info = {"job_state": "Q"} + job.info = {"job_state": "Q"} job.job_id = "5129195" assert scheduler.update_status(job=job) == JobStatus.QUEUED assert job.status == JobStatus.QUEUED - job.job_info = {"job_state": "R"} + job.info = {"job_state": "R"} assert scheduler.update_status(job=job) == JobStatus.RUNNING assert job.status == JobStatus.RUNNING From f58a761c581c7a1c4b4260b6dfab3282d7bb721b Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:40:20 +0200 Subject: [PATCH 24/54] adapt pbs update_status to new job info fetching --- src/osekit/job/job.py | 6 ++++-- src/osekit/job/scheduler/pbs.py | 10 ---------- tests/test_job.py | 19 +++++++++---------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/osekit/job/job.py b/src/osekit/job/job.py index e6aeb0d7b..a55cbbe73 100644 --- a/src/osekit/job/job.py +++ b/src/osekit/job/job.py @@ -25,7 +25,8 @@ class JobStatus(Enum): ``PREPARED``: The job file has been written but not submitted. ``QUEUED``: The job has been queued. ``RUNNING``: The job is currently running. - ``COMPLETED``: The job has been completed. + ``SUSPENDED``: The job has been suspended or is held. + ``COMPLETED``: The job is exiting or has been completed. """ @@ -179,7 +180,8 @@ def status(self) -> JobStatus: ``PREPARED``: The job file has been written but not submitted. ``QUEUED``: The job has been queued. ``RUNNING``: The job is currently running. - ``COMPLETED``: The job has been completed. + ``SUSPENDED``: The job has been suspended or is held. + ``COMPLETED``: The job is exiting or has been completed. """ return self._status diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index a01e5cda1..f0d76bc7a 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -201,16 +201,6 @@ def update_status(self, job: Job) -> JobStatus: return job.status self.update_info(job=job) - - if job.status == JobStatus.COMPLETED: - return job.status - - job_state = { - "Q": JobStatus.QUEUED, - "R": JobStatus.RUNNING, - } - if job.info["job_state"] in job_state: - job.status = job_state[job.info["job_state"]] return job.status @staticmethod diff --git a/tests/test_job.py b/tests/test_job.py index 3fd4106bd..c42d85432 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -312,25 +312,24 @@ def test_pbs_update_status(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N job.path.write_text("prickly pear") assert scheduler.update_status(job=job) == JobStatus.PREPARED + def mock_update_info( + job: Job, + status: JobStatus, + *args: list, + **kwargs: dict, + ) -> None: + job.status = status + monkeypatch.setattr( scheduler, "update_info", - lambda job: None, + lambda job: mock_update_info(job=job, status=JobStatus.QUEUED), ) - job.info = {"job_state": "Q"} job.job_id = "5129195" assert scheduler.update_status(job=job) == JobStatus.QUEUED assert job.status == JobStatus.QUEUED - job.info = {"job_state": "R"} - assert scheduler.update_status(job=job) == JobStatus.RUNNING - assert job.status == JobStatus.RUNNING - - job.status = JobStatus.COMPLETED - assert scheduler.update_status(job=job) == JobStatus.COMPLETED - assert job.status == JobStatus.COMPLETED - def test_pbs_job_builder_write(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: called = {} From cba7b4ec848d348f0bd47d36ade029c28ef5737e Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:12:45 +0200 Subject: [PATCH 25/54] add Pbs.update_info() test where qstat returns nothing --- tests/test_job.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_job.py b/tests/test_job.py index c42d85432..53ad1589c 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -282,6 +282,28 @@ class Dummy: scheduler.update_info(job=job) +def test_pbs_update_info_without_qstat_output(monkeypatch: pytest.MonkeyPatch) -> None: + job = Job(Path("pompom.py")) + job.job_id = "17112014" + job.status = JobStatus.RUNNING + + class Dummy: + stdout = "" + stderr = "" + + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: Dummy(), + ) + + Pbs().update_info(job=job) + + # No qstat output shouldn't do anything + assert job.job_id == "17112014" + assert job.status == JobStatus.RUNNING + + def test_pbs_update_info_error(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" From 5135591d772c8992716a1147ed04232906a88d24 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:54:57 +0200 Subject: [PATCH 26/54] move pbs submit logic up to scheduler ABC --- src/osekit/job/scheduler/pbs.py | 60 ++++----------------------- src/osekit/job/scheduler/scheduler.py | 28 ++++++++++++- tests/test_job.py | 8 ++-- 3 files changed, 37 insertions(+), 59 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index f0d76bc7a..93f4a4ea9 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -34,6 +34,8 @@ class Pbs(Scheduler): "F": JobStatus.COMPLETED, } + SUBMIT_CMD = "qsub" + def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: """Initialize the PBS scheduler.""" self.queue = queue @@ -90,54 +92,6 @@ def _build_job_specification(self, job: Job) -> str: if value ) - def submit( - self, - job: Job, - dependencies: dict[str, Job | str | list[Job | str]] | None = None, - ) -> None: - """Submit the job to the scheduler. - - Parameters - ---------- - job: Job - Job to submit to the scheduler. - dependencies: dict[str, Job | str | list[Job|str]] - The dependencies of the submitted job. - The keys of the dictionary are the dependency types, - see https://help.altair.com/2022.1.0/PBS%20Professional/PBSReferenceGuide2022.1.pdf#page=151 - for the list of supported values. - The values are the other jobs (or their ID) ``job`` depends on - with the given dependency type. - If ``None``, the job is submitted without any dependency. - - """ - if self.update_status(job=job) is not JobStatus.PREPARED: - msg = "Job should be written before being submitted." - raise ValueError(msg) - - cmd = ["qsub"] - - if dependencies is not None: - dependency_str = self._build_dependency_string(dependencies) - if dependency_str: - cmd.extend(["-W", f"depend={dependency_str}"]) - - cmd.append(str(job.path)) - - try: - request = subprocess.run( - cmd, - capture_output=True, - text=True, - check=False, - ) - except subprocess.CalledProcessError as e: - msg = f"Submission failed with exit code {e.returncode}" - raise RuntimeError(msg) from e - - job.job_id = request.stdout.split(".", maxsplit=1)[0].strip() - self.update_status(job=job) - def update_info(self, job: Job) -> None: """Request info about the job and update it.""" if job.job_id is None: @@ -245,21 +199,21 @@ def _build_dependency_string( Examples -------- >>> Pbs._build_dependency_string({"afterok": "1234567"}) - 'afterok:1234567' + '-W depend=afterok:1234567' >>> Pbs._build_dependency_string({"afterok": ["1234567","4567891"]}) - 'afterok:1234567:4567891' + '-W depend=afterok:1234567:4567891' >>> from pathlib import Path >>> job = Job(Path()) >>> job._id = "7894561" >>> Pbs._build_dependency_string({"afterany":job}) - 'afterany:7894561' + '-W depend=afterany:7894561' >>> from pathlib import Path >>> job1 = Job(Path()) >>> job1._id = "7894561" >>> job2 = Job(Path()) >>> job2._id = "4839572" >>> Pbs._build_dependency_string({"afterany":[job1,job2]}) - 'afterany:7894561:4839572' + '-W depend=afterany:7894561:4839572' """ # Check that types are valid before submitting @@ -268,7 +222,7 @@ def _build_dependency_string( id_str = cls._parse_job_ids(dependencies=dependencies) - return ",".join( + return "-W depend=" + ",".join( f"{dependency_type}:{':'.join(ids)}" for dependency_type, ids in id_str.items() ) diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index 5c0c3e91d..13dad0c71 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -1,3 +1,4 @@ +import subprocess from abc import ABC, abstractmethod from pathlib import Path @@ -8,6 +9,7 @@ class Scheduler(ABC): """Abstract class representing a job scheduler.""" JOB_FILE_EXTENSION = "job" + SUBMIT_CMD = "" def write(self, job: Job, path: Path) -> None: """Write a job script to file. @@ -53,7 +55,6 @@ def _build_job_specification(self, job: Job) -> str: """ ... - @abstractmethod def submit( self, job: Job, @@ -74,7 +75,30 @@ def submit( If ``None``, the job is submitted without any dependency. """ - ... + if self.update_status(job=job) is not JobStatus.PREPARED: + msg = "Job should be written before being submitted." + raise ValueError(msg) + + cmd = [self.SUBMIT_CMD] + + if dependencies: + cmd.extend(self._build_dependency_string(dependencies=dependencies).split()) + + cmd.append(str(job.path)) + + try: + request = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + ) + except subprocess.CalledProcessError as e: + msg = f"Submission failed with exit code {e.returncode}" + raise RuntimeError(msg) from e + + job.job_id = request.stdout.split(".", maxsplit=1)[0].strip() + self.update_status(job=job) @abstractmethod def update_info(self, job: Job) -> None: diff --git a/tests/test_job.py b/tests/test_job.py index 53ad1589c..1ed412ad0 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -528,22 +528,22 @@ def test_pbs_validate_dependency_type() -> None: [ pytest.param( {"afterok": "1234567"}, - "afterok:1234567", + "-W depend=afterok:1234567", id="one_type_one_job", ), pytest.param( {"afterok": ["1234567", "2345678"]}, - "afterok:1234567:2345678", + "-W depend=afterok:1234567:2345678", id="one_type_multiple_jobs", ), pytest.param( {"afterok": "1234567", "afterany": "2345678"}, - "afterok:1234567,afterany:2345678", + "-W depend=afterok:1234567,afterany:2345678", id="multiple_types_one_job", ), pytest.param( {"afterok": ["1234567", "2345678"], "afterany": ["3456789", "4567890"]}, - "afterok:1234567:2345678,afterany:3456789:4567890", + "-W depend=afterok:1234567:2345678,afterany:3456789:4567890", id="multiple_types_multiple_jobs", ), ], From 43ce75ff5974e50c247106fa0d920ce68d6fb45c Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:32:06 +0200 Subject: [PATCH 27/54] move Pbs.update_info() method up in scheduler ABC --- src/osekit/job/scheduler/pbs.py | 30 +-------------- src/osekit/job/scheduler/scheduler.py | 53 +++++++++++++++++++++++++-- tests/test_job.py | 2 +- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 93f4a4ea9..44d877f8c 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -1,4 +1,3 @@ -import subprocess import typing from typing import Literal @@ -34,7 +33,8 @@ class Pbs(Scheduler): "F": JobStatus.COMPLETED, } - SUBMIT_CMD = "qsub" + SUBMIT_CMD: typing.ClassVar = "qsub" + INFO_CMD: typing.ClassVar = ["qstat", "-x"] def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: """Initialize the PBS scheduler.""" @@ -92,32 +92,6 @@ def _build_job_specification(self, job: Job) -> str: if value ) - def update_info(self, job: Job) -> None: - """Request info about the job and update it.""" - if job.job_id is None: - return - - try: - request = subprocess.run( - ["qstat", "-x", str(job.job_id)], - capture_output=True, - text=True, - check=False, - ) - stdout = request.stdout - except subprocess.CalledProcessError as e: - msg = f"Qstat failed with exit code {e.returncode}" - raise RuntimeError(msg) from e - - if not stdout: - err = request.stderr - if "Unknown Job Id" in err: - msg = f"Unknown Job Id {job.job_id}" - raise ValueError(msg) - return - - self._parse_info_str(job=job, info=stdout) - @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested qstat info string.""" diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index 13dad0c71..e9c7e48e7 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -1,4 +1,5 @@ import subprocess +import typing from abc import ABC, abstractmethod from pathlib import Path @@ -8,8 +9,9 @@ class Scheduler(ABC): """Abstract class representing a job scheduler.""" - JOB_FILE_EXTENSION = "job" - SUBMIT_CMD = "" + JOB_FILE_EXTENSION: typing.ClassVar = "job" + SUBMIT_CMD: typing.ClassVar = "" + INFO_CMD: typing.ClassVar = [] def write(self, job: Job, path: Path) -> None: """Write a job script to file. @@ -100,9 +102,52 @@ def submit( job.job_id = request.stdout.split(".", maxsplit=1)[0].strip() self.update_status(job=job) - @abstractmethod def update_info(self, job: Job) -> None: - """Request info about the job and update it.""" + """Request info about the job and update it. + + Parameters + ---------- + job: Job + Job for which to update the info. + + """ + if job.job_id is None: + return + + try: + request = subprocess.run( + [*self.INFO_CMD, str(job.job_id)], + capture_output=True, + text=True, + check=False, + ) + stdout = request.stdout + except subprocess.CalledProcessError as e: + msg = f"{self.INFO_CMD[0]} failed with exit code {e.returncode}" + raise RuntimeError(msg) from e + + if not stdout: + err = request.stderr + if err: + msg = f"{job.job_id}: {err}" + raise ValueError(msg) + return + + self._parse_info_str(job=job, info=stdout) + + @classmethod + @abstractmethod + def _parse_info_str(cls, job: Job, info: str) -> None: + """Parse the info string to update a job info. + + Parameters + ---------- + job: Job + Job of which the info is updated + info: str + Info string as returned by a request. + + """ ... @abstractmethod diff --git a/tests/test_job.py b/tests/test_job.py index 1ed412ad0..30ee789d3 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -319,7 +319,7 @@ def __init__(self) -> None: ) scheduler = Pbs() - with pytest.raises(RuntimeError, match="Qstat failed with exit code 5"): + with pytest.raises(RuntimeError, match=r"qstat failed.*code 5"): scheduler.update_info(job=job) From bb7c2395970ee8ed57d6e43d15dd168fe33718cb Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:51:21 +0200 Subject: [PATCH 28/54] move Pbs.update_status() logic up to scheduler ABC --- src/osekit/job/scheduler/pbs.py | 20 -------------------- src/osekit/job/scheduler/scheduler.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 44d877f8c..61df5798e 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -111,26 +111,6 @@ def _parse_info_str(cls, job: Job, info: str) -> None: if status := cls.JOB_STATUS_CODES.get(kvp["S"], False): job.status = status - def update_status(self, job: Job) -> JobStatus: - """Request info about the job and update its status. - - Returns - ------- - JobStatus: - The updated status of the job. - - """ - if job.job_id is None: - job.status = ( - JobStatus.PREPARED - if job.path and job.path.exists() - else JobStatus.UNPREPARED - ) - return job.status - - self.update_info(job=job) - return job.status - @staticmethod def _build_venv_string(job: Job) -> str: """Bash script used for activating the conda virtual environment.""" diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index e9c7e48e7..7408771d7 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -150,7 +150,6 @@ def _parse_info_str(cls, job: Job, info: str) -> None: """ ... - @abstractmethod def update_status(self, job: Job) -> JobStatus: """Request info about the job and update its status. @@ -160,7 +159,16 @@ def update_status(self, job: Job) -> JobStatus: The updated status of the job. """ - ... + if job.job_id is None: + job.status = ( + JobStatus.PREPARED + if job.path and job.path.exists() + else JobStatus.UNPREPARED + ) + return job.status + + self.update_info(job=job) + return job.status @staticmethod @abstractmethod From 639979339265c39458186cf37d5775f24365cae4 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:55:38 +0200 Subject: [PATCH 29/54] move Pbs._validate_dependency_type() logic up to scheduler ABC --- src/osekit/job/scheduler/pbs.py | 9 --------- src/osekit/job/scheduler/scheduler.py | 11 +++++++++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 61df5798e..dfe64354e 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -119,15 +119,6 @@ def _build_venv_string(job: Job) -> str: f"conda activate {job.venv_name}" ) - @classmethod - def _validate_dependency_type(cls, dependency_type: str) -> None: - if dependency_type not in cls._VALID_DEPENDENCY_TYPES: - msg = ( - f"Unsupported dependency type '{dependency_type}'.\n" - f"Expected one of:\n\t{'\n\t'.join(sorted(cls._VALID_DEPENDENCY_TYPES))}." - ) - raise ValueError(msg) - @classmethod def _build_dependency_string( cls, diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index 7408771d7..746b753c7 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -9,6 +9,7 @@ class Scheduler(ABC): """Abstract class representing a job scheduler.""" + _VALID_DEPENDENCY_TYPES: typing.ClassVar = frozenset() JOB_FILE_EXTENSION: typing.ClassVar = "job" SUBMIT_CMD: typing.ClassVar = "" INFO_CMD: typing.ClassVar = [] @@ -175,8 +176,14 @@ def update_status(self, job: Job) -> JobStatus: def _build_venv_string(job: Job) -> str: ... @classmethod - @abstractmethod - def _validate_dependency_type(cls, dependency_type: str) -> None: ... + def _validate_dependency_type(cls, dependency_type: str) -> None: + """Validate dependency types in the dependencies instruction.""" + if dependency_type not in cls._VALID_DEPENDENCY_TYPES: + msg = ( + f"Unsupported dependency type '{dependency_type}'.\n" + f"Expected one of:\n\t{'\n\t'.join(sorted(cls._VALID_DEPENDENCY_TYPES))}." + ) + raise ValueError(msg) @staticmethod def _parse_job_ids( From 6606a0315ed89fa8fa99dd21685b847249e527f6 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:27:21 +0200 Subject: [PATCH 30/54] extract pbs-specific ID parsing from qsub stdout --- src/osekit/job/scheduler/pbs.py | 5 +++++ src/osekit/job/scheduler/scheduler.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index dfe64354e..9e12f7380 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -92,6 +92,11 @@ def _build_job_specification(self, job: Job) -> str: if value ) + @staticmethod + def _parse_job_id(submit_output: str) -> str: + """Parse the output of the submit command.""" + return submit_output.split(".", maxsplit=1)[0].strip() + @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested qstat info string.""" diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index 746b753c7..27e4b3922 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -100,9 +100,15 @@ def submit( msg = f"Submission failed with exit code {e.returncode}" raise RuntimeError(msg) from e - job.job_id = request.stdout.split(".", maxsplit=1)[0].strip() + job.job_id = self._parse_job_id(submit_output=request.stdout) self.update_status(job=job) + @staticmethod + @abstractmethod + def _parse_job_id(submit_output: str) -> str: + """Parse the job id from the submit cmd output.""" + ... + def update_info(self, job: Job) -> None: """Request info about the job and update it. From 63c6cd007771c51e54b8bb5bef2ace3cf9175b07 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:49:43 +0200 Subject: [PATCH 31/54] simplify qstat info parsing --- src/osekit/job/scheduler/pbs.py | 21 ++++------ .../job_status_request_results/pbs.txt | 42 +++++++++++++++++-- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 9e12f7380..897bef4e8 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -34,7 +34,7 @@ class Pbs(Scheduler): } SUBMIT_CMD: typing.ClassVar = "qsub" - INFO_CMD: typing.ClassVar = ["qstat", "-x"] + INFO_CMD: typing.ClassVar = ["qstat", "-f"] def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: """Initialize the PBS scheduler.""" @@ -100,20 +100,15 @@ def _parse_job_id(submit_output: str) -> str: @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested qstat info string.""" - keys, _, values = info.splitlines() + kvp_lines = (line for line in info.splitlines() if "=" in line) + kvp_lines = dict(line.split("=", maxsplit=1) for line in kvp_lines) + kvp_lines = {k.strip(): v.strip() for k, v in kvp_lines.items()} - # Get keys order in the string - known_keys = ["Job id", "Name", "User", "Time Use", "S", "Queue"] - keys = sorted(known_keys, key=keys.index) + job.info["user"] = kvp_lines["Job_Owner"].split("@", maxsplit=1)[0] + job.info["time"] = kvp_lines.get("resources_used.walltime", "00:00:00") + job.info["queue"] = kvp_lines["queue"] - # Get the associated values - kvp = dict(zip(keys, values.split(), strict=True)) - - job.info["user"] = kvp["User"] - job.info["time"] = kvp["Time Use"] - job.info["queue"] = kvp["Queue"] - - if status := cls.JOB_STATUS_CODES.get(kvp["S"], False): + if status := cls.JOB_STATUS_CODES.get(kvp_lines["job_state"], False): job.status = status @staticmethod diff --git a/tests/_static/job_status_request_results/pbs.txt b/tests/_static/job_status_request_results/pbs.txt index c77af3532..8be9ac450 100644 --- a/tests/_static/job_status_request_results/pbs.txt +++ b/tests/_static/job_status_request_results/pbs.txt @@ -1,3 +1,39 @@ -Job id Name User Time Use S Queue ----------------- ---------------- ---------------- -------- - ----- -7137005.a24films0 SwissArmyMan daniels 00:10:37 R jetski +Job Id: 7137005.a24films0 + Job_Name = SwissArmyMan + Job_Owner = daniels@a24films.fr + job_state = R + queue = jetski + server = a24films0 + Checkpoint = u + ctime = Mon Sep 7 08:00:18 2026 + Error_Path = a24films0.a24films.fr:/home1/a24/filmls/SwissArmyMan.err + Hold_Types = n + Join_Path = oe + Keep_Files = n + Mail_Points = a + mtime = Mon Sep 7 08:00:18 2026 + Output_Path = a24films0.a24films.fr:/home1/a24/filmls/SwissArmyMan.out + Priority = 0 + qtime = Mon Sep 7 08:00:18 2026 + Rerunable = True + Resource_List.mem = 112gb + Resource_List.mpiprocs = 28 + Resource_List.ncpus = 24 + Resource_List.nodect = 1 + Resource_List.place = group=switch + Resource_List.select = 1:ncpus=24:mem=112g + Resource_List.walltime = 00:30:00 + resources_used.cpupercent = 1 + resources_used.cput = 01:01:25 + resources_used.mem = 40678964kb + resources_used.ncpus = 56 + resources_used.vmem = 3828570148kb + resources_used.walltime = 00:10:37 + substate = 10 + Variable_List = PBS_O_SYSTEM=Linux,PBS_O_SHELL=/bin/csh, + etime = Mon Sep 7 08:00:18 2026 + run_count = 15 + eligible_time = 00:00:00 + Submit_arguments = /home/a24films-exp/workspace/swiss/scheduled__2 + 026-09-07T07-00-00/army_man/run.sh + project = _pbs_project_default From 178a9bb1af829ed662c56682e3e287af642b100c Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:14:40 +0200 Subject: [PATCH 32/54] extract PBS-specific logic from Scheduler.update_info() --- src/osekit/job/scheduler/pbs.py | 38 ++++++++++++++++++++++++- src/osekit/job/scheduler/scheduler.py | 40 +++++++++++++++------------ 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/osekit/job/scheduler/pbs.py b/src/osekit/job/scheduler/pbs.py index 897bef4e8..64a85c350 100644 --- a/src/osekit/job/scheduler/pbs.py +++ b/src/osekit/job/scheduler/pbs.py @@ -1,3 +1,4 @@ +import subprocess import typing from typing import Literal @@ -34,7 +35,6 @@ class Pbs(Scheduler): } SUBMIT_CMD: typing.ClassVar = "qsub" - INFO_CMD: typing.ClassVar = ["qstat", "-f"] def __init__(self, queue: Literal["omp", "mpi"] = "omp") -> None: """Initialize the PBS scheduler.""" @@ -97,6 +97,42 @@ def _parse_job_id(submit_output: str) -> str: """Parse the output of the submit command.""" return submit_output.split(".", maxsplit=1)[0].strip() + @staticmethod + def _get_info(job: Job) -> str: + """Request information about a job. + + Parameters + ---------- + job: Job + Job for which the information is requested. + + Returns + ------- + str: + The information string, as returned by the ``qstat -f`` command. + + """ + try: + request = subprocess.run( + ["qstat", "-f", str(job.job_id)], + capture_output=True, + text=True, + check=False, + ) + if request.stdout: + return request.stdout + + except subprocess.CalledProcessError as e: + msg = f"qstat failed with exit code {e.returncode}" + raise RuntimeError(msg) from e + + err = request.stderr + if err: + msg = f"{job.job_id}: {err}" + raise ValueError(msg) + + return "" + @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested qstat info string.""" diff --git a/src/osekit/job/scheduler/scheduler.py b/src/osekit/job/scheduler/scheduler.py index 27e4b3922..d60c491a1 100644 --- a/src/osekit/job/scheduler/scheduler.py +++ b/src/osekit/job/scheduler/scheduler.py @@ -121,26 +121,30 @@ def update_info(self, job: Job) -> None: if job.job_id is None: return - try: - request = subprocess.run( - [*self.INFO_CMD, str(job.job_id)], - capture_output=True, - text=True, - check=False, - ) - stdout = request.stdout - except subprocess.CalledProcessError as e: - msg = f"{self.INFO_CMD[0]} failed with exit code {e.returncode}" - raise RuntimeError(msg) from e - - if not stdout: - err = request.stderr - if err: - msg = f"{job.job_id}: {err}" - raise ValueError(msg) + info = self._get_info(job=job) + if not info: return - self._parse_info_str(job=job, info=stdout) + self._parse_info_str(job=job, info=info) + + @staticmethod + @abstractmethod + def _get_info(job: Job) -> str: + """Request information about a job. + + Parameters + ---------- + job: Job + Job for which the information is requested. + + Returns + ------- + str: + The information string. + Depends on the scheduler. + + """ + ... @classmethod @abstractmethod From 35e30be111717d5cf4051e3c18e28780f3f72876 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:00:39 +0200 Subject: [PATCH 33/54] mock Scheduler methods rather than Pbs ones --- tests/test_job.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 30ee789d3..d1e416b97 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -188,7 +188,7 @@ def mock_update_status(self, job: Job) -> JobStatus: updated_jobs.append(job) return JobStatus.PREPARED - monkeypatch.setattr(Pbs, "update_status", mock_update_status) + monkeypatch.setattr(Scheduler, "update_status", mock_update_status) assert job.status == JobStatus.PREPARED pbs.submit(job=job) @@ -219,7 +219,7 @@ def __init__(self) -> None: def mock_update_status(self: Pbs, job: Job) -> JobStatus: return JobStatus.PREPARED - monkeypatch.setattr(Pbs, "update_status", mock_update_status) + monkeypatch.setattr(Scheduler, "update_status", mock_update_status) # Submit error should leave the job prepared: with pytest.raises(RuntimeError, match="Submission failed with exit code 5"): @@ -335,6 +335,7 @@ def test_pbs_update_status(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N assert scheduler.update_status(job=job) == JobStatus.PREPARED def mock_update_info( + self: Scheduler, job: Job, status: JobStatus, *args: list, @@ -343,9 +344,9 @@ def mock_update_info( job.status = status monkeypatch.setattr( - scheduler, + Scheduler, "update_info", - lambda job: mock_update_info(job=job, status=JobStatus.QUEUED), + lambda self, job: mock_update_info(self, job=job, status=JobStatus.QUEUED), ) job.job_id = "5129195" @@ -368,7 +369,7 @@ def mock_write(self: Pbs, job: Job, path: Path) -> None: job.status = JobStatus.PREPARED monkeypatch.setattr("osekit.job.builder.Job", DummyJob) - monkeypatch.setattr(Pbs, "write", mock_write) + monkeypatch.setattr(Scheduler, "write", mock_write) job_config = JobConfig( nb_nodes=2, @@ -457,8 +458,8 @@ def mock_update_status(self: Scheduler, job: Job) -> JobStatus: return job.status monkeypatch.setattr("osekit.job.job.Job", DummyJob) - monkeypatch.setattr(Pbs, "submit", mock_submit) - monkeypatch.setattr(Pbs, "update_status", mock_update_status) + monkeypatch.setattr(Scheduler, "submit", mock_submit) + monkeypatch.setattr(Scheduler, "update_status", mock_update_status) jobs = [ DummyJob(name="unprepared", status=JobStatus.UNPREPARED), @@ -498,7 +499,7 @@ def test_pbs_build_dependencies_string_validates_type( def mock_validate(dependency_type: str) -> None: validate_calls.append(dependency_type) - monkeypatch.setattr(Pbs, "_validate_dependency_type", mock_validate) + monkeypatch.setattr(Scheduler, "_validate_dependency_type", mock_validate) dependencies = {"afterok": "1234567", "afterany": ["2345678", "3456789"]} Pbs()._build_dependency_string( @@ -602,7 +603,7 @@ def mock_update_status(self: Pbs, job: Job) -> JobStatus: return JobStatus.PREPARED monkeypatch.setattr(subprocess, "run", fake_run) - monkeypatch.setattr(Pbs, "update_status", mock_update_status) + monkeypatch.setattr(Scheduler, "update_status", mock_update_status) scheduler.submit(job=job, dependencies={"afterok": "1234567"}) From beb076cf6643740c232e65907e7ef1515d0eb5cf Mon Sep 17 00:00:00 2001 From: Gautzilla Date: Mon, 24 Aug 2026 15:40:53 +0200 Subject: [PATCH 34/54] add slurm empty class --- src/osekit/job/scheduler/slurm.py | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/osekit/job/scheduler/slurm.py diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py new file mode 100644 index 000000000..9373484a2 --- /dev/null +++ b/src/osekit/job/scheduler/slurm.py @@ -0,0 +1,89 @@ +from pathlib import Path + +from osekit.job.job import Job, JobStatus +from osekit.job.scheduler.scheduler import Scheduler + + +class Slurm(Scheduler): + """Abstract class representing a job scheduler.""" + + JOB_FILE_EXTENSION = "slurm" + + def write(self, job: Job, path: Path) -> None: + """Write a job script to file. + + Parameters + ---------- + job: Job + Job of which to write the script. + path: Path + Path of the file in which the job script is written. + + """ + + def submit( + self, + job: Job, + dependency: Job | list[Job] | str | list[str] | None = None, + ) -> None: + """Submit the job to the scheduler. + + Parameters + ---------- + job: Job + Job to submit to the scheduler. + dependency: Job | list[Job] | str | None + Job dependency. Can be: + - A ``Job`` instance: will wait for that job to complete successfully + - A ``list[Job]``: will wait for all jobs to complete successfully + - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification + - ``None``: no dependency + + """ + + def update_info(self, job: Job) -> None: + """Request info about the job and update it.""" + + def update_status(self, job: Job) -> JobStatus: + """Request info about the job and update its status. + + Returns + ------- + JobStatus: + The updated status of the job. + + """ + + @staticmethod + def _build_venv_string(job: Job) -> str: + """Bash script used for activating the conda virtual environment.""" + + @classmethod + def _validate_dependency_type(cls, dependency_type: str) -> None: + pass + + @staticmethod + def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: + pass + + @classmethod + def _build_dependency_string( + cls, + dependency: str | Job | list[str] | list[Job], + dependency_type: str = "", + ) -> str: + """Build a job dependency string. + + Parameters + ---------- + dependency: Job | str + ``Job`` or job ID to depend on. + dependency_type: str + Type of dependency. + + Returns + ------- + str + Job dependency string. + + """ From 26b6409c84b32082619d8d562e20c171ae7107a4 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:30:58 +0200 Subject: [PATCH 35/54] implement Slurm._build_job_specifications() method --- src/osekit/job/scheduler/slurm.py | 51 +++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index 9373484a2..d911ca898 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -1,4 +1,4 @@ -from pathlib import Path +from typing import Literal from osekit.job.job import Job, JobStatus from osekit.job.scheduler.scheduler import Scheduler @@ -9,17 +9,56 @@ class Slurm(Scheduler): JOB_FILE_EXTENSION = "slurm" - def write(self, job: Job, path: Path) -> None: - """Write a job script to file. + def __init__(self, partition: Literal["cpu", "gpu", "ops"] = "cpu") -> None: + """Initialize the SLURM scheduler.""" + self.partition = partition + + @property + def partition(self) -> str: + """Partition in which the job will be submitted.""" + return self._partition + + @partition.setter + def partition(self, partition: Literal["omp", "mpi"]) -> None: + self._partition = partition + + def _build_job_specification(self, job: Job) -> str: + """Build the job specification string. Parameters ---------- job: Job - Job of which to write the script. - path: Path - Path of the file in which the job script is written. + The job for which to build the specifications. + + Returns + ------- + str: + Job specification string. + It includes the name of the job, the requested resources, + output log directories, etc. """ + specifications = { + "nodes": job.nb_nodes, + "cpus-per-task": job.ncpus, + "mem": job.mem, + "job-name": job.name, + "partition": self.partition, + "time": job.walltime_str, + "output": f"{job.output_folder}/{job.name}.out" + if job.output_folder + else None, + "error": f"{job.output_folder}/{job.name}.err" + if job.output_folder + else None, + } + + if job.ngpus is not None: + specifications["gpus"] = job.ngpus + + return "\n".join( + f"#SBATCH --{key}={value}" for key, value in specifications.items() if value + ) def submit( self, From 11d5e323b4a0eeb045696c4d62b093adc9e90ed7 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:33:10 +0200 Subject: [PATCH 36/54] =?UTF-8?q?implement=20Slurm.=5Fbuild=5Fvenv=5Fstrin?= =?UTF-8?q?g(=C3=83=C3=83)=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/osekit/job/scheduler/slurm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index d911ca898..d9655cf1f 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -96,6 +96,7 @@ def update_status(self, job: Job) -> JobStatus: @staticmethod def _build_venv_string(job: Job) -> str: """Bash script used for activating the conda virtual environment.""" + return f"module load cond\nconda activate {job.venv_name}" @classmethod def _validate_dependency_type(cls, dependency_type: str) -> None: From 87834c0a17009b8a70dceeb502ad6348b6c18195 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:00:23 +0200 Subject: [PATCH 37/54] add slurm build job specifications test --- src/osekit/job/scheduler/slurm.py | 4 ++-- tests/test_job.py | 34 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index d9655cf1f..b88cf112c 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -45,10 +45,10 @@ def _build_job_specification(self, job: Job) -> str: "job-name": job.name, "partition": self.partition, "time": job.walltime_str, - "output": f"{job.output_folder}/{job.name}.out" + "output": f"{job.output_folder / job.name}.out" if job.output_folder else None, - "error": f"{job.output_folder}/{job.name}.err" + "error": f"{job.output_folder / job.name}.err" if job.output_folder else None, } diff --git a/tests/test_job.py b/tests/test_job.py index d1e416b97..c6075d17d 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -12,6 +12,7 @@ from osekit.job.job import Job, JobStatus from osekit.job.scheduler.pbs import Pbs from osekit.job.scheduler.scheduler import Scheduler +from osekit.job.scheduler.slurm import Slurm def test_properties() -> None: @@ -81,6 +82,39 @@ def test_pbs_build_job_specifications() -> None: assert expected_specification in specifications +def test_slurm_build_job_specifications() -> None: + job = Job( + script_path=Path(), + config=JobConfig( + nb_nodes=2, + ncpus=3, + ngpus=1, + mem="16gb", + walltime=Timedelta(hours=2), + venv_name="cool_env", + ), + output_folder=Path(r"cool/folder"), + name="cool_job", + ) + + specifications = ( + Slurm(partition="gpu")._build_job_specification(job=job).splitlines() + ) + + for expected_specification in ( + "#SBATCH --job-name=cool_job", + "#SBATCH --partition=gpu", + "#SBATCH --nodes=2", + "#SBATCH --cpus-per-task=3", + "#SBATCH --mem=16gb", + "#SBATCH --gpus=1", + "#SBATCH --time=02:00:00", + f"#SBATCH --output={Path('cool/folder') / 'cool_job.out'}", + f"#SBATCH --error={Path('cool/folder') / 'cool_job.err'}", + ): + assert expected_specification in specifications + + def test_write_pbs(tmp_path: Path) -> None: script = tmp_path / "shpouik_shpouik.py" script.write_text("print('edgar')") From d8d41a003e4b340ea666ac430b24162e59eb3a31 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:05:51 +0200 Subject: [PATCH 38/54] add slurm venv str test --- src/osekit/job/scheduler/slurm.py | 2 +- tests/test_job.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index b88cf112c..a39868c11 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -96,7 +96,7 @@ def update_status(self, job: Job) -> JobStatus: @staticmethod def _build_venv_string(job: Job) -> str: """Bash script used for activating the conda virtual environment.""" - return f"module load cond\nconda activate {job.venv_name}" + return f"module load conda\nconda activate {job.venv_name}" @classmethod def _validate_dependency_type(cls, dependency_type: str) -> None: diff --git a/tests/test_job.py b/tests/test_job.py index c6075d17d..96c042a39 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -771,3 +771,12 @@ def patch_create_job(self: JobBuilder, **kwargs: str) -> None: ), ): assert job[0] == f"{job_name}_{idx}" + + +def test_slurm_venv_str() -> None: + slurm = Slurm() + job = Job(script_path=Path(), config=JobConfig(venv_name="cool_venv")) + venv_commands = slurm._build_venv_string(job=job).splitlines() + + assert venv_commands[0] == "module load conda" + assert venv_commands[1] == "conda activate cool_venv" From decef666ad6ae1566652a2c1f9a64893fb0f29ca Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:32:32 +0200 Subject: [PATCH 39/54] adapt slurm method to new dependencies system --- src/osekit/job/scheduler/slurm.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index a39868c11..b55b27d18 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -63,7 +63,7 @@ def _build_job_specification(self, job: Job) -> str: def submit( self, job: Job, - dependency: Job | list[Job] | str | list[str] | None = None, + dependencies: dict[str, Job | str | list[Job | str]] | None = None, ) -> None: """Submit the job to the scheduler. @@ -71,12 +71,13 @@ def submit( ---------- job: Job Job to submit to the scheduler. - dependency: Job | list[Job] | str | None - Job dependency. Can be: - - A ``Job`` instance: will wait for that job to complete successfully - - A ``list[Job]``: will wait for all jobs to complete successfully - - A ``str``: job ID (e.g., ``"12345.datarmor"``) or dependency specification - - ``None``: no dependency + dependencies: dict[str, Job | str | list[Job|str]] + The dependencies of the submitted job. + The keys of the dictionary are the dependency types, + that are proper to the scheduler. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. + If ``None``, the job is submitted without any dependency. """ @@ -109,17 +110,19 @@ def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: @classmethod def _build_dependency_string( cls, - dependency: str | Job | list[str] | list[Job], - dependency_type: str = "", + dependencies: dict[str, Job | str | list[Job | str]], ) -> str: """Build a job dependency string. Parameters ---------- - dependency: Job | str - ``Job`` or job ID to depend on. - dependency_type: str - Type of dependency. + dependencies: dict[str, Job | str | list[Job|str]] + The dependencies of the submitted job. + The keys of the dictionary are the dependency types, + that are proper to the scheduler. + The values are the other jobs (or their ID) ``job`` depends on + with the given dependency type. + If ``None``, the job is submitted without any dependency. Returns ------- From 72ec8161def857d5ca8e984491ed36fce890e64b Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:22:55 +0200 Subject: [PATCH 40/54] add slurm status request result file --- tests/_static/job_status_request_results/slurm.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/_static/job_status_request_results/slurm.txt diff --git a/tests/_static/job_status_request_results/slurm.txt b/tests/_static/job_status_request_results/slurm.txt new file mode 100644 index 000000000..7dfdca07a --- /dev/null +++ b/tests/_static/job_status_request_results/slurm.txt @@ -0,0 +1,2 @@ + JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 7137005 jetski SwissArmyMan daniels R 10:37 1 compute-114-9 From 376e3cf0b365ffbe144dd4223bbb53bf55e46788 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:58:13 +0200 Subject: [PATCH 41/54] add Slurm.SUBMIT_CMD attribute --- src/osekit/job/scheduler/slurm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index b55b27d18..4f3446db7 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -8,6 +8,7 @@ class Slurm(Scheduler): """Abstract class representing a job scheduler.""" JOB_FILE_EXTENSION = "slurm" + SUBMIT_CMD = "sbatch" def __init__(self, partition: Literal["cpu", "gpu", "ops"] = "cpu") -> None: """Initialize the SLURM scheduler.""" From 301486fde5182dadf6f8f3304cf04ee254adf74c Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:01:56 +0200 Subject: [PATCH 42/54] remove Slurm.submit() method that doesnt need overriding anymore --- src/osekit/job/scheduler/slurm.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index 4f3446db7..d8f4c4053 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -61,27 +61,6 @@ def _build_job_specification(self, job: Job) -> str: f"#SBATCH --{key}={value}" for key, value in specifications.items() if value ) - def submit( - self, - job: Job, - dependencies: dict[str, Job | str | list[Job | str]] | None = None, - ) -> None: - """Submit the job to the scheduler. - - Parameters - ---------- - job: Job - Job to submit to the scheduler. - dependencies: dict[str, Job | str | list[Job|str]] - The dependencies of the submitted job. - The keys of the dictionary are the dependency types, - that are proper to the scheduler. - The values are the other jobs (or their ID) ``job`` depends on - with the given dependency type. - If ``None``, the job is submitted without any dependency. - - """ - def update_info(self, job: Job) -> None: """Request info about the job and update it.""" From 625bcde7cda01a97ddb9094f83fd43e1288025fc Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:49:19 +0200 Subject: [PATCH 43/54] implement Slurm.update_info() method --- src/osekit/job/scheduler/slurm.py | 45 +++++++++++++++++++++++++++---- tests/test_job.py | 28 +++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index d8f4c4053..04fac6fc4 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -1,3 +1,4 @@ +import typing from typing import Literal from osekit.job.job import Job, JobStatus @@ -7,8 +8,16 @@ class Slurm(Scheduler): """Abstract class representing a job scheduler.""" - JOB_FILE_EXTENSION = "slurm" - SUBMIT_CMD = "sbatch" + JOB_FILE_EXTENSION: typing.ClassVar = "slurm" + INFO_CMD: typing.ClassVar = ["squeue", "--jobs"] + SUBMIT_CMD: typing.ClassVar = "sbatch" + JOB_STATUS_CODES: typing.ClassVar = { + "PD": JobStatus.QUEUED, + "R": JobStatus.RUNNING, + "S": JobStatus.SUSPENDED, + "CG": JobStatus.COMPLETED, + "CD": JobStatus.COMPLETED, + } def __init__(self, partition: Literal["cpu", "gpu", "ops"] = "cpu") -> None: """Initialize the SLURM scheduler.""" @@ -61,9 +70,6 @@ def _build_job_specification(self, job: Job) -> str: f"#SBATCH --{key}={value}" for key, value in specifications.items() if value ) - def update_info(self, job: Job) -> None: - """Request info about the job and update it.""" - def update_status(self, job: Job) -> JobStatus: """Request info about the job and update its status. @@ -74,6 +80,35 @@ def update_status(self, job: Job) -> JobStatus: """ + @classmethod + def _parse_info_str(cls, job: Job, info: str) -> None: + """Parse the info from the requested squeue info string.""" + keys, values = info.splitlines() + + # Get keys order in the string + known_keys = [ + "JOBID", + "PARTITION", + "NAME", + "USER", + "ST", + "TIME", + "NODES", + "NODELIST(REASON)", + ] + keys = sorted(known_keys, key=keys.index) + + # Get the associated values + kvp = dict(zip(keys, values.split(), strict=True)) + + job.info["user"] = kvp["USER"] + job.info["time"] = kvp["TIME"] + job.info["partition"] = kvp["PARTITION"] + job.info["node_list"] = kvp["NODELIST(REASON)"] + + if status := cls.JOB_STATUS_CODES.get(kvp["ST"], False): + job.status = status + @staticmethod def _build_venv_string(job: Job) -> str: """Bash script used for activating the conda virtual environment.""" diff --git a/tests/test_job.py b/tests/test_job.py index 96c042a39..42430c0b7 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -297,6 +297,34 @@ class Dummy: } +def test_slurm_update_info_parse_stdout(monkeypatch: pytest.MonkeyPatch) -> None: + job = Job(script_path=Path("fontaines.py"), name="SwissArmyMan") + job.job_id = "7137005" + + class Dummy: + stdout = ( + Path(__file__).parent / "_static/job_status_request_results/slurm.txt" + ).read_text() + stderr = "" + + monkeypatch.setattr( + subprocess, + "run", + lambda *args, **kwargs: Dummy(), + ) + scheduler = Slurm() + scheduler.update_info(job=job) + assert job.job_id == "7137005" + assert job.name == "SwissArmyMan" + assert job.status == JobStatus.RUNNING + assert job.info == { + "user": "daniels", + "time": "10:37", + "partition": "jetski", + "node_list": "compute-114-9", + } + + def test_pbs_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" From b76a7f93c02636a801b7233d5a3cd914e302cc3f Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:52:16 +0200 Subject: [PATCH 44/54] remove Scheduler.update_status() override --- src/osekit/job/scheduler/slurm.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index 04fac6fc4..05e4304aa 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -70,16 +70,6 @@ def _build_job_specification(self, job: Job) -> str: f"#SBATCH --{key}={value}" for key, value in specifications.items() if value ) - def update_status(self, job: Job) -> JobStatus: - """Request info about the job and update its status. - - Returns - ------- - JobStatus: - The updated status of the job. - - """ - @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested squeue info string.""" From 88c7605d5089d1954ee23c6358eee3efa5f61b41 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:05:14 +0200 Subject: [PATCH 45/54] implement Slurm._build_dependency_string() --- src/osekit/job/scheduler/slurm.py | 63 ++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index 05e4304aa..2666aa39f 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -8,6 +8,17 @@ class Slurm(Scheduler): """Abstract class representing a job scheduler.""" + _VALID_DEPENDENCY_TYPES: typing.ClassVar = frozenset( + { + "after", + "afterany", + "afterburstbuffer", + "aftercorr", + "afternotok", + "afterok", + "singleton", + }, + ) JOB_FILE_EXTENSION: typing.ClassVar = "slurm" INFO_CMD: typing.ClassVar = ["squeue", "--jobs"] SUBMIT_CMD: typing.ClassVar = "sbatch" @@ -104,18 +115,12 @@ def _build_venv_string(job: Job) -> str: """Bash script used for activating the conda virtual environment.""" return f"module load conda\nconda activate {job.venv_name}" - @classmethod - def _validate_dependency_type(cls, dependency_type: str) -> None: - pass - - @staticmethod - def _validate_dependency(dependency: list[str] | list[Job]) -> list[str]: - pass - @classmethod def _build_dependency_string( cls, dependencies: dict[str, Job | str | list[Job | str]], + *, + instructions_or: bool = False, ) -> str: """Build a job dependency string. @@ -124,14 +129,52 @@ def _build_dependency_string( dependencies: dict[str, Job | str | list[Job|str]] The dependencies of the submitted job. The keys of the dictionary are the dependency types, - that are proper to the scheduler. + see https://slurm.schedmd.com/sbatch.html The values are the other jobs (or their ID) ``job`` depends on with the given dependency type. If ``None``, the job is submitted without any dependency. + instructions_or: bool, optional + If ``True``, the instructions in the ``dependencies`` list + are joined with a logical OR (``?`` character in Slurm). + If ``False``, the instructions are joined with a logical + AND (``,`` character in Slurm). Returns ------- str Job dependency string. - """ + Examples + -------- + >>> Slurm._build_dependency_string({"afterok": "1234567"}) + '-d afterok:1234567' + >>> Slurm._build_dependency_string({"afterok": ["1234567","4567891"]}) + '-d afterok:1234567:4567891' + >>> Slurm._build_dependency_string({"afterok": ["1234567","4567891"], "afterany":"7654321"}, instructions_or=True) + '-d afterok:1234567:4567891?afterany:7654321' + >>> from pathlib import Path + >>> job = Job(Path()) + >>> job._id = "7894561" + >>> Slurm._build_dependency_string({"afterany":job}) + '-d afterany:7894561' + >>> from pathlib import Path + >>> job1 = Job(Path()) + >>> job1._id = "7894561" + >>> job2 = Job(Path()) + >>> job2._id = "4839572" + >>> Slurm._build_dependency_string({"afterany":[job1,job2]}) + '-d afterany:7894561:4839572' + + """ # noqa: E501 + # Check that types are valid before submitting + for dependency_type in dependencies: + cls._validate_dependency_type(dependency_type=dependency_type) + + id_str = cls._parse_job_ids(dependencies=dependencies) + + logical_join_character = "?" if instructions_or else "," + + return "-d " + logical_join_character.join( + f"{dependency_type}:{':'.join(ids)}" + for dependency_type, ids in id_str.items() + ) From ecba34c33078dac5ab41c21e294d3c456a7ca528 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:32:17 +0200 Subject: [PATCH 46/54] implement Slurm._parse_job_id() --- src/osekit/job/scheduler/slurm.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index 2666aa39f..c9fbb5ef4 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -81,6 +81,28 @@ def _build_job_specification(self, job: Job) -> str: f"#SBATCH --{key}={value}" for key, value in specifications.items() if value ) + @staticmethod + def _parse_job_id(submit_output: str) -> str: + """Parse the output of the submit command. + + Parameters + ---------- + submit_output: str + stdout after a successful sbatch cmd. + + Returns + ------- + str: + ID of the submitted job. + + Examples + -------- + >>> Slurm._parse_job_id(submit_output="Submitted batch job 3647090") + '3647090' + + """ + return submit_output.removeprefix("Submitted batch job ") + @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested squeue info string.""" From 41b9690583c0b3b84d9a6e1ab72b9626e52f8d3f Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:41:53 +0200 Subject: [PATCH 47/54] strip \n from sbatch stdout in Slurm._parse_job_id --- src/osekit/job/scheduler/slurm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index c9fbb5ef4..2a4eb7b7a 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -83,7 +83,7 @@ def _build_job_specification(self, job: Job) -> str: @staticmethod def _parse_job_id(submit_output: str) -> str: - """Parse the output of the submit command. + r"""Parse the output of the submit command. Parameters ---------- @@ -97,11 +97,11 @@ def _parse_job_id(submit_output: str) -> str: Examples -------- - >>> Slurm._parse_job_id(submit_output="Submitted batch job 3647090") + >>> Slurm._parse_job_id(submit_output="Submitted batch job 3647090\n") '3647090' """ - return submit_output.removeprefix("Submitted batch job ") + return submit_output.removeprefix("Submitted batch job ").strip("\n") @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: From 45c82a228ec221e18ebe925d4c25766b0eaca7f2 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 12:02:25 +0200 Subject: [PATCH 48/54] simplify Slurm squeue stdout parsing --- src/osekit/job/scheduler/slurm.py | 50 +++++++++---------- .../job_status_request_results/slurm.txt | 2 - .../slurm_squeue.txt | 1 + tests/test_job.py | 3 +- 4 files changed, 28 insertions(+), 28 deletions(-) delete mode 100644 tests/_static/job_status_request_results/slurm.txt create mode 100644 tests/_static/job_status_request_results/slurm_squeue.txt diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index 2a4eb7b7a..b173f4853 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -20,7 +20,12 @@ class Slurm(Scheduler): }, ) JOB_FILE_EXTENSION: typing.ClassVar = "slurm" - INFO_CMD: typing.ClassVar = ["squeue", "--jobs"] + INFO_CMD: typing.ClassVar = [ + "squeue", + "--noheader", + '--format="%i|%P|%j|%u|%t|%M|%D|%R', + "--jobs", + ] SUBMIT_CMD: typing.ClassVar = "sbatch" JOB_STATUS_CODES: typing.ClassVar = { "PD": JobStatus.QUEUED, @@ -106,30 +111,25 @@ def _parse_job_id(submit_output: str) -> str: @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested squeue info string.""" - keys, values = info.splitlines() - - # Get keys order in the string - known_keys = [ - "JOBID", - "PARTITION", - "NAME", - "USER", - "ST", - "TIME", - "NODES", - "NODELIST(REASON)", - ] - keys = sorted(known_keys, key=keys.index) - - # Get the associated values - kvp = dict(zip(keys, values.split(), strict=True)) - - job.info["user"] = kvp["USER"] - job.info["time"] = kvp["TIME"] - job.info["partition"] = kvp["PARTITION"] - job.info["node_list"] = kvp["NODELIST(REASON)"] - - if status := cls.JOB_STATUS_CODES.get(kvp["ST"], False): + values = info.strip().split("|", maxsplit=7) + + ( + _job_id, + partition, + _name, + user, + status, + time, + nodes, + node_list, + ) = values + + job.info["user"] = user + job.info["time"] = time + job.info["partition"] = partition + job.info["node_list"] = node_list + + if status := cls.JOB_STATUS_CODES.get(status, False): job.status = status @staticmethod diff --git a/tests/_static/job_status_request_results/slurm.txt b/tests/_static/job_status_request_results/slurm.txt deleted file mode 100644 index 7dfdca07a..000000000 --- a/tests/_static/job_status_request_results/slurm.txt +++ /dev/null @@ -1,2 +0,0 @@ - JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) - 7137005 jetski SwissArmyMan daniels R 10:37 1 compute-114-9 diff --git a/tests/_static/job_status_request_results/slurm_squeue.txt b/tests/_static/job_status_request_results/slurm_squeue.txt new file mode 100644 index 000000000..4945e8902 --- /dev/null +++ b/tests/_static/job_status_request_results/slurm_squeue.txt @@ -0,0 +1 @@ +7137005|jetski|SwissArmyMan|daniels|R|10:37|1|compute-114-9 diff --git a/tests/test_job.py b/tests/test_job.py index 42430c0b7..b68e3828d 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -303,7 +303,8 @@ def test_slurm_update_info_parse_stdout(monkeypatch: pytest.MonkeyPatch) -> None class Dummy: stdout = ( - Path(__file__).parent / "_static/job_status_request_results/slurm.txt" + Path(__file__).parent + / "_static/job_status_request_results/slurm_squeue.txt" ).read_text() stderr = "" From c284bba93fc496531a661db64c6fa6a2dc486322 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 15:46:32 +0200 Subject: [PATCH 49/54] add slurm sacct request for completed jobs --- src/osekit/job/scheduler/slurm.py | 92 ++++++++++++++++--- .../slurm_sacct.txt | 1 + .../slurm_squeue.txt | 2 +- 3 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 tests/_static/job_status_request_results/slurm_sacct.txt diff --git a/src/osekit/job/scheduler/slurm.py b/src/osekit/job/scheduler/slurm.py index b173f4853..dfb65229f 100644 --- a/src/osekit/job/scheduler/slurm.py +++ b/src/osekit/job/scheduler/slurm.py @@ -1,3 +1,4 @@ +import subprocess import typing from typing import Literal @@ -20,19 +21,13 @@ class Slurm(Scheduler): }, ) JOB_FILE_EXTENSION: typing.ClassVar = "slurm" - INFO_CMD: typing.ClassVar = [ - "squeue", - "--noheader", - '--format="%i|%P|%j|%u|%t|%M|%D|%R', - "--jobs", - ] SUBMIT_CMD: typing.ClassVar = "sbatch" JOB_STATUS_CODES: typing.ClassVar = { - "PD": JobStatus.QUEUED, - "R": JobStatus.RUNNING, - "S": JobStatus.SUSPENDED, - "CG": JobStatus.COMPLETED, - "CD": JobStatus.COMPLETED, + "PENDING": JobStatus.QUEUED, + "RUNNING": JobStatus.RUNNING, + "SUSPENDED": JobStatus.SUSPENDED, + "COMPLETING": JobStatus.RUNNING, + "COMPLETED": JobStatus.COMPLETED, } def __init__(self, partition: Literal["cpu", "gpu", "ops"] = "cpu") -> None: @@ -108,6 +103,75 @@ def _parse_job_id(submit_output: str) -> str: """ return submit_output.removeprefix("Submitted batch job ").strip("\n") + @classmethod + def _get_info(cls, job: Job) -> str: + """Request information about a job. + + Parameters + ---------- + job: Job + Job for which the information is requested. + + Returns + ------- + str: + The information string, as returned by the ``squeue`` command. + If the job is complete and doesn't appear in the queue, + the info is requested through the ``sacct`` command, parsed + by a specific parser, and an empty string is returned. + + """ + request = subprocess.run( + [ + "/usr/bin/squeue", + "--jobs", + str(job.job_id), + "--noheader", + ( + "--format=" + '"%i|' # ID + "%P|" # Partition + "%j|" # Job name + "%u|" # User name + "%T|" # Status + "%M|" # Time + "%D" # Nodes nb + '|%R"' + ), # Nodes List (Reason) + ], + capture_output=True, + text=True, + check=False, + ) + + if info := request.stdout.strip(): + return info + + # The job doesn't appear in squeue anymore + request = subprocess.run( + [ + "/usr/bin/sacct", + "--jobs", + str(job.job_id), + "--allocations", + "--noheader", + "--parsable2", + "--format=JobID,Partition,JobName,User,State,Elapsed,NNodes,NodeList,Reason", + ], + capture_output=True, + text=True, + check=False, + ) + + if acct_info := request.stdout.strip(): + return acct_info + + if error := request.stderr.strip(): + msg = f"{job.job_id}: {error}" + raise ValueError(msg) + + return "" + @classmethod def _parse_info_str(cls, job: Job, info: str) -> None: """Parse the info from the requested squeue info string.""" @@ -127,6 +191,12 @@ def _parse_info_str(cls, job: Job, info: str) -> None: job.info["user"] = user job.info["time"] = time job.info["partition"] = partition + + # If info is fetched by sacct, node_list could contain + # both the node name and the reason, sill separated by a "|" + if "|" in node_list: + node, reason = node_list.split("|", maxsplit=1) + node_list = f"{node} ({reason})" job.info["node_list"] = node_list if status := cls.JOB_STATUS_CODES.get(status, False): diff --git a/tests/_static/job_status_request_results/slurm_sacct.txt b/tests/_static/job_status_request_results/slurm_sacct.txt new file mode 100644 index 000000000..3c290d728 --- /dev/null +++ b/tests/_static/job_status_request_results/slurm_sacct.txt @@ -0,0 +1 @@ +7137005|jetski|SwissArmyMan|daniels|COMPLETED|00:10:37|1|compute-114-9|Dependency diff --git a/tests/_static/job_status_request_results/slurm_squeue.txt b/tests/_static/job_status_request_results/slurm_squeue.txt index 4945e8902..525554a0c 100644 --- a/tests/_static/job_status_request_results/slurm_squeue.txt +++ b/tests/_static/job_status_request_results/slurm_squeue.txt @@ -1 +1 @@ -7137005|jetski|SwissArmyMan|daniels|R|10:37|1|compute-114-9 +7137005|jetski|SwissArmyMan|daniels|RUNNING|10:37|1|compute-114-9 From f6daf05ccc455265837dbe127bafb303cd13b426 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 15:53:10 +0200 Subject: [PATCH 50/54] add Slurm._parse_job_id test --- tests/test_job.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_job.py b/tests/test_job.py index b68e3828d..0ceed0ae0 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -809,3 +809,9 @@ def test_slurm_venv_str() -> None: assert venv_commands[0] == "module load conda" assert venv_commands[1] == "conda activate cool_venv" + + +def test_slurm_parse_job_id() -> None: + submit_output = "Submitted batch job 3647090\n" + job_id = "3647090" + assert Slurm()._parse_job_id(submit_output=submit_output) == job_id From 26190bdcb1bdc752c34aa5c0f507291dbed80f81 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:16:12 +0200 Subject: [PATCH 51/54] add Slurm get_info test for completed jobs --- .../slurm_sacct.txt | 2 +- tests/test_job.py | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/_static/job_status_request_results/slurm_sacct.txt b/tests/_static/job_status_request_results/slurm_sacct.txt index 3c290d728..28f6bf2a2 100644 --- a/tests/_static/job_status_request_results/slurm_sacct.txt +++ b/tests/_static/job_status_request_results/slurm_sacct.txt @@ -1 +1 @@ -7137005|jetski|SwissArmyMan|daniels|COMPLETED|00:10:37|1|compute-114-9|Dependency +7137005|jetski|SwissArmyMan|daniels|COMPLETED|10:37|1|compute-114-9|Dependency diff --git a/tests/test_job.py b/tests/test_job.py index 0ceed0ae0..c62df1877 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -326,6 +326,42 @@ class Dummy: } +def test_slurm_get_info_completed_job(monkeypatch: pytest.MonkeyPatch) -> None: + job = Job(script_path=Path("fontaines.py"), name="SwissArmyMan") + job.job_id = "7137005" + + class EmptyRequest: + def __init__(self) -> None: + self.stdout = "" + self.stderr = "" + + class DummySacctRequest: + def __init__(self) -> None: + self.stdout = ( + Path(__file__).parent + / "_static/job_status_request_results/slurm_sacct.txt" + ).read_text() + self.stderr = "" + + def mock_run(*args, **kwargs) -> EmptyRequest | DummySacctRequest: + cmd = args[0][0] + return EmptyRequest() if "squeue" in cmd else DummySacctRequest() + + monkeypatch.setattr(subprocess, "run", mock_run) + + scheduler = Slurm() + scheduler.update_info(job=job) + assert job.job_id == "7137005" + assert job.name == "SwissArmyMan" + assert job.status == JobStatus.COMPLETED + assert job.info == { + "user": "daniels", + "time": "10:37", + "partition": "jetski", + "node_list": "compute-114-9 (Dependency)", + } + + def test_pbs_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" From 7469fb591aed4e4b88384dbca87a9e4c4acd4766 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:33:20 +0200 Subject: [PATCH 52/54] add slurm get_info() error test --- tests/test_job.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_job.py b/tests/test_job.py index c62df1877..d4ac8d4aa 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -362,6 +362,31 @@ def mock_run(*args, **kwargs) -> EmptyRequest | DummySacctRequest: } +def test_slurm_get_info_error(monkeypatch: pytest.MonkeyPatch) -> None: + job = Job(script_path=Path("fontaines.py"), name="SwissArmyMan") + job.job_id = "7137005" + + class EmptyRequest: + def __init__(self) -> None: + self.stdout = "" + self.stderr = "" + + class DummySacctRequestError: + def __init__(self) -> None: + self.stdout = "" + self.stderr = "timeout" + + def mock_run(*args, **kwargs) -> EmptyRequest | DummySacctRequestError: + cmd = args[0][0] + return EmptyRequest() if "squeue" in cmd else DummySacctRequestError() + + monkeypatch.setattr(subprocess, "run", mock_run) + + scheduler = Slurm() + with pytest.raises(ValueError, match=r"7137005.*timeout"): + scheduler.update_info(job=job) + + def test_pbs_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" From 260a14cc7b082b143e84a343ae42425020f8fba2 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:36:33 +0200 Subject: [PATCH 53/54] add slurm get_info() test with no output --- tests/test_job.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_job.py b/tests/test_job.py index d4ac8d4aa..88be071f9 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -387,6 +387,35 @@ def mock_run(*args, **kwargs) -> EmptyRequest | DummySacctRequestError: scheduler.update_info(job=job) +def test_slurm_get_info_no_output(monkeypatch: pytest.MonkeyPatch) -> None: + job = Job(script_path=Path("fontaines.py"), name="SwissArmyMan") + job.job_id = "7137005" + job.status = JobStatus.PREPARED + + class EmptyRequest: + def __init__(self) -> None: + self.stdout = "" + self.stderr = "" + + class DummySacctRequest: + def __init__(self) -> None: + self.stdout = "" + self.stderr = "" + + def mock_run(*args, **kwargs) -> EmptyRequest | DummySacctRequest: + cmd = args[0][0] + return EmptyRequest() if "squeue" in cmd else DummySacctRequest() + + monkeypatch.setattr(subprocess, "run", mock_run) + + scheduler = Slurm() + scheduler.update_info(job=job) # Shouldn't raise + # Information should not have changed + assert job.job_id == "7137005" + assert job.status == JobStatus.PREPARED + assert job.info == {} + + def test_pbs_update_info_unknown_job_raises(monkeypatch: pytest.MonkeyPatch) -> None: job = Job(Path("pompom.py")) job.job_id = "17112014" From 2c8446273b9bcc669164002962f3262ab3de4545 Mon Sep 17 00:00:00 2001 From: Gautzilla <72027971+Gautzilla@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:43:36 +0200 Subject: [PATCH 54/54] add dependency_string slurm test --- tests/test_job.py | 97 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 18 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 88be071f9..409d29d0f 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -677,6 +677,25 @@ def test_pbs_validate_dependency_type() -> None: assert e.match(supported) +def id_to_job(job_id: str | list[str]) -> Job | list[Job]: + """Convert a Job ID ``job_id`` to a Job object with an ID of ``job_id`` + + If ``job_id`` is a list, converts the list of job_ids to a list of jobs + with the given IDs.""" + if isinstance(job_id, str): + job = Job(Path()) + job._id = job_id + job.status = JobStatus.QUEUED + return job + output = [] + for j_id in job_id: + job = Job(Path()) + job._id = j_id + job.status = JobStatus.QUEUED + output.append(job) + return output + + @pytest.mark.parametrize( ("dependencies", "expected"), [ @@ -710,28 +729,70 @@ def test_pbs_build_dependencies_string( assert Pbs()._build_dependency_string(dependencies=dependencies) == expected # %% Dependencies string from Job instances - def id_to_job(job_id: str | list[str]) -> Job | list[Job]: - """Convert a Job ID ``job_id`` to a Job object with an ID of ``job_id`` - - If ``job_id`` is a list, converts the list of job_ids to a list of jobs - with the given IDs.""" - if isinstance(job_id, str): - job = Job(Path()) - job._id = job_id - job.status = JobStatus.QUEUED - return job - output = [] - for j_id in job_id: - job = Job(Path()) - job._id = j_id - job.status = JobStatus.QUEUED - output.append(job) - return output - dependencies = {key: id_to_job(value) for key, value in dependencies.items()} assert Pbs()._build_dependency_string(dependencies=dependencies) == expected +@pytest.mark.parametrize( + ("dependencies", "instructions_or", "expected"), + [ + pytest.param( + {"afterok": "1234567"}, + False, + "-d afterok:1234567", + id="one_type_one_job", + ), + pytest.param( + {"afterok": ["1234567", "2345678"]}, + False, + "-d afterok:1234567:2345678", + id="one_type_multiple_jobs", + ), + pytest.param( + {"afterok": "1234567", "afterany": "2345678"}, + False, + "-d afterok:1234567,afterany:2345678", + id="multiple_types_one_job", + ), + pytest.param( + {"afterok": ["1234567", "2345678"], "afterany": ["3456789", "4567890"]}, + False, + "-d afterok:1234567:2345678,afterany:3456789:4567890", + id="multiple_types_multiple_jobs", + ), + pytest.param( + {"afterok": ["1234567", "2345678"], "afterany": ["3456789", "4567890"]}, + True, + "-d afterok:1234567:2345678?afterany:3456789:4567890", + id="logical_or", + ), + ], +) +def test_slurm_build_dependencies_string( + dependencies: dict[str, str | list[str]], + instructions_or: bool, + expected: str, +) -> None: + # %% Dependencies string from job IDs + assert ( + Slurm()._build_dependency_string( + dependencies=dependencies, + instructions_or=instructions_or, + ) + == expected + ) + + # %% Dependencies string from Job instances + dependencies = {key: id_to_job(value) for key, value in dependencies.items()} + assert ( + Slurm()._build_dependency_string( + dependencies=dependencies, + instructions_or=instructions_or, + ) + == expected + ) + + def test_submit_pbs_adds_dependency_flag( tmp_path: Path, monkeypatch: pytest.MonkeyPatch,