diff --git a/.agents/skills/creating-wdl-modules/SKILL.md b/.agents/skills/creating-wdl-modules/SKILL.md new file mode 100644 index 0000000..8c81107 --- /dev/null +++ b/.agents/skills/creating-wdl-modules/SKILL.md @@ -0,0 +1,80 @@ +--- +name: creating-wdl-modules +description: Use when creating, adding, scaffolding, bootstrapping, or updating a WDL module that wraps a bioinformatics command-line tool in this repository. +--- + +# Creating WDL Modules + +## Overview + +Create reusable WDL 1.4 tool modules only after proving upstream license and +container provenance. A module is complete when its typed tasks, documentation, +native Sprocket tests, and repository-wide CI-equivalent checks all pass. + +## Required order + +1. Inspect `README.md`, `CONTRIBUTING.md`, `.github/pull_request_template.md`, + `sprocket.toml`, CI, and existing modules. +2. Fix the exact upstream tool version and commands in scope. +3. Read [license-review.md](references/license-review.md). Complete the license + decision before creating module files. +4. Resolve a digest-pinned official upstream image. Use BioContainers only + through the documented fallback. +5. Read [module-convention.md](references/module-convention.md). Derive typed + task interfaces from the pinned version's per-subcommand CLI help, and + cross-check source to exclude only hidden developer flags. Represent every + finite public CLI choice set as a WDL enum during interface derivation. Look + up the tool on `bio.tools` before deciding whether the manifest has an ID. + Record upstream tool provenance using the current `tools` object fields. +6. Create or update the manifest, README, root WDL entrypoint, one WDL file per + public subcommand, one adjacent native test YAML per subcommand, + provenance-documented fixtures, and shared `test/fixtures`. Give every WDL + document a two-sentence module documentation comment that states its purpose + and names every task and user-defined type it exports. Fixtures may be + original or deterministically generated with the pinned tool. +7. Read [quality-gates.md](references/quality-gates.md). Run the targeted + module checks, then every repository-wide CI-equivalent check. +8. Report provenance evidence and the factual PR tools-table row. + +## Hard gates + +Stop rather than improvise when: + +- the upstream license is prohibited, missing, or ambiguous; +- neither the upstream project nor BioContainers publishes a suitable image; +- an immutable image digest cannot be verified; +- fixtures are copied or have unknown or undocumented provenance; +- authoritative docs do not support the proposed task interface; or +- formatting, linting, native tests, or CI still fail. + +## Non-negotiable rules + +| Pressure | Required response | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| "Ship now; review the license later" | License review precedes scaffolding. | +| "The tag is specific enough" | WDL container must include the immutable sha256 digest; a bare tag is not pinned. | +| "Use any working image" | Only an official upstream image or BioContainers fallback—searched in that order and digest-pinned—is accepted. | +| "Tests can follow" | Every public task ships with passing `sprocket dev test` coverage before merge. | +| "I'll track tests in a follow-up issue" | No follow-up issue substitutes for `sprocket dev test` passing at merge time. | +| "A smoke test is enough for now" | Native `sprocket dev test` coverage is required; minimal smoke tests are not equivalent. | +| "`extra_args` supports that option" | Every public functional option in the pinned CLI has a typed input; `extra_args` only covers options absent from that CLI. | +| "Silence the lint or change expected output" | Fix the cause; do not weaken the gate without evidence of an incorrect gate. | +| "A `String` is simpler for these choices" | Represent every finite public CLI choice set as a WDL enum with values matching the pinned CLI tokens. | +| "Spell out every enum value" | Omit an explicit enum value when it is identical to the choice name; assign a value only when the pinned CLI token differs. | +| "One WDL file is simpler" | Put each public subcommand in its own folder and WDL file, then selectively re-export it from the module entrypoint. | +| "`meta` and `parameter_meta` are still needed" | Omit them when Sprocket `##` comments document the same task, inputs, and outputs. | +| "The README already describes the module" | Give every WDL document a two-sentence `##` module comment that states its purpose and names every task and user-defined type it exports. | +| "Generated fixtures need no provenance" | Document the pinned tool version, exact generation and transformation commands, seed, and installation method. | +| "Backticks are cosmetic" | Enclose literals in backticks throughout prose documentation, manifest strings, WDL metadata, prose code comments, and runtime error messages. | + +## Completion contract + +Do not claim completion until the changed module's native tests and all +repository CI-equivalent commands pass. Do not check contributor legal +attestations or maintainer-only PR checkboxes on another person's behalf. + +## Cross-client note + +The canonical skill lives under `.agents/skills`. Claude uses the committed +`.claude/skills` symlink. Windows checkouts must enable Developer Mode and Git +symlink support or Claude cannot discover this repository skill. diff --git a/.agents/skills/creating-wdl-modules/references/license-review.md b/.agents/skills/creating-wdl-modules/references/license-review.md new file mode 100644 index 0000000..b50f83e --- /dev/null +++ b/.agents/skills/creating-wdl-modules/references/license-review.md @@ -0,0 +1,64 @@ +# License and Container Review + +## Evidence record + +Before writing module files, record: + +| Field | Required evidence | +| ------------ | ------------------------------------------------------- | +| Tool | Canonical upstream project name and URL | +| Version | Exact release tag or immutable commit | +| License | Full name and SPDX identifier | +| License file | Direct URL pinned to the selected tag or commit | +| Commands | Authoritative documentation for each wrapped command | +| Container | Registry, immutable digest, and publisher | + +Prefer the upstream repository's license file at the selected release. Package +metadata may corroborate it but does not replace available upstream evidence. +Repeat the review for every version update. + +## Policy decision + +Compare the verified license with `README.md`: + +- Continue only when the license is on the approved list. +- Stop for AGPL or another prohibited use-triggered license. +- Stop and present the evidence for maintainer review when the license is + unlisted, unclear, conflicting, or missing. + +Do not create a partial module before this decision. + +## Container decision + +1. Search the upstream project for an official image matching the exact selected + version and supported platform. +2. If no suitable official image exists, search BioContainers for the exact + upstream version and supported platform. +3. Resolve the selected image's OCI digest with: + + ```bash + docker buildx imagetools inspect + ``` + +4. Put `@sha256:` in WDL. +5. Verify the image executes the expected tool version through the module's + Sprocket tests. + +When BioContainers supplies the fallback, explain why no suitable official +upstream image was used in the module README. Stop if neither source is +available. An image from another third party and a mutable tag are not +fallbacks. + +## Pull-request handoff + +Provide this factual table: + +```markdown +| Tool | Version | License | SPDX | License file | Needs review? | +| ------------------------------- | :---------: | ------------------- | :------: | ------------------------------------------- | :-----------: | +| []() | `` | | `` | [Link]() | ☐ | +``` + +Leave `Needs review?` as `☐` for an approved license. Change it to `☒` only +for an unlisted license that the maintainer must review. Never check a +contributor's right-to-license attestation or any maintainer-only checkbox. diff --git a/.agents/skills/creating-wdl-modules/references/module-convention.md b/.agents/skills/creating-wdl-modules/references/module-convention.md new file mode 100644 index 0000000..056121b --- /dev/null +++ b/.agents/skills/creating-wdl-modules/references/module-convention.md @@ -0,0 +1,145 @@ +# Module Convention + +## Layout + +Use one root-level lowercase kebab-case directory per upstream tool: + +```text +/ +├── module.json +├── README.md +├── .wdl +├── / +│ ├── .wdl +│ └── test/ +│ └── .yaml +└── test/ + └── fixtures/ +``` + +Module trees may not contain symlinks. `.wdl` is the explicit module +entrypoint. For a tool with subcommands, put each public subcommand task in +`/.wdl` and selectively import it into the entrypoint +so consumers can write `import { } from `. Keep a tool with +no subcommands in the entrypoint. Do not add workflows. + +## Manifest + +`module.json` follows the schema identified by its `$schema` field. Engines +ignore unrecognized fields for forward compatibility, but repository manifests +use only standardized fields. A manifest must: + +- use the module license expression `MIT OR Apache-2.0`; +- set `entrypoint` to `.wdl`; +- set `repository` to the canonical repository URL; +- use a `tools` array whose entries contain required `name`, `version`, and + SPDX `license` strings, plus optional `url` and `ids` fields; +- search `bio.tools` by tool name and upstream URL before assigning an ID; +- encode identifiers as an `ids` array of registered CURIE strings, such as + `["biotools:"]`, only when the `bio.tools` record is an exact + upstream match; omit `ids` when no authoritative identifier exists; +- enclose code-like literals in backticks within prose fields such as + `description`; +- use `{}` for `dependencies` unless WDL imports another module; each declared + dependency uses either a local `path` or a `git` URL with exactly one of + `version`, `tag`, `branch`, or `commit`, plus an optional repository `path`; +- contain no comments, duplicate keys, or trailing commas. + +The module's release version and the wrapped tool's `tools[].version` serve +different purposes. Changing the wrapped tool in a way that changes expected +outputs requires a new module Git tag even though `module.json` has no +top-level version field. + +## WDL tasks + +Every document declares `version 1.4`. For tools with subcommands, name each +task after its subcommand, e.g., `filter`, so the module supports +`import { filter } from fq`. For tools without subcommands, use a concise +snake_case task name. + +Begin every WDL document with a Sprocket `##` module documentation comment +immediately before the `version` declaration. Its first sentence states the +document's purpose. Its second sentence names every task and user-defined type +that the document exports. Keep license, container, and fixture provenance in +the README rather than duplicating them in this comment. + +Treat the pinned executable's actual per-subcommand CLI help as the authoritative +option surface. Cross-check source code to identify flags hidden from help; README +omissions do not narrow required coverage. Every public functional option must have +an explicit typed input. Exclude only help, version, and hidden developer flags. +`extra_args` supports options absent from the pinned CLI, such as future upstream +additions; it never substitutes for a typed input for a current option. + +Every finite public CLI choice set uses a top-level WDL enum declared before all +tasks. Enum values exactly match the pinned CLI tokens. Free-form `String` is +reserved for genuinely open-ended input such as filenames, record identifiers, +and regular expressions. When a choice's value is identical to its name, omit +the redundant explicit value. Assign a value only when the pinned CLI token +differs from the WDL choice name. Extract enum values for use in Bash with +`value()`. + +For each task: + +- expose stable options as typed inputs; +- place `Array[String] extra_args = []` last; +- provide conservative overridable `cpu`, `memory`, `disk_gib`, and + digest-pinned `container` defaults; +- use `requirements`, never deprecated `runtime` or the `docker` alias; +- use a heredoc command beginning with `set -euo pipefail`; +- use WDL `env` declarations for string, file, directory, and argument-file + values that enter Bash; +- put `# shellcheck disable=SC2154` before `set -euo pipefail`; this is the only + default suppression and reconciles WDL runtime `env` exports with ShellCheck; +- do not begin input identifiers with `input`, which Sprocket rejects; +- serialize `extra_args` with `write_lines`, load them using `mapfile -t`, and + expand the Bash array as `"${extra_args[@]}"`; +- declare deterministic, exact output paths rather than broad globs; and +- use Sprocket `##` comments to document the task, every input, and every output; + do not add redundant `meta` or `parameter_meta` blocks. + +Use Sprocket documentation comments beginning with `##`. Place documentation +comments immediately before every task, user-defined type, enum choice or +struct member, input declaration, and output declaration. Use ordinary `#` +prose comments for private declarations because `sprocket dev doc` does not +publish them and Sprocket reports `##` there as `UnusedDocComments`. Keep +`meta` and `parameter_meta` because they remain part of the WDL task interface. +Enable documentation comments for `sprocket dev doc` with +`doc.with_doc_comments = true` in `sprocket.toml`. + +In manifest prose, `meta`, `parameter_meta`, documentation comments, prose +comments, and runtime error messages, enclose code-like literals in backticks. +This includes tool names, versions, task names, subcommand names, option flags, +file paths, identifiers, default values, enum values, and format suffixes such +as `.gz`. Quote shell error strings so backticks remain literal rather than +command substitutions. Syntactic directives such as +`# shellcheck disable=SC2154` are exempt where backticks would break the tool. + +`extra_args` preserves argument boundaries but remains a trusted-caller escape +hatch: the wrapper cannot decide whether the upstream tool treats an argument +as dangerous or invalid. + +## README + +Document: + +1. the wrapped tool, exact version, and upstream URL; +2. the upstream license and version-pinned license URL; +3. the container URI, digest, and publisher; +4. why BioContainers replaced an official upstream image, when applicable; +5. every public task and its material behavior, including enum types and their + available choices for any finite option set; and +6. a `Fixture provenance` section with the pinned tool version, installation + method, exact deterministic generation and transformation commands, seed, + and an explicit statement that the data is synthetic when fixtures are + generated; + and +7. `sprocket dev test `. + +Enclose code-like literals in backticks throughout README prose: tool names, +version strings, task names, option names, enum choice names and values, file +paths, and format suffixes. Do not backtick ordinary domain terms such as +"FASTQ" or "read pair". + +Do not copy upstream prose, code, binaries, or fixtures into the repository. +Fixtures must either be authored for the module or deterministically generated +by the pinned tool with documented provenance. diff --git a/.agents/skills/creating-wdl-modules/references/quality-gates.md b/.agents/skills/creating-wdl-modules/references/quality-gates.md new file mode 100644 index 0000000..55b87a3 --- /dev/null +++ b/.agents/skills/creating-wdl-modules/references/quality-gates.md @@ -0,0 +1,61 @@ +# Quality Gates + +## Test placement + +Sprocket matches each `.wdl` with an adjacent `test/.yaml`. For +tools with subcommands, keep each native test YAML beside its subcommand WDL +under `/test/.yaml`. Put shared minimal fixtures under +the module's root `test/fixtures`. Fixtures may be authored for the module or +generated deterministically by the pinned tool when the README records the +exact command, seed, version, installation method, transformation commands, and +synthetic-data status. Never use copied or unknown fixtures. +Sprocket writes execution state under ignored `test/runs` directories. + +Every public task needs at least one native test. Add cases or an input matrix +for meaningful option branches. Assert every supported semantic property: + +- `exit_code` or intentional `should_fail`; +- exact Boolean, string, integer, or float values; +- array/map length or emptiness and first/last array elements; +- file and directory basenames with `Name`; and +- meaningful stdout/stderr regular expressions. + +Do not use the schema's `custom` assertion until the repository's minimum +Sprocket release implements it. Do not add test-only WDL outputs merely to +inspect opaque file contents. + +## Development loop + +Format changed WDL: + +```bash +sprocket format overwrite +``` + +Run the smallest relevant checks: + +```bash +sprocket dev module verify --manifest-path +sprocket lint +sprocket dev test +``` + +Then reproduce CI: + +```bash +sprocket format check . +sprocket lint . +sprocket dev doc --check . + +shopt -s nullglob +for manifest in ./*/module.json; do + module_dir="$(dirname "$manifest")" + sprocket dev module verify --manifest-path "$module_dir" + sprocket dev test "$module_dir" +done +``` + +If a gate fails, preserve the evidence, fix the root cause, rerun the smallest +failing command, then rerun the complete gate. Do not add broad exceptions, +remove assertions, change expected output without upstream evidence, or weaken +CI. diff --git a/.agents/skills/creating-wdl-modules/templates/module.json.tmpl b/.agents/skills/creating-wdl-modules/templates/module.json.tmpl new file mode 100644 index 0000000..ba603a6 --- /dev/null +++ b/.agents/skills/creating-wdl-modules/templates/module.json.tmpl @@ -0,0 +1,17 @@ +{ + "$schema": "https://raw.githubusercontent.com/openwdl/wdl/8199dd7e3f17322e827225d6e6bc886dac139207/modules/schemas/module.schema.json", + "name": "__TOOL_DISPLAY_NAME__", + "license": "MIT OR Apache-2.0", + "description": "WDL tasks for `__TOOL_DISPLAY_NAME__`", + "repository": "https://github.com/stjude-rust-labs/modules", + "entrypoint": "__TOOL__.wdl", + "tools": [ + { + "name": "__TOOL_DISPLAY_NAME__", + "version": "__VERSION__", + "license": "__UPSTREAM_SPDX__", + "url": "__UPSTREAM_URL__" + } + ], + "dependencies": {} +} diff --git a/.agents/skills/creating-wdl-modules/templates/module.wdl.tmpl b/.agents/skills/creating-wdl-modules/templates/module.wdl.tmpl new file mode 100644 index 0000000..38ca08d --- /dev/null +++ b/.agents/skills/creating-wdl-modules/templates/module.wdl.tmpl @@ -0,0 +1,57 @@ +## Wraps `__tool__` for __tool_purpose__. +## Exports the `__operation__` task. +version 1.4 + +## __task_description__. +task __operation__ { + input { + ## __input_description__. + env File __input_variable__ + + ## __output_name_description__. + env String output_name = "__output_name__" + + ## Minimum CPU cores. + Int cpu = 1 + + ## Minimum memory with units. + String memory = "2 GiB" + + ## Minimum disk space in GiB. + Int disk_gib = 10 + + ## Digest-pinned official upstream image or BioContainers fallback. + String container = "__image_tag__@sha256:__digest__" + + ## Additional trusted arguments absent from the pinned CLI. + Array[String] extra_args = [] + } + + # File preserving trusted extra-argument boundaries. + env File extra_args_file = write_lines(extra_args) + + command <<< + # shellcheck disable=SC2154 + set -euo pipefail + + extra_args=() + mapfile -t extra_args < "$extra_args_file" + + __tool__ __operation__ \ + "${extra_args[@]}" \ + "$__input_variable__" \ + > "$output_name" + >>> + + output { + ## __output_description__. + File __output_variable__ = output_name + } + + requirements { + container: container + cpu: cpu + memory: memory + disks: disk_gib + } +} diff --git a/.agents/skills/creating-wdl-modules/templates/module.yaml.tmpl b/.agents/skills/creating-wdl-modules/templates/module.yaml.tmpl new file mode 100644 index 0000000..f61ab82 --- /dev/null +++ b/.agents/skills/creating-wdl-modules/templates/module.yaml.tmpl @@ -0,0 +1,10 @@ +__operation__: + - name: default + inputs: + __input_variable__: + - input.txt + assertions: + exit_code: 0 + outputs: + __output_variable__: + - Name: __output_name__ diff --git a/.claude/skills/creating-wdl-modules b/.claude/skills/creating-wdl-modules new file mode 120000 index 0000000..e2b3104 --- /dev/null +++ b/.claude/skills/creating-wdl-modules @@ -0,0 +1 @@ +../../.agents/skills/creating-wdl-modules \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8442ce9..c29d762 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,61 +1,37 @@ ## Description - + ## Tools referenced - + Needs review? — Change ☐ to ☒ if the license is not on the approved list and requires maintainer evaluation before this PR can be merged. --> -| Tool | Version | License | SPDX identifier | License file | Needs review? | -| ---------------------------------------------- | ------- | ----------- | --------------- | ------------------------------------------------------------- | ------------- | -| [ExampleTool](https://github.com/example/tool) | `1.2.3` | MIT License | `MIT` | [Link](https://github.com/example/tool/blob/main/LICENSE) | [X] | -| | | | | | | +| Tool | Version | License | SPDX | License file | Needs review? | +| :--------------------------------------------- | :-----: | :---------- | :---: | :-------------------------------------------------------- | :-----------: | +| [ExampleTool](https://github.com/example/tool) | `1.2.3` | MIT License | `MIT` | [Link](https://github.com/example/tool/blob/main/LICENSE) | ☐ | +| | | | | | ☐ | ## Checklist - - -- [ ] My contribution contains only original workflow logic. It does not include - third-party source code, binaries, container images, or copied third-party - scripts or similar materials. -- [ ] I have identified every external tool my module invokes and listed its - license in the table above. -- [ ] Every tool listed uses a license from the - [approved list](../README.md#approved-licenses), or I have flagged it for - review in the table above. -- [ ] No tool uses a license from the - [prohibited list](../README.md#prohibited-licenses) or any other license that - imposes obligations triggered by network interaction, SaaS deployment, or - similar use-based mechanisms. -- [ ] If this is a version bump, I have re-verified that the tool's license has - not changed in a way that is incompatible with this repository's policies. -- [ ] I have the right to license my contribution under the terms of this - repository (MIT or Apache-2.0). +PR submitters must complete this checklist. Pull requests will not be reviewed until every item is checked. If an item does not apply, check it to acknowledge that you have read and considered it. + +- [ ] My contribution contains only original workflow logic. It does not include third-party source code, binaries, container images, or copied third-party scripts or similar materials. +- [ ] I have identified every external tool my module invokes and listed its license in the table above. +- [ ] Every tool listed uses a license from the [approved list](../README.md#approved-licenses), or I have flagged it for review in the table above. +- [ ] No tool uses a license from the [prohibited list](../README.md#prohibited-licenses). +- [ ] If this is a version bump, I have re-verified that the tool's license has not changed in a way that is incompatible with this repository's policies (check yes if this is not a version bump). +- [ ] I have the right to distribute my contribution under both the MIT and Apache-2.0 licenses. ## Maintainer checklist - - -- [ ] I have verified that every tool listed in the tools table carries a - license from the [approved list](../README.md#approved-licenses), or have - reviewed and approved any license flagged for review. -- [ ] No tool uses a license from the - [prohibited list](../README.md#prohibited-licenses) or any other license that - imposes obligations triggered by network interaction, SaaS deployment, or - similar use-based mechanisms. +A repository maintainer must complete this checklist. If you are not a maintainer, leave every item unchecked. + +- [ ] I have verified that every tool listed in the tools table carries a license from the [approved list](../README.md#approved-licenses), or have reviewed and approved any license flagged for review. +- [ ] No tool uses a license from the [prohibited list](../README.md#prohibited-licenses). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d57de67 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: ci + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + quality: + name: quality + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v7.0.1 + + - name: Install Sprocket + uses: stjude-rust-labs/setup-sprocket@v1.0.1 + with: + branch: feat/module-porcelain + github-token: ${{ github.token }} + + - name: Verify portable skill + shell: bash + run: | + test -L .claude/skills/creating-wdl-modules + test "$(readlink .claude/skills/creating-wdl-modules)" = \ + "../../.agents/skills/creating-wdl-modules" + test -f .claude/skills/creating-wdl-modules/SKILL.md + + - name: Check WDL formatting + run: sprocket format check . + + - name: Lint WDL + run: sprocket lint . + + - name: Check WDL documentation + run: sprocket dev doc --check . + + - name: Test modules + shell: bash + run: | + shopt -s nullglob + for manifest in ./*/module.json; do + module_dir="$(dirname "$manifest")" + sprocket dev module verify --manifest-path "$module_dir" + sprocket dev test "$module_dir" + done diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..4ff7767 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,57 @@ +name: pages + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: build + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v7.0.1 + + - name: Install Sprocket + uses: stjude-rust-labs/setup-sprocket@v1.0.1 + with: + branch: feat/module-porcelain + github-token: ${{ github.token }} + + - name: Build documentation + run: sprocket dev doc --output _site --overwrite . + + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@v5.0.0 + with: + path: _site + + deploy: + name: deploy + if: github.ref == 'refs/heads/main' + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + concurrency: + group: pages + cancel-in-progress: false + + steps: + - name: Configure GitHub Pages + uses: actions/configure-pages@v6.0.0 + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5.0.0 diff --git a/.gitignore b/.gitignore index b61d7a7..54a62f8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,12 @@ miniwdl_run/ _LAST/ out/ +# Generated native-test runs +*/test/runs/ + +# Generated documentation +/docs/ + # Python __pycache__/ *.py[cod] diff --git a/fq/README.md b/fq/README.md new file mode 100644 index 0000000..e723151 --- /dev/null +++ b/fq/README.md @@ -0,0 +1,92 @@ +# fq + +This module wraps [`fq`](https://github.com/stjude-rust-labs/fq) `0.12.0`, +a command-line tool for manipulating FASTQ files. + +Upstream releases `fq` under the [MIT +License](https://github.com/stjude-rust-labs/fq/blob/v0.12.0/LICENSE.txt). +The module runs the BioContainers image +`quay.io/biocontainers/fq:0.12.0--h9ee0642_0@sha256:74b59572f1d05b4829b45b599ee04311c8b3acec510f3cfb879f23b4bbd2090b`. +This manifest is `linux/amd64`; Apple Silicon development requires container +emulation. + +## Tasks + +The root `fq.wdl` entrypoint selectively re-exports each task, so consumers can +import only the subcommands they use: + +```wdl +import { filter } from fq +``` + +### `filter` + +Filters one or more aligned FASTQ files by a bare-ID allowlist or sequence +regular expression. The wrapper requires at least one source and equal source +and output counts instead of accepting upstream's silent zip truncation. +Providing neither filter mode preserves `fq`'s byte-for-byte passthrough +behavior; passthrough output names should retain the input compression suffix. + +### `generate` + +Generates paired FASTQ files with configurable seed, record count, read length, +and output names. Output names ending in `.gz` enable upstream compression. + +### `lint` + +Validates a single FASTQ file or a paired set. All public validator controls +are typed as WDL enums. Invalid data fails the task in both `panic` and `log` +modes; `log` collects all detected errors before failing. Successful runs +return a validation log. + +`lint_mode` accepts `FqLintMode.Panic` (`"panic"`) or `FqLintMode.Log` +(`"log"`). `single_read_validation_level` and `paired_read_validation_level` +each accept `FqValidationLevel.Low` (`"low"`), `FqValidationLevel.Medium` +(`"medium"`), or `FqValidationLevel.High` (`"high"`). `disabled_validators` +accepts an array of `FqValidator` values: `S001` through `S007` and `P001`. + +### `subsample` + +Subsamples single or paired FASTQ files by exactly one probability or exact +record count. Read-two source and destination must be supplied together. +Probability bounds remain validated by upstream `fq`. + +## Testing + +### Fixture provenance + +The paired FASTQ fixtures are synthetic data generated by `fq` `0.12.0`. +Install the pinned version using the upstream README's manual method: + +```bash +git clone --depth 1 --branch v0.12.0 \ + https://github.com/stjude-rust-labs/fq.git /tmp/fq-v0.12.0 +cargo install --locked --path /tmp/fq-v0.12.0 +``` + +Regenerate the paired fixtures from the repository root with: + +```bash +fq generate --seed 7 --record-count 4 --read-length 8 \ + fq/test/fixtures/reads_R1.fastq \ + fq/test/fixtures/reads_R2.fastq +``` + +Derive `allowlist.txt` from the second generated read-one record and +`invalid.fastq` from the first generated read-one record with: + +```bash +awk 'NR == 5 { id = substr($0, 2); sub(/\/1([[:space:]].*)?$/, "", id); print id }' \ + fq/test/fixtures/reads_R1.fastq > fq/test/fixtures/allowlist.txt +awk 'NR <= 4 { if (NR == 2) $0 = "X" substr($0, 2); print }' \ + fq/test/fixtures/reads_R1.fastq > fq/test/fixtures/invalid.fastq +``` + +The first command records the bare identifier from the second read-one record. +The second command replaces the first sequence base with `X`. + +Run the module's native tests with: + +```bash +sprocket dev test fq +``` diff --git a/fq/filter/filter.wdl b/fq/filter/filter.wdl new file mode 100644 index 0000000..ddad06d --- /dev/null +++ b/fq/filter/filter.wdl @@ -0,0 +1,113 @@ +## Wraps `fq filter` to retain aligned FASTQ records by name or sequence pattern. +## Exports the `filter` task. +version 1.4 + +## Filters aligned FASTQ inputs with `fq filter`. +task filter { + input { + ## Aligned FASTQ source files. + Array[File] sources + + ## Output filenames aligned with `sources`. + Array[String] output_names + + ## Optional file containing bare record identifiers to retain. + File? names + + ## Optional regular expression for sequences to retain. + String? sequence_pattern + + ## Minimum CPU cores. + Int cpu = 1 + + ## Minimum memory with units. + String memory = "1 GiB" + + ## Minimum disk space in GiB. + Int disk_gib = 10 + + ## Digest-pinned `fq` `0.12.0` BioContainers image. + String container = "quay.io/biocontainers/fq:0.12.0--h9ee0642_0@sha256:74b59572f1d05b4829b45b599ee04311c8b3acec510f3cfb879f23b4bbd2090b" + + ## Trusted arguments absent from the pinned `fq` CLI. + Array[String] extra_args = [] + } + + # File containing one localized source path per line. + env File sources_file = write_lines(sources) + + # File containing one output filename per line. + env File output_names_file = write_lines(output_names) + + # Optional bare-ID allowlist path or an empty string. + env String names_value = if defined(names) + then "~{select_first([ + names, + ])}" + else "" + + # Optional sequence regular expression or an empty string. + env String sequence_pattern_value = select_first([ + sequence_pattern, + "", + ]) + + # File preserving trusted extra-argument boundaries. + env File extra_args_file = write_lines(extra_args) + + command <<< + # shellcheck disable=SC2154 + set -euo pipefail + export NO_COLOR=1 + + mapfile -t source_args < "$sources_file" + mapfile -t output_name_args < "$output_names_file" + mapfile -t extra_args < "$extra_args_file" + + if (( ${#source_args[@]} == 0 )); then + printf '%s\n' "\`filter\` requires at least one \`source\`" >&2 + exit 2 + fi + + if (( ${#source_args[@]} != ${#output_name_args[@]} )); then + printf '%s\n' "\`sources\` and \`output_names\` must have equal lengths" >&2 + exit 2 + fi + + if [[ -n "$names_value" && -n "$sequence_pattern_value" ]]; then + printf '%s\n' "\`names\` and \`sequence_pattern\` are mutually exclusive" >&2 + exit 2 + fi + + filter_args=() + if [[ -n "$names_value" ]]; then + filter_args+=(--names "$names_value") + fi + if [[ -n "$sequence_pattern_value" ]]; then + filter_args+=(--sequence-pattern "$sequence_pattern_value") + fi + + destination_args=() + for output_name in "${output_name_args[@]}"; do + destination_args+=(--dsts "$output_name") + done + + fq filter \ + "${filter_args[@]}" \ + "${destination_args[@]}" \ + "${extra_args[@]}" \ + "${source_args[@]}" + >>> + + output { + ## Filtered FASTQ outputs in input order. + Array[File] filtered_fastqs = output_names + } + + requirements { + container: container + cpu: cpu + memory: memory + disks: disk_gib + } +} diff --git a/fq/filter/test/filter.yaml b/fq/filter/test/filter.yaml new file mode 100644 index 0000000..ab02c0f --- /dev/null +++ b/fq/filter/test/filter.yaml @@ -0,0 +1,87 @@ +filter: + - name: passthrough + inputs: + sources: + - [reads_R1.fastq] + output_names: + - [passthrough.fastq] + assertions: + exit_code: 0 + stdout: + - "fq-filter start" + - "fq-filter end" + outputs: + filtered_fastqs: + - Length: 1 + - First: + Name: passthrough.fastq + - name: names_paired + inputs: + sources: + - [reads_R1.fastq, reads_R2.fastq] + output_names: + - [named_R1.fastq, named_R2.fastq] + names: + - allowlist.txt + assertions: + exit_code: 0 + stdout: + - "read 1 names" + outputs: + filtered_fastqs: + - Length: 2 + - First: + Name: named_R1.fastq + - Last: + Name: named_R2.fastq + - name: sequence_pattern + inputs: + sources: + - [reads_R1.fastq] + output_names: + - [pattern.fastq] + sequence_pattern: + - "^TC" + assertions: + exit_code: 0 + stdout: + - "sequence matches" + outputs: + filtered_fastqs: + - Length: 1 + - First: + Name: pattern.fastq + - name: rejects_empty_sources + inputs: + sources: + - [] + output_names: + - [] + assertions: + should_fail: true + stderr: + - "`filter` requires at least one `source`" + - name: rejects_mismatched_arrays + inputs: + sources: + - [reads_R1.fastq, reads_R2.fastq] + output_names: + - [only_one.fastq] + assertions: + should_fail: true + stderr: + - "`sources` and `output_names` must have equal lengths" + - name: rejects_both_filter_modes + inputs: + sources: + - [reads_R1.fastq] + output_names: + - [invalid.fastq] + names: + - allowlist.txt + sequence_pattern: + - "^TC" + assertions: + should_fail: true + stderr: + - "`names` and `sequence_pattern` are mutually exclusive" diff --git a/fq/fq.wdl b/fq/fq.wdl new file mode 100644 index 0000000..d1c44f5 --- /dev/null +++ b/fq/fq.wdl @@ -0,0 +1,15 @@ +## Wraps `fq` for filtering, generating, validating, and subsampling FASTQ data. +## Exports the `filter`, `generate`, `lint`, and `subsample` tasks and the `FqLintMode`, +## `FqValidationLevel`, and `FqValidator` types. +version 1.4 + +import { filter } from "filter/filter.wdl" +import { generate } from "generate/generate.wdl" +import { FqLintMode, FqValidationLevel, FqValidator, lint } from "lint/lint.wdl" +import { subsample } from "subsample/subsample.wdl" + +## Identifies the upstream `fq` release wrapped by the module. +struct FqRelease { + ## Exact upstream `fq` version. + String tool_version +} diff --git a/fq/generate/generate.wdl b/fq/generate/generate.wdl new file mode 100644 index 0000000..ba21695 --- /dev/null +++ b/fq/generate/generate.wdl @@ -0,0 +1,91 @@ +## Wraps `fq generate` to create seeded synthetic paired FASTQ data. +## Exports the `generate` task. +version 1.4 + +## Generates a random paired FASTQ dataset with `fq generate`. +task generate { + input { + ## Optional random-number seed. + Int? seed + + ## Number of read pairs to generate. + Int record_count = 10000 + + ## Bases per generated read. + Int read_length = 101 + + ## Read-one output filename; `.gz` enables compression. + env String r1_output_name = "generated_R1.fastq.gz" + + ## Read-two output filename; `.gz` enables compression. + env String r2_output_name = "generated_R2.fastq.gz" + + ## Minimum CPU cores. + Int cpu = 1 + + ## Minimum memory with units. + String memory = "1 GiB" + + ## Minimum disk space in GiB. + Int disk_gib = 10 + + ## Digest-pinned `fq` `0.12.0` BioContainers image. + String container = "quay.io/biocontainers/fq:0.12.0--h9ee0642_0@sha256:74b59572f1d05b4829b45b599ee04311c8b3acec510f3cfb879f23b4bbd2090b" + + ## Trusted arguments absent from the pinned `fq` CLI. + Array[String] extra_args = [] + } + + # Optional random-number seed or an empty string. + env String seed_value = if defined(seed) + then "~{select_first([ + seed, + ])}" + else "" + + # Record count serialized for `fq generate`. + env String record_count_value = "~{record_count}" + + # Read length serialized for `fq generate`. + env String read_length_value = "~{read_length}" + + # File preserving trusted extra-argument boundaries. + env File extra_args_file = write_lines(extra_args) + + command <<< + # shellcheck disable=SC2154 + set -euo pipefail + export NO_COLOR=1 + + mapfile -t extra_args < "$extra_args_file" + + option_args=( + --record-count "$record_count_value" + --read-length "$read_length_value" + ) + if [[ -n "$seed_value" ]]; then + option_args+=(--seed "$seed_value") + fi + + fq generate \ + "${option_args[@]}" \ + "${extra_args[@]}" \ + "$r1_output_name" \ + "$r2_output_name" + >>> + + output { + ## Generated read-one FASTQ. + File r1_fastq = r1_output_name + + ## Generated read-two FASTQ. + File r2_fastq = r2_output_name + } + + requirements { + container: container + cpu: cpu + memory: memory + disks: disk_gib + } +} diff --git a/fq/generate/test/generate.yaml b/fq/generate/test/generate.yaml new file mode 100644 index 0000000..c214a18 --- /dev/null +++ b/fq/generate/test/generate.yaml @@ -0,0 +1,22 @@ +generate: + - name: seeded + inputs: + seed: + - 7 + record_count: + - 4 + read_length: + - 8 + r1_output_name: + - generated_R1.fastq + r2_output_name: + - generated_R2.fastq + assertions: + exit_code: 0 + stdout: + - "generated 4 records" + outputs: + r1_fastq: + - Name: generated_R1.fastq + r2_fastq: + - Name: generated_R2.fastq diff --git a/fq/lint/lint.wdl b/fq/lint/lint.wdl new file mode 100644 index 0000000..5ce5912 --- /dev/null +++ b/fq/lint/lint.wdl @@ -0,0 +1,176 @@ +## Wraps `fq lint` to validate single or paired FASTQ data. +## Exports the `lint` task and the `FqLintMode`, `FqValidationLevel`, and `FqValidator` +## types. +version 1.4 + +## Controls whether `fq lint` stops at the first error or logs all errors. +enum FqLintMode[String] { + ## Stops at the first validation error. + Panic = "panic", + + ## Logs all validation errors before failing. + Log = "log", +} + +## Sets the highest validator level used by `fq lint`. +enum FqValidationLevel[String] { + ## Runs low-level validators. + Low = "low", + + ## Runs low- and medium-level validators. + Medium = "medium", + + ## Runs low-, medium-, and high-level validators. + High = "high", +} + +## Identifies an `fq lint` validator that callers may disable. +enum FqValidator { + ## Low-level check requiring the plus line to begin with `+`. + S001, + + ## Medium-level check allowing only case-insensitive `ACGTN` sequence characters. + S002, + + ## High-level check requiring the record name to begin with `@`. + S003, + + ## Low-level check requiring nonempty name, sequence, plus, and quality lines. + S004, + + ## High-level check requiring equal sequence and quality lengths. + S005, + + ## Medium-level check allowing only printable ASCII quality characters from `!` to `~`. + S006, + + ## High-level check rejecting duplicate read-one names in paired input. + S007, + + ## Medium-level check requiring paired records to have the same normalized name. + P001, +} + +## Validates single or paired FASTQ inputs with `fq lint`. +task lint { + input { + ## Read-one FASTQ input. + env File r1_fastq + + ## Optional read-two FASTQ input. + File? r2_fastq + + ## Whether `fq lint` stops at the first error or logs all errors. + FqLintMode lint_mode = FqLintMode.Panic + + ## Highest single-read validator level to run. + FqValidationLevel single_read_validation_level = FqValidationLevel.High + + ## Highest paired-read validator level to run. + FqValidationLevel paired_read_validation_level = FqValidationLevel.High + + ## Validator codes to disable. + Array[FqValidator] disabled_validators = [] + + ## Optional upstream record-name separator. + String? record_definition_separator + + ## Validation log filename. + env String report_name = "fq-lint.log" + + ## Minimum CPU cores. + Int cpu = 1 + + ## Minimum memory with units. + String memory = "1 GiB" + + ## Minimum disk space in GiB. + Int disk_gib = 10 + + ## Digest-pinned `fq` `0.12.0` BioContainers image. + String container = "quay.io/biocontainers/fq:0.12.0--h9ee0642_0@sha256:74b59572f1d05b4829b45b599ee04311c8b3acec510f3cfb879f23b4bbd2090b" + + ## Trusted arguments absent from the pinned `fq` CLI. + Array[String] extra_args = [] + } + + # CLI token for `lint_mode`. + env String lint_mode_value = value(lint_mode) + + # CLI token for `single_read_validation_level`. + env String single_read_validation_level_value = value(single_read_validation_level) + + # CLI token for `paired_read_validation_level`. + env String paired_read_validation_level_value = value(paired_read_validation_level) + + # Optional read-two path or an empty string. + env String r2_fastq_value = select_first([ + r2_fastq, + "", + ]) + + # Optional record-definition separator or an empty string. + env String separator_value = select_first([ + record_definition_separator, + "", + ]) + + # File containing one disabled validator code per line. + env File disabled_validators_file = write_lines(disabled_validators) + + # File preserving trusted extra-argument boundaries. + env File extra_args_file = write_lines(extra_args) + + command <<< + # shellcheck disable=SC2154 + set -euo pipefail + export NO_COLOR=1 + + mapfile -t disabled_validators < "$disabled_validators_file" + mapfile -t extra_args < "$extra_args_file" + + option_args=( + --lint-mode "$lint_mode_value" + --single-read-validation-level "$single_read_validation_level_value" + --paired-read-validation-level "$paired_read_validation_level_value" + ) + for validator in "${disabled_validators[@]}"; do + option_args+=(--disable-validator "$validator") + done + if [[ -n "$separator_value" ]]; then + option_args+=(--record-definition-separator "$separator_value") + fi + + positional_args=("$r1_fastq") + if [[ -n "$r2_fastq_value" ]]; then + positional_args+=("$r2_fastq_value") + fi + + status=0 + if fq lint \ + "${option_args[@]}" \ + "${extra_args[@]}" \ + "${positional_args[@]}" \ + > "$report_name" + then + : + else + status=$? + fi + + cat "$report_name" + exit "$status" + >>> + + output { + ## Validation log from a successful `fq lint` run. + File validation_report = report_name + } + + requirements { + container: container + cpu: cpu + memory: memory + disks: disk_gib + } +} diff --git a/fq/lint/test/lint.yaml b/fq/lint/test/lint.yaml new file mode 100644 index 0000000..488665f --- /dev/null +++ b/fq/lint/test/lint.yaml @@ -0,0 +1,80 @@ +lint: + - name: valid_single + inputs: + r1_fastq: + - reads_R1.fastq + assertions: + exit_code: 0 + stdout: + - "read 4 records" + - name: valid_paired_all_options + inputs: + r1_fastq: + - reads_R1.fastq + r2_fastq: + - reads_R2.fastq + lint_mode: + - Log + single_read_validation_level: + - Medium + paired_read_validation_level: + - Medium + disabled_validators: + - [S006] + record_definition_separator: + - "/" + assertions: + exit_code: 0 + stdout: + - "read 4 \\* 2 records" + outputs: + validation_report: + - Name: fq-lint.log + - name: disabled_validator + inputs: + r1_fastq: + - invalid.fastq + disabled_validators: + - [S002] + report_name: + - disabled-validator.log + assertions: + exit_code: 0 + stdout: + - "read 1 records" + outputs: + validation_report: + - Name: disabled-validator.log + - name: invalid_fastq + inputs: + r1_fastq: + - invalid.fastq + assertions: + should_fail: true + stderr: + - "\\[S002\\]" + - name: valid_low_levels + inputs: + r1_fastq: + - reads_R1.fastq + single_read_validation_level: + - Low + paired_read_validation_level: + - Low + assertions: + exit_code: 0 + stdout: + - "read 4 records" + outputs: + validation_report: + - Name: fq-lint.log + - name: invalid_log_mode + inputs: + r1_fastq: + - invalid.fastq + lint_mode: + - Log + assertions: + should_fail: true + stdout: + - "\\[S002\\]" diff --git a/fq/module.json b/fq/module.json new file mode 100644 index 0000000..040fa86 --- /dev/null +++ b/fq/module.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://raw.githubusercontent.com/openwdl/wdl/8199dd7e3f17322e827225d6e6bc886dac139207/modules/schemas/module.schema.json", + "name": "fq", + "license": "MIT OR Apache-2.0", + "description": "WDL tasks for `fq`", + "repository": "https://github.com/stjude-rust-labs/modules", + "entrypoint": "fq.wdl", + "tools": [ + { + "name": "fq", + "version": "0.12.0", + "license": "MIT", + "url": "https://github.com/stjude-rust-labs/fq" + } + ], + "dependencies": {} +} diff --git a/fq/subsample/subsample.wdl b/fq/subsample/subsample.wdl new file mode 100644 index 0000000..780196a --- /dev/null +++ b/fq/subsample/subsample.wdl @@ -0,0 +1,149 @@ +## Wraps `fq subsample` to retain FASTQ records by probability or exact count. +## Exports the `subsample` task. +version 1.4 + +## Subsamples single or paired FASTQ inputs with `fq subsample`. +task subsample { + input { + ## Read-one FASTQ input. + env File r1_fastq + + ## Optional read-two FASTQ input. + File? r2_fastq + + ## Optional probability strictly between `0.0` and `1.0`. + Float? probability + + ## Optional exact number of records to retain. + Int? record_count + + ## Optional random-number seed. + Int? seed + + ## Read-one output filename; `.gz` enables compression. + env String r1_output_name = "subsampled_R1.fastq.gz" + + ## Optional read-two output filename; `.gz` enables compression. + String? r2_output_name + + ## Minimum CPU cores. + Int cpu = 1 + + ## Minimum memory with units. + String memory = "1 GiB" + + ## Minimum disk space in GiB. + Int disk_gib = 10 + + ## Digest-pinned `fq` `0.12.0` BioContainers image. + String container = "quay.io/biocontainers/fq:0.12.0--h9ee0642_0@sha256:74b59572f1d05b4829b45b599ee04311c8b3acec510f3cfb879f23b4bbd2090b" + + ## Trusted arguments absent from the pinned `fq` CLI. + Array[String] extra_args = [] + } + + # Optional read-two path or an empty string. + env String r2_fastq_value = if defined(r2_fastq) + then "~{select_first([ + r2_fastq, + ])}" + else "" + + # Optional sampling probability or an empty string. + env String probability_value = if defined(probability) + then "~{select_first([ + probability, + ])}" + else "" + + # Optional record count or an empty string. + env String record_count_value = if defined(record_count) + then "~{select_first([ + record_count, + ])}" + else "" + + # Optional random-number seed or an empty string. + env String seed_value = if defined(seed) + then "~{select_first([ + seed, + ])}" + else "" + + # Optional read-two output filename or an empty string. + env String r2_output_name_value = select_first([ + r2_output_name, + "", + ]) + + # File preserving trusted extra-argument boundaries. + env File extra_args_file = write_lines(extra_args) + + command <<< + # shellcheck disable=SC2154 + set -euo pipefail + export NO_COLOR=1 + + mapfile -t extra_args < "$extra_args_file" + + quantity_count=0 + quantity_args=() + if [[ -n "$probability_value" ]]; then + quantity_args+=(--probability "$probability_value") + ((quantity_count += 1)) + fi + if [[ -n "$record_count_value" ]]; then + quantity_args+=(--record-count "$record_count_value") + ((quantity_count += 1)) + fi + if (( quantity_count != 1 )); then + printf '%s\n' "exactly one of \`probability\` or \`record_count\` is required" >&2 + exit 2 + fi + + r2_pair_count=0 + if [[ -n "$r2_fastq_value" ]]; then + ((r2_pair_count += 1)) + fi + if [[ -n "$r2_output_name_value" ]]; then + ((r2_pair_count += 1)) + fi + if (( r2_pair_count == 1 )); then + printf '%s\n' "\`r2_fastq\` and \`r2_output_name\` must be provided together" >&2 + exit 2 + fi + + option_args=("${quantity_args[@]}" --r1-dst "$r1_output_name") + if [[ -n "$seed_value" ]]; then + option_args+=(--seed "$seed_value") + fi + if [[ -n "$r2_output_name_value" ]]; then + option_args+=(--r2-dst "$r2_output_name_value") + fi + + positional_args=("$r1_fastq") + if [[ -n "$r2_fastq_value" ]]; then + positional_args+=("$r2_fastq_value") + fi + + fq subsample \ + "${option_args[@]}" \ + "${extra_args[@]}" \ + "${positional_args[@]}" + >>> + + output { + ## Subsampled read-one FASTQ. + File r1_subsampled_fastq = r1_output_name + + ## Optional subsampled read-two FASTQ. + File? r2_subsampled_fastq = r2_output_name + } + + requirements { + container: container + cpu: cpu + memory: memory + disks: disk_gib + } +} diff --git a/fq/subsample/test/subsample.yaml b/fq/subsample/test/subsample.yaml new file mode 100644 index 0000000..2d745fe --- /dev/null +++ b/fq/subsample/test/subsample.yaml @@ -0,0 +1,108 @@ +subsample: + - name: exact_paired + inputs: + r1_fastq: + - reads_R1.fastq + r2_fastq: + - reads_R2.fastq + record_count: + - 2 + seed: + - 13 + r1_output_name: + - subsampled_R1.fastq + r2_output_name: + - subsampled_R2.fastq + assertions: + exit_code: 0 + stdout: + - "sampled 2/4 \\(50.0%\\) records" + outputs: + r1_subsampled_fastq: + - Name: subsampled_R1.fastq + r2_subsampled_fastq: + - Defined: true + - Name: subsampled_R2.fastq + - name: probability_single + inputs: + r1_fastq: + - reads_R1.fastq + probability: + - 0.5 + seed: + - 13 + r1_output_name: + - probability_R1.fastq + assertions: + exit_code: 0 + stdout: + - "sampled [0-4]/4" + outputs: + r1_subsampled_fastq: + - Name: probability_R1.fastq + r2_subsampled_fastq: + - Defined: false + - name: rejects_missing_quantity + inputs: + r1_fastq: + - reads_R1.fastq + r1_output_name: + - invalid.fastq + assertions: + should_fail: true + stderr: + - "exactly one of `probability` or `record_count` is required" + - name: rejects_both_quantities + inputs: + r1_fastq: + - reads_R1.fastq + probability: + - 0.5 + record_count: + - 2 + r1_output_name: + - invalid.fastq + assertions: + should_fail: true + stderr: + - "exactly one of `probability` or `record_count` is required" + - name: rejects_missing_r2_output + inputs: + r1_fastq: + - reads_R1.fastq + r2_fastq: + - reads_R2.fastq + record_count: + - 2 + r1_output_name: + - invalid_R1.fastq + assertions: + should_fail: true + stderr: + - "`r2_fastq` and `r2_output_name` must be provided together" + - name: rejects_missing_r2_source + inputs: + r1_fastq: + - reads_R1.fastq + record_count: + - 2 + r1_output_name: + - invalid_R1.fastq + r2_output_name: + - invalid_R2.fastq + assertions: + should_fail: true + stderr: + - "`r2_fastq` and `r2_output_name` must be provided together" + - name: upstream_rejects_probability_boundary + inputs: + r1_fastq: + - reads_R1.fastq + probability: + - 0.0 + r1_output_name: + - invalid.fastq + assertions: + should_fail: true + stderr: + - "invalid probability" diff --git a/fq/test/fixtures/allowlist.txt b/fq/test/fixtures/allowlist.txt new file mode 100644 index 0000000..c561c4c --- /dev/null +++ b/fq/test/fixtures/allowlist.txt @@ -0,0 +1 @@ +fqlib4:444:GKTNCER:4:9:1621:4688 diff --git a/fq/test/fixtures/invalid.fastq b/fq/test/fixtures/invalid.fastq new file mode 100644 index 0000000..f5fbb2f --- /dev/null +++ b/fq/test/fixtures/invalid.fastq @@ -0,0 +1,4 @@ +@fqlib4:444:GKTNCER:3:22:4527:3404/1 +XCATCTGG ++ +724=8587 diff --git a/fq/test/fixtures/reads_R1.fastq b/fq/test/fixtures/reads_R1.fastq new file mode 100644 index 0000000..f9517ba --- /dev/null +++ b/fq/test/fixtures/reads_R1.fastq @@ -0,0 +1,16 @@ +@fqlib4:444:GKTNCER:3:22:4527:3404/1 +GCATCTGG ++ +724=8587 +@fqlib4:444:GKTNCER:4:9:1621:4688/1 +CTTGTCGG ++ +;7=81376 +@fqlib4:444:GKTNCER:2:38:8965:971/1 +ATGATATA ++ +4546567; +@fqlib4:444:GKTNCER:6:54:1108:5787/1 +TGCGCTAC ++ +5:466557 diff --git a/fq/test/fixtures/reads_R2.fastq b/fq/test/fixtures/reads_R2.fastq new file mode 100644 index 0000000..3b73576 --- /dev/null +++ b/fq/test/fixtures/reads_R2.fastq @@ -0,0 +1,16 @@ +@fqlib4:444:GKTNCER:3:22:4527:3404/2 +TGCGCGAT ++ +83847664 +@fqlib4:444:GKTNCER:4:9:1621:4688/2 +CGGTCACA ++ +7:635948 +@fqlib4:444:GKTNCER:2:38:8965:971/2 +CATTCCAC ++ +44668975 +@fqlib4:444:GKTNCER:6:54:1108:5787/2 +TCGATGCG ++ +56138524 diff --git a/sprocket.toml b/sprocket.toml new file mode 100644 index 0000000..2b99c06 --- /dev/null +++ b/sprocket.toml @@ -0,0 +1,28 @@ +#:schema https://raw.githubusercontent.com/stjude-rust-labs/sprocket/v0.28.0/jsonschemas/sprocket.toml.json + +[format] +indent = 4 +max_line_length = 90 +sort_imports = true +sort_inputs = false +trailing_commas = true +newline_style = "unix" + +[check] +deny_warnings = true +deny_notes = true +all_lint_rules = true +except = ["MetaSections"] + +[check.lint] +bash_set_options = ["errexit", "nounset", "pipefail"] + +[test] +parallelism = 4 + +[doc] +with_doc_comments = true + +[common.wdl.feature_flags] +wdl_1_3 = true +wdl_1_4 = true