Summarize command

.github/workflows/build.yml

@@ -38,8 +38,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{ env.CONDA }}/envs
-          key:
-            conda-${{ runner.os }}--${{ runner.arch }}--${{
+          key: conda-${{ runner.os }}--${{ runner.arch }}--${{
             steps.get-date.outputs.today }}-${{
             hashFiles('environments/dev.yml') }}-${{ env.CACHE_NUMBER
             }}
@@ -49,8 +48,7 @@ jobs:
         id: cache
 
       - name: Update environment
-        run:
-          conda env update -n nomadic-dev -f environments/dev.yml
+        run: conda env update -n nomadic-dev -f environments/dev.yml
         if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Conda info
@@ -62,15 +60,19 @@ jobs:
         run: |
           pip install . --no-deps
 
+      # Currently not working because links have changed, enable once fixed
+      # - name: Download reference genome
+      #   run: |
+      #     nomadic download -r Pf3D7
+
       - name: Run tests
         run: |
           pytest
-      
+
       - name: Smoke test nomadic
         run: |
           nomadic --help
 
-
   ruff:
     runs-on: ubuntu-latest
     steps:

build/conda/meta.yaml

@@ -29,17 +29,19 @@ requirements:
     - numpy
     - pandas
     - openpyxl
+    - statsmodels
     - seaborn
     - click
     - platformdirs
     - dash
     - pysam
     - pyyaml
     - i18nice
+    - pydantic
     # tools
     - minimap2
-    - samtools >=1.20
-    - bcftools >=1.20
+    - samtools >=1.23
+    - bcftools >=1.23
     - bedtools
     - delve-bio =0.3.*
     - rsync

environments/dev.yml

@@ -5,8 +5,8 @@ channels:
 dependencies:
   - python >= 3.10
   - minimap2
-  - samtools >=1.20
-  - bcftools >=1.20
+  - samtools >=1.23
+  - bcftools >=1.23
   - htslib
   - bedtools
   - delve-bio =0.3.*
@@ -16,11 +16,13 @@ dependencies:
   - pandas
   - openpyxl
   - numpy
+  - statsmodels
   - seaborn
   - click
   - platformdirs
   - pyyaml
   - i18nice
+  - pydantic
   - black
   - flake8
   - pip
@@ -29,3 +31,5 @@ dependencies:
   - mkdocs-material
   - pymdown-extensions
   - rsync
+  - pip:
+      - pytest-snapshot

environments/run.yml

@@ -5,20 +5,22 @@ channels:
 dependencies:
   - python >= 3.10
   - minimap2
-  - samtools >=1.20
-  - bcftools >=1.20
+  - samtools >=1.23
+  - bcftools >=1.23
   - delve-bio =0.3.*
   - htslib
   - bedtools
   - dash
   - pandas
   - openpyxl
   - numpy
+  - statsmodels
   - pysam
   - seaborn
   - click
   - platformdirs
   - pyyaml
   - i18nice
+  - pydantic
   - pip
   - rsync

pyproject.toml

@@ -1,3 +1,10 @@
 [build-system]
 requires = ["setuptools>=42.0", "wheel"]
 build-backend = "setuptools.build_meta"
+
+[tool.ruff.lint]
+ignore = [
+    "UP045", # To me, Optional seems shorter
+    "RUF012", # will fix this later
+    "C408", # Would need a lot of changes, fix maybe later
+]

scripts/run_realtime.sh

@@ -1,8 +1,7 @@
 # Example of how to run nomadic realtime
 # 2023/07/12, J.Hendry
 
-nomadic realtime \
--e 0000-00-00_example \
+nomadic realtime 0000-00-00_example \
 -f example_data/minknow/fastq_pass \
 -m example_data/metadata/sample_info.csv \
 -b example_data/beds/nomads8.amplicons.bed --call

scripts/simulate_sequencing.py

@@ -3,9 +3,10 @@
 # 2023/06/01, J. Hendry
 
 import os
-import shutil
 import random
+import shutil
 import time
+
 import pandas as pd
 
 AWAITED_FASTQ_DIR = "example_data/awaited_fastqs"

setup.cfg

@@ -20,10 +20,12 @@ install_requires =
     numpy
     pandas
     openpyxl
+    statsmodels
     platformdirs
     pyyaml
     seaborn
     i18nice
+    pydantic
 
 python_requires = >=3.10
 zip_safe = no
@@ -32,6 +34,9 @@ zip_safe = no
 nomadic.realtime.dashboard =
     assets/*
     translations/*
+nomadic.summarize.dashboard =
+    assets/*
+    translations/*
 nomadic.start =
     data/**
 

src/nomadic/cli.py

@@ -2,20 +2,21 @@
 
 import click
 
-from nomadic.configure.commands import configure
 from nomadic.backup.commands import backup
+from nomadic.configure.commands import configure
 from nomadic.dashboard.commands import dashboard
 from nomadic.download.commands import download
 from nomadic.process.commands import process
 from nomadic.realtime.commands import realtime
 from nomadic.share.commands import share
 from nomadic.start.commands import start
+from nomadic.summarize.commands import summarize
 
 
 # From: https://stackoverflow.com/questions/47972638/how-can-i-define-the-order-of-click-sub-commands-in-help
 class OrderedGroup(click.Group):
     def __init__(self, name=None, commands=None, **attrs):
-        super(OrderedGroup, self).__init__(name, commands, **attrs)
+        super().__init__(name, commands, **attrs)
         #: the registered subcommands by their exported names.
         self.commands = commands or OrderedDict()
 
@@ -30,14 +31,14 @@ def cli():
     Mobile sequencing and analysis in real-time
 
     """
-    pass
 
 
 cli.add_command(start)
 cli.add_command(download)
 cli.add_command(realtime)
 cli.add_command(process)
 cli.add_command(dashboard)
+cli.add_command(summarize)
 cli.add_command(share)
 cli.add_command(backup)
 cli.add_command(configure)

src/nomadic/configure/commands.py

@@ -18,7 +18,6 @@ def configure():
     """
     Configure different nomadics functionality. This mostly sets standard options in '.config.yaml' that can be overwritten from the command line.
     """
-    pass
 
 
 @configure.command(short_help="Configure the nomadic share command.")

src/nomadic/dashboard/main.py

@@ -2,41 +2,16 @@
 from typing import Optional
 
 from nomadic.realtime.dashboard.builders import CallingRTDashboard, MappingRTDashboard
-from nomadic.util.experiment import ExperimentDirectories
-from nomadic.util.metadata import find_metadata
-from nomadic.util.regions import RegionBEDParser
+from nomadic.util.experiment import ExperimentDirectories, find_metadata, find_regions
 from nomadic.util.settings import load_settings
 
 
-def find_regions(input_dir: str) -> RegionBEDParser:
-    """
-    Given an experiment directory, search for the metadata CSV file in thee
-    expected location
-
-    TODO: Bad duplication from above, can write inner function
-    """
-
-    metadata_dir = os.path.join(input_dir, "metadata")
-    beds = [
-        f"{metadata_dir}/{file}"
-        for file in os.listdir(metadata_dir)
-        if file.endswith(".bed") and not file.endswith(".lowcomplexity_mask.bed")
-    ]  # TODO: what about no-suffix files?
-
-    if len(beds) != 1:  # Could alternatively load and LOOK
-        raise FileNotFoundError(
-            f"Expected one region BED file (*.bed) at {metadata_dir}, but found {len(beds)}."
-        )
-
-    return RegionBEDParser(beds[0])
-
-
 def variant_calling_performed(expt_dirs: ExperimentDirectories) -> bool:
     """
     Check if the variant calling TSV is present
     """
 
-    return os.path.exists(expt_dirs.get_summary_files().variants)
+    return os.path.exists(expt_dirs.get_summary_files().aa_changes)
 
 
 def main(input_dir: str, host: str, port: Optional[int] = None) -> None:
@@ -57,8 +32,10 @@ def main(input_dir: str, host: str, port: Optional[int] = None) -> None:
     """
 
     metadata = find_metadata(input_dir)
-    expt_dirs = ExperimentDirectories(input_dir, metadata)
     regions = find_regions(input_dir)
+    expt_dirs = ExperimentDirectories(
+        input_dir, metadata.barcodes, os.path.basename(regions.path)
+    )
     settings = load_settings(expt_dirs.get_settings_file())
 
     if settings is not None:
@@ -94,15 +71,15 @@ def main(input_dir: str, host: str, port: Optional[int] = None) -> None:
         print("  Variant calling: True")
         dashboard = CallingRTDashboard(
             **shared_kwargs,
-            variant_csv=summary_files.variants,
+            variant_csv=summary_files.aa_changes,
         )
     else:
         print("  Variant calling: False")
         dashboard = MappingRTDashboard(**shared_kwargs)
     print("Done.")
 
-    print("")
+    print()
     print("Launching dashboard (press CNTRL+C to exit):")
-    print("")
+    print()
 
     dashboard.run(debug=False, host=host, port=port)

src/nomadic/download/commands.py

@@ -1,4 +1,5 @@
 import click
+
 from .references import REFERENCE_COLLECTION
 
 

src/nomadic/download/corrections.py

@@ -1,5 +1,5 @@
-from typing import List
 from dataclasses import dataclass
+
 from nomadic.util.fasta import load_fasta_as_dict, write_fasta_from_dict
 
 
@@ -19,7 +19,7 @@ class NucleotideChange:
 )
 
 
-def update_reference_genome(fasta_path: str, mutations: List[NucleotideChange]) -> None:
+def update_reference_genome(fasta_path: str, mutations: list[NucleotideChange]) -> None:
     """
     Update a reference genome by reverting mutations; do this IN PLACE
 

src/nomadic/download/downloader.py

@@ -37,7 +37,7 @@ def download_fasta(self, create_mask: bool = False):
                 url=self.ref.fasta_url, filename=self.ref.fasta_path
             )
             print("Done.")
-            print("")
+            print()
         else:
             print("Already downloaded FASTA.")
 
@@ -82,9 +82,7 @@ def _standardise_gff(self) -> None:
         # Standardise
         gff_df = load_gff(self.ref.gff_path)
         gff_df.query("feature in @KEEP_FIELDS", inplace=True)
-        gff_df["feature"] = [
-            to_gff3[f] if f in to_gff3 else f for f in gff_df["feature"]
-        ]
+        gff_df["feature"] = [to_gff3.get(f, f) for f in gff_df["feature"]]
 
         # Rename attributes to what bcftools expects
         # see https://samtools.github.io/bcftools/bcftools-man.html#csq

src/nomadic/download/main.py

@@ -1,6 +1,6 @@
+from .corrections import DHPS, update_reference_genome
 from .downloader import ReferenceDownloader
 from .references import REFERENCE_COLLECTION
-from .corrections import DHPS, update_reference_genome
 
 
 def main(reference_name: str, all: bool = False) -> None:

src/nomadic/download/references.py

@@ -1,9 +1,8 @@
 import os
-from nomadic.util.exceptions import ReferenceGenomeMissingError
 from abc import ABC, abstractmethod
 
 from nomadic.util.dirs import user_data_dir
-
+from nomadic.util.exceptions import ReferenceGenomeMissingError
 
 # ================================================================
 # Base classes for reference sequences

src/nomadic/map/mappers.py

@@ -1,9 +1,8 @@
 import os
-import subprocess
 import shlex
+import subprocess
 from abc import ABC, abstractmethod
 
-
 # ================================================================
 # Define abstract base class for different mapping
 # algorithms
@@ -41,11 +40,10 @@ def map_from_fastqs(self, fastq_dir=None, fastq_paths=None):
 
         """
         if fastq_dir is not None:
-            fastq_dir = fastq_dir
             self.input_fastqs = [
                 f"{fastq_dir}/{fastq}"
                 for fastq in os.listdir(fastq_dir)
-                if fastq.endswith(".fastq") or fastq.endswith(".fastq.gz")
+                if fastq.endswith((".fastq", ".fastq.gz"))
             ]
         elif fastq_paths is not None:
             self.input_fastqs = fastq_paths
@@ -57,7 +55,6 @@ def _define_mapping_command(self, output_bam, *, threads: int, flags):
         """
         Define the command for the mapping algorithm
         """
-        pass
 
     def run(self, output_bam: str, threads: int, verbose=False):
         """