include empty samples in multiqc sample overview

Author: Joon-Klaps

assets/samplesheets/samplesheet.csv

@@ -1,3 +1,4 @@
 sample,fastq_1,fastq_2
 SRR11140744,https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/sispa/SRR11140744_R1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/sispa/SRR11140744_R2.fastq.gz
 SRR11140748,https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/sispa/SRR11140748_R1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/sispa/SRR11140748_R2.fastq.gz
+empty-SRR,https://raw.githubusercontent.com/Joon-Klaps/nextclade_data/master/data/fastq/empty_1.fastq.gz,https://raw.githubusercontent.com/Joon-Klaps/nextclade_data/master/data/fastq/empty_2.fastq.gz

bin/custom_multiqc.py

@@ -10,9 +10,8 @@
 from typing import Dict, List, Optional, Tuple, Union
 
 import multiqc as mqc
-import numpy as np
 import pandas as pd
-from multiqc.plots import bargraph, table
+from multiqc.plots import bargraph
 from multiqc.types import Anchor
 from utils.constant_variables import CLUSTER_PCONFIG
 from utils.file_tools import filelist_to_df, get_module_selection, read_in_quast, write_df
@@ -178,6 +177,21 @@ def file_choices(choices, fname):
     )
     return parser.parse_args(argv)
 
+def get_failed_samples(samples: List[str]) -> List[str]:
+    """
+    Get failed samples from the modules
+        - sample_low_reads
+        - samples_without_contigs
+    """
+    if (samples_low_reads :=  get_module_data(mqc, 'samples_low_reads')):
+        logger.info("samples_low_reads %s", samples_low_reads)
+        samples.extend([k for k in samples_low_reads.keys()])
+
+    if (samples_without_contigs := get_module_data(mqc, 'samples_without_contigs')):
+        logger.info("samples_without_contigs %s", samples_without_contigs)
+        samples.extend([k for k in samples_without_contigs.keys() ])
+
+    return samples
 
 def load_custom_data(args) -> List[pd.DataFrame]:
     """
@@ -409,11 +423,12 @@ def extract_mqc_data(table_headers: Union[str, Path]) -> Optional[pd.DataFrame]:
     return join_df(result, data) if data else result, columns_result
 
 
-def write_results(contigs_mqc, constrains_mqc, constrains_genstats, args) -> int:
+def write_results(contigs_mqc: pd.DataFrame, constrains_mqc: pd.DataFrame, constrains_genstats: pd.DataFrame) -> int:
     """
     Write the results to files.
     """
-    samples = []
+    samples = get_failed_samples([])
+    logger.info("samples %s", samples)
     if not contigs_mqc.empty:
         logger.info("Writing Unfiltered Denovo constructs table file: contigs_overview.tsv")
         samples.extend(contigs_mqc["sample"])
@@ -511,7 +526,7 @@ def main(argv=None):
 
     coalesced_constrains, constrains_genstats = reformat_constrain_df(constrains_mqc, renamed_columns, args)
 
-    write_results(contigs_mqc, coalesced_constrains, constrains_genstats, args)
+    write_results(contigs_mqc, coalesced_constrains, constrains_genstats)
     return 0
 
 

bin/utils/pandas_tools.py

@@ -42,7 +42,7 @@ def reorder_rows(dataframe):
     rank_dict = {step: rank for rank, step in enumerate(ordered_list, start=1)}
 
     # Sort the DataFrame by 'step' based on the ranking dictionary
-    df["rank"] = df["step"].replace(rank_dict)
+    df["rank"] = df["step"].map(pd.Series(rank_dict))
     df = df.sort_values(["sample", "cluster", "rank"])
 
     return df

conf/modules.config

@@ -894,7 +894,7 @@ process {
             withName: RENAME_FASTA_HEADER_SINGLETON {
                 ext.prefix = { "${meta.id}_singleton" } // DON'T CHANGE
                 publishDir = [
-                    path: { "${params.outdir}/consensus/seq/consensus/${meta.sample}"},
+                    path: { "${params.outdir}/consensus/seq/scaffold_consensus/${meta.sample}"},
                     mode: params.publish_dir_mode,
                     pattern: "*.fasta",
                     saveAs: { filename -> params.prefix || params.global_prefix  ? "${params.global_prefix}-$filename" : filename }
@@ -957,7 +957,7 @@ process {
             withName: RENAME_FASTA_HEADER_CONTIG_CONSENSUS{
                 ext.prefix = { "${meta.id}_consensus" } // DON'T CHANGE
                 publishDir = [
-                    path: { "${params.outdir}/consensus/seq/consensus/${meta.sample}"},
+                    path: { "${params.outdir}/consensus/seq/scaffold_consensus/${meta.sample}"},
                     mode: params.publish_dir_mode,
                     pattern: "*.fasta",
                     saveAs: { filename -> params.prefix || params.global_prefix  ? "${params.global_prefix}-$filename" : filename }

docs/output.md

@@ -666,9 +666,9 @@ The consensus sequences are generated by [`BCFTools`](http://samtools.github.io/
 ???- abstract "Output files - iterations & variants"
 
     - `consensus`
-        - `seq/<it# | consensus | singleton | constrain>/ `
+        - `seq/<it# | scaffold_consensus | variant-calling | constrain>/ `
             - `<sample-id>/*.fasta`: A fasta file containing the consensus sequence.
-        - `mask/<it# | consensus | singleton | constrain>`
+        - `mask/<it# | variant-calling | constrain>`
             - `<sample-id>/*.qual.txt`: A log file of the consensus run containing statistics. [`iVar` only]
             - `<sample-id>/*.bed`: A bed file containing the masked regions. [`BCFtools` only]
             - `<sample-id>/*.mpileup`: A mpileup file containing information on the depth and the quality of each aligned base.

subworkflows/local/utils_nfcore_viralgenie_pipeline/main.nf

@@ -381,7 +381,7 @@ def lowReadSamplesToMultiQC(tsv_data, min_trimmed_reads) {
                 [
                     "id: 'samples_low_reads'",
                     "anchor: 'WARNING: Filtered samples'",
-                    "section_name: 'Samples with to few reads'",
+                    "section_name: 'Samples with too few reads'",
                     "format: 'tsv'",
                     "description: 'Samples that did not have the minimum number of reads (<${min_trimmed_reads}) after trimming, complexity filtering & host removal'",
                     "plot_type: 'table'"