From 570cac615da67830d2de01dd89bbdae0fd711d3f Mon Sep 17 00:00:00 2001 From: Jason Stajich Date: Mon, 29 Jun 2026 09:05:32 -0700 Subject: [PATCH] feat(#4): versions.yml + conf/base.config + conf/modules.config Establishes the conventions every future module extraction copies. conf/base.config Label-based portable resource tiers (process_single/low/medium/high/long). Wired into nextflow.config before profiles so withName: in profile_annotate.config still overrides for UCR HPCC. Decouples module definitions from infrastructure. conf/modules.config Extracts publishDir from modules/local/ into config. The three annotation modules (ANTISMASH_RUN, INTERPROSCAN_RUN, SIGNALP_RUN) no longer hard-code publishDir "${params.target}" inline. modules/local/*.nf All four extracted modules updated: - Remove inline cpus/memory/time (replaced by resource-tier label) - Remove inline publishDir (moved to conf/modules.config) - Add second label 'process_low' / 'process_medium' alongside existing provisioning label (antismash / interproscan / signalp / setup) - Add 'versions.yml' output (emit: versions) with real version capture in script: and stub versions in stub: - Add emit: name to primary output so .out.results / .out.versions are unambiguous funannotate.nf - ch_versions channel accumulates versions.yml from ASM_STATS, ANTISMASH_RUN, INTERPROSCAN_RUN, SIGNALP_RUN whenever those processes run - ch_versions.unique().collectFile() writes logs/nextflow/software_versions.yml - Call sites updated to use .out.results / .out.versions Stub-run gate: 15 completed, 0 failed. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01AbWwmm9v6dJn13sgAJn3Sa --- conf/base.config | 51 +++++++++++++++++++++++++++++++ conf/modules.config | 40 ++++++++++++++++++++++++ funannotate.nf | 22 +++++++++++-- modules/local/antismash_run.nf | 21 +++++++------ modules/local/asm_stats.nf | 33 ++++++++------------ modules/local/interproscan_run.nf | 20 +++++++----- modules/local/signalp_run.nf | 19 +++++++----- nextflow.config | 4 +++ 8 files changed, 162 insertions(+), 48 deletions(-) create mode 100644 conf/base.config create mode 100644 conf/modules.config diff --git a/conf/base.config b/conf/base.config new file mode 100644 index 0000000..d60403f --- /dev/null +++ b/conf/base.config @@ -0,0 +1,51 @@ +/* + * conf/base.config — portable resource defaults, applied to every run + * + * Processes declare a resource tier via `label 'process_low'` etc. These are + * the fallback values used when no profile-specific `withName:` block overrides + * them (e.g., on a local workstation or a non-UCR HPC). + * + * UCR HPCC annotate runs override per-process with `withName:` blocks in + * conf/profile_annotate.config. Provisioning (Lmod module loading) is handled + * by conf/provision_ucr_hpcc.config via withLabel: 'antismash' etc. + * + * Tier summary (base portable values): + * process_single 1 cpu 6 GB 4 h + * process_low 2 cpus 12 GB 8 h + * process_medium 8 cpus 32 GB 24 h + * process_high 16 cpus 64 GB 72 h + * process_long 8 cpus 32 GB 120 h (duration-bound, moderate memory) + */ + +process { + + withLabel:process_single { + cpus = 1 + memory = '6 GB' + time = '4 h' + } + + withLabel:process_low { + cpus = 2 + memory = '12 GB' + time = '8 h' + } + + withLabel:process_medium { + cpus = 8 + memory = '32 GB' + time = '24 h' + } + + withLabel:process_high { + cpus = 16 + memory = '64 GB' + time = '72 h' + } + + withLabel:process_long { + cpus = 8 + memory = '32 GB' + time = '120 h' + } +} diff --git a/conf/modules.config b/conf/modules.config new file mode 100644 index 0000000..6c9e106 --- /dev/null +++ b/conf/modules.config @@ -0,0 +1,40 @@ +/* + * conf/modules.config — per-module publishDir and ext.args + * + * Keeps process definitions in modules/local/ free of pipeline-specific paths. + * Every publishDir that references params.target or other pipeline-level + * directories belongs here rather than in the module file. + * + * ext.args can be used to pass extra CLI flags to tools without editing modules. + */ + +process { + + withName: 'ASM_STATS' { + // storeDir handles persistence; no publishDir needed. + } + + withName: 'ANTISMASH_RUN' { + publishDir = [ + path: { params.target }, + mode: 'copy', + overwrite: true + ] + } + + withName: 'INTERPROSCAN_RUN' { + publishDir = [ + path: { params.target }, + mode: 'copy', + overwrite: true + ] + } + + withName: 'SIGNALP_RUN' { + publishDir = [ + path: { params.target }, + mode: 'copy', + overwrite: true + ] + } +} diff --git a/funannotate.nf b/funannotate.nf index 2e6a962..a3ab2ff 100644 --- a/funannotate.nf +++ b/funannotate.nf @@ -1796,6 +1796,8 @@ workflow { // postpredict — meta only no genome filter (annotate/update paths) INPUT_CHECK() def jobs = INPUT_CHECK.out.genomes + + def ch_versions = Channel.empty() if (params.debug.toBoolean()) { jobs.view { meta, gz -> "[CHANNEL] Submitting: out=${meta.id}, asmid=${meta.asmid}, transl_table=${meta.transl_table}, gz=${gz}" } } @@ -1875,6 +1877,7 @@ workflow { file(params.samples), file(params.genome_dir) ) + ch_versions = ch_versions.mix(ASM_STATS.out.versions) } else { log.info "Assembly statistics already exist: ${asm_stats_gz}" } @@ -2178,7 +2181,8 @@ workflow { asDir.isDirectory() && asDir.list()?.any { it.endsWith('.json') || it.endsWith('.json.gz') } } ANTISMASH_RUN(as_todo) - def as_completed = ANTISMASH_RUN.out + ch_versions = ch_versions.mix(ANTISMASH_RUN.out.versions) + def as_completed = ANTISMASH_RUN.out.results .map { meta, _files -> meta } annotate_ready_ch = as_completed.mix(as_done) } @@ -2191,7 +2195,8 @@ workflow { file("${params.target}/${meta.id}/annotate_misc/iprscan.xml").exists() } INTERPROSCAN_RUN(ipr_todo) - def ipr_completed = INTERPROSCAN_RUN.out + ch_versions = ch_versions.mix(INTERPROSCAN_RUN.out.versions) + def ipr_completed = INTERPROSCAN_RUN.out.results .map { meta, _xml -> meta } annotate_ready_ch = ipr_completed.mix(ipr_done) } @@ -2204,7 +2209,8 @@ workflow { file("${params.target}/${meta.id}/annotate_misc/signalp.results.txt").exists() } SIGNALP_RUN(sp_todo) - def sp_completed = SIGNALP_RUN.out + ch_versions = ch_versions.mix(SIGNALP_RUN.out.versions) + def sp_completed = SIGNALP_RUN.out.results .map { meta, _txt -> meta } annotate_ready_ch = sp_completed.mix(sp_done) } @@ -2251,5 +2257,15 @@ workflow { } } // end if (!params.stop_after_sra_fetch || !params.run_sra_fetch) } + + // Collect software versions from all processes that emit versions.yml. + // Written to logs/software_versions.yml alongside the trace file. + ch_versions + .unique() + .collectFile( + name: 'software_versions.yml', + storeDir: "${launchDir}/logs/nextflow", + newLine: true + ) } diff --git a/modules/local/antismash_run.nf b/modules/local/antismash_run.nf index cf14d70..2985a3b 100644 --- a/modules/local/antismash_run.nf +++ b/modules/local/antismash_run.nf @@ -1,25 +1,19 @@ process ANTISMASH_RUN { label 'antismash' + label 'process_medium' tag "${meta.id}" - cpus 8 - memory '16 GB' - time '60h' - - publishDir "${params.target}", mode: 'copy', overwrite: true - input: val(meta) output: - tuple val(meta), path("${meta.id}/antismash_local/**") + tuple val(meta), path("${meta.id}/antismash_local/**"), emit: results + path 'versions.yml', emit: versions script: def out = meta.id def gbk = "${params.target}/${out}/predict_results/${out}.gbk" """ - # Accept a compressed prediction (.gbk.gz); antismash needs it uncompressed, so - # inflate a local copy in the work dir when only the gzipped form is present. GBK="${gbk}" if [ ! -f "\$GBK" ] && [ -f "${gbk}.gz" ]; then zcat "${gbk}.gz" > ${out}.predict.gbk @@ -38,6 +32,11 @@ process ANTISMASH_RUN { -c ${task.cpus} \\ \$GBK pigz ${out}/antismash_local/*.json + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + antismash: \$(antismash --version 2>&1 | grep -oP '(?<=antiSMASH )\\S+' || antismash --version 2>&1 | head -1) + END_VERSIONS """ stub: @@ -46,5 +45,9 @@ process ANTISMASH_RUN { mkdir -p ${out}/antismash_local touch ${out}/antismash_local/${out}.json.gz touch ${out}/antismash_local/index.html + cat <<-END_VERSIONS > versions.yml + "${task.process}": + antismash: 7.1.0 + END_VERSIONS """ } diff --git a/modules/local/asm_stats.nf b/modules/local/asm_stats.nf index 26d8049..3056286 100644 --- a/modules/local/asm_stats.nf +++ b/modules/local/asm_stats.nf @@ -1,29 +1,16 @@ -/* - * asm_stats — Generate assembly statistics for clean genomes - * - * This module generates asm_stats.tsv with columns: ASMID, total_length_bp, N50_bp, contig_count. - * Stats are used by earlgrey_mask.nf to select representative genomes per species (SELECT_REPS). - * - * Include in your workflow: - * include { ASM_STATS } from './modules/asm_stats' - * ASM_STATS(samples_csv, genome_dir) - */ - process ASM_STATS { label 'setup' + label 'process_low' storeDir { params.tables_dir } - cpus 4 - memory '8 GB' - time '2h' - input: path samples path genome_dir output: path 'asm_stats.tsv.gz', emit: stats + path 'versions.yml', emit: versions script: """ @@ -34,12 +21,10 @@ process ASM_STATS { printf 'ASMID\\ttotal_length_bp\\tN50_bp\\tcontig_count\\n' > \$TMPFILE - # Extract ASMIDs from samples.csv awk -F',' 'NR>1 {print \$2}' ${samples} | sort -u | while read asmid; do [ -z "\$asmid" ] && continue - asmid="\$(echo "\$asmid" | xargs)" # trim whitespace + asmid="\$(echo "\$asmid" | xargs)" - # Look for genome file: prefer .fa.gz, fall back to .fa, then .masked.fasta.gz if [ -f "${genome_dir}/\${asmid}.fa.gz" ]; then genome="${genome_dir}/\${asmid}.fa.gz" elif [ -f "${genome_dir}/\${asmid}.fa" ]; then @@ -53,7 +38,6 @@ process ASM_STATS { continue fi - # Use seqkit to compute stats total_bp=\$(seqkit stats -T "\$genome" 2>/dev/null | tail -n 1 | awk '{print \$4}') n50=\$(seqkit fx2tab -l "\$genome" 2>/dev/null | sort -rn -k2 | \\ awk -v total="\$total_bp" 'BEGIN{sum=0} {sum+=\$2; if(sum >= total/2) {print \$2; exit}}') @@ -67,12 +51,19 @@ process ASM_STATS { done pigz -c \$TMPFILE > asm_stats.tsv.gz - echo "[INFO] Assembly statistics written: asm_stats.tsv.gz" + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + seqkit: \$(seqkit version 2>&1 | sed 's/seqkit v//') + END_VERSIONS """ stub: """ printf 'ASMID\\ttotal_length_bp\\tN50_bp\\tcontig_count\\n' | pigz -c > asm_stats.tsv.gz - echo "[STUB] ASM_STATS" + cat <<-END_VERSIONS > versions.yml + "${task.process}": + seqkit: 2.8.0 + END_VERSIONS """ } diff --git a/modules/local/interproscan_run.nf b/modules/local/interproscan_run.nf index 0695da3..e69532a 100644 --- a/modules/local/interproscan_run.nf +++ b/modules/local/interproscan_run.nf @@ -1,19 +1,14 @@ -// IPRSCAN5 process INTERPROSCAN_RUN { label 'interproscan' + label 'process_medium' tag "${meta.id}" - cpus 8 - memory '32 GB' - time '60h' - - publishDir "${params.target}", mode: 'copy', overwrite: true - input: val(meta) output: - tuple val(meta), path("${meta.id}/annotate_misc/iprscan.xml") + tuple val(meta), path("${meta.id}/annotate_misc/iprscan.xml"), emit: results + path 'versions.yml', emit: versions script: def out = meta.id @@ -26,6 +21,11 @@ process INTERPROSCAN_RUN { mkdir -p ${out}/annotate_misc interproscan.sh -i ${proteins} -f XML -o ${out}/annotate_misc/iprscan.xml \\ -dp -goterms -pa -t p -cpu ${task.cpus} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + interproscan: \$(interproscan.sh --version 2>&1 | grep -oP '\\d+\\.\\d+\\.\\d+' | head -1) + END_VERSIONS """ stub: @@ -33,5 +33,9 @@ process INTERPROSCAN_RUN { """ mkdir -p ${out}/annotate_misc touch ${out}/annotate_misc/iprscan.xml + cat <<-END_VERSIONS > versions.yml + "${task.process}": + interproscan: 5.65-97.0 + END_VERSIONS """ } diff --git a/modules/local/signalp_run.nf b/modules/local/signalp_run.nf index 6b03d85..5de0ffe 100644 --- a/modules/local/signalp_run.nf +++ b/modules/local/signalp_run.nf @@ -1,18 +1,14 @@ process SIGNALP_RUN { label 'signalp' + label 'process_medium' tag "${meta.id}" - cpus 8 - memory '16 GB' - time '12h' - - publishDir "${params.target}", mode: 'copy', overwrite: true - input: val(meta) output: - tuple val(meta), path("${meta.id}/annotate_misc/signalp.results.txt") + tuple val(meta), path("${meta.id}/annotate_misc/signalp.results.txt"), emit: results + path 'versions.yml', emit: versions script: def out = meta.id @@ -30,6 +26,11 @@ process SIGNALP_RUN { mkdir -p ${out}/annotate_misc cp \$TMPDIR/${out}_signalp/prediction_results.txt ${out}/annotate_misc/signalp.results.txt rm -rf \$TMPDIR/${out}_signalp + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + signalp: \$(signalp6 --version 2>&1 | grep -oP '\\d+\\.\\d+\\S*' | head -1) + END_VERSIONS """ stub: @@ -37,5 +38,9 @@ process SIGNALP_RUN { """ mkdir -p ${out}/annotate_misc touch ${out}/annotate_misc/signalp.results.txt + cat <<-END_VERSIONS > versions.yml + "${task.process}": + signalp: 6.0g + END_VERSIONS """ } diff --git a/nextflow.config b/nextflow.config index e915a87..1db09dd 100644 --- a/nextflow.config +++ b/nextflow.config @@ -65,6 +65,10 @@ params { help = false } +// ── Base resource tiers + per-module publishDir (applied before profiles) ───── +includeConfig 'conf/base.config' +includeConfig 'conf/modules.config' + // ── Process defaults (apply to every process, every profile) ────────────────── process { shell = ['/bin/bash', '-l']