Merge branch 'nextflow-version' into 'dev'

nf version

See merge request epi2melabs/workflows/wf-transcriptomes!117

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [updated]
+### Changed 
+- Nextflow minimum required version to 23.04.2
+
 ## [v0.2.1]
 ### Changed
 - Any sample aliases that contain spaces will be replaced with underscores.

lib/NfcoreSchema.groovy

@@ -182,8 +182,8 @@ class NfcoreSchema {
             println ''
             log.error 'ERROR: Validation of pipeline parameters failed!'
             JSONObject exceptionJSON = e.toJSON()
-            printExceptions(exceptionJSON, params_json, log, enums)
+            HashSet<String> observed_exceptions = []
-            println ''
+            printExceptions(exceptionJSON, params_json, log, enums, raw_schema, observed_exceptions)
             has_error = true
         }
 
@@ -196,7 +196,7 @@ class NfcoreSchema {
                 warn_msg = warn_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}"
             }
             log.warn warn_msg
-            log.info "- ${colors.dim}Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}"
+            log.info "${colors.dim}- Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}"
             println ''
         }
 
@@ -361,16 +361,150 @@ class NfcoreSchema {
         return output
     }
 
+    // Traverse the raw_schema to resolve a $ref, returning a JSONObject or JSONArray.
+    private static def getSchemaLocation(raw_schema, schema_components) {
+        // a quick search did not reveal how to resolve a $ref inside a
+        // JSONObject so we'll go with traversing it ourselves!
+        def head = raw_schema
+        for (component in schema_components) {
+            if (component == '#') {
+                continue
+            }
+            else if (component.isNumber()) {
+                // assume numeric components are indexing an array
+                head = head[component as int]
+            }
+            else {
+                // otherwise grab by key
+                head = head[component]
+            }
+        }
+        return head
+    }
+
+    // Return a map of dependencies for a path in a schema, that relate to a key that is known to be missing.
+    // Each key in the map is a parameter that has been switched on and has a requirement on the missing key.
+    // The value of each key in the map is a list of all parameters that are required by the corresponding key.
+    private static HashMap<String, String> getSchemaDependencies(raw_schema, schema_components, missing_key) {
+        def head = getSchemaLocation(raw_schema, schema_components[1..-1] + 'dependencies')
+        HashMap<String, String> requirements = []
+        for (key in head.keySet()) {
+            if (head[key].contains(missing_key)) {
+                requirements[key] = head[key]
+            }
+        }
+        return requirements
+    }
+
+    // Return a list of all required parameters that are adjacent to a given path in a schema
+    private static ArrayList getSchemaRequirements(raw_schema, schema_components) {
+        def head = getSchemaLocation(raw_schema, schema_components[1..-2])
+        ArrayList requirements = []
+        for (entry in head) {
+            // this could be an allOf with more allOf/oneOf/anyOf inside
+            // we'll ignore anything that isn't a required key
+            if (entry.has("required")) {
+                requirements += entry["required"]
+            }
+        }
+        return requirements
+    }
+
     //
     // Loop over nested exceptions and print the causingException
     //
-    private static void printExceptions(ex_json, params_json, log, enums) {
+    private static void printExceptions(ex_json, params_json, log, enums, raw_schema, observed_exceptions) {
         def causingExceptions = ex_json['causingExceptions']
         if (causingExceptions.length() == 0) {
-            def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/
+            // Missing required param...
-            // Missing required param
+            if (ex_json["keyword"] == "required") {
-            if (m.matches()) {
+                def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/
-                log.error "* Missing required parameter: --${m[0][1]}"
+                // ...but what sort of required param
+                ArrayList schema_components = ex_json['schemaLocation'].split('/')
+                if (schema_components[-1].isNumber()) {
+                    // schemaLocation ends in array position, indicating use of
+                    // xOf in the schema. this is some sort of complex definition
+                    String schema_error_location = schema_components[1..-2].join('.')
+                    String composition_type = schema_components[-2]
+                    if (schema_error_location in observed_exceptions){
+                        return
+                    }
+                    // try to check the schema to get the adjacent requirements
+                    ArrayList requirement_group = []
+                    try {
+                        requirement_group = getSchemaRequirements(raw_schema, schema_components)
+                    }
+                    catch (groovy.lang.MissingPropertyException e) {}
+                    if (requirement_group.size() == 0) {
+                        // fall back to old error if we cannot read the schema
+                        log.error "* Missing required parameter: --${m[0][1]}"
+                        return
+                    }
+                    else if (composition_type == "allOf") {
+                        // similarly, use the old error for all members of an allOf
+                        for (choice in requirement_group) {
+                            log.error "* Missing required parameter: --${choice}"
+                        }
+                        return
+                    }
+                    // construct an appropriate message for the other types of composition
+                    String stderr_msg_prefix = "* Bad parameter configuration. "
+                    if (composition_type == "oneOf") {
+                        stderr_msg_prefix += "You must select only one option of:"
+                    }
+                    else if (composition_type == "anyOf") {
+                        stderr_msg_prefix += "You must select at least one of:"
+                    }
+                    ArrayList stderr_msg = [stderr_msg_prefix]
+                    // glue choices to error
+                    for (choice in requirement_group) {
+                        stderr_msg += "    --${choice}"
+                    }
+                    log.error stderr_msg.join('\n')
+                    // suppress similar warnings
+                    observed_exceptions.add(schema_error_location)
+                }
+                else {
+                    // boring simple required
+                    log.error "* Missing required parameter: --${m[0][1]}"
+                }
+            }
+            // Missing dependent param
+            else if (ex_json['keyword'] == "dependencies") {
+                def m = ex_json['message'] =~ /property \[([^\]]+)\] is required/
+                HashMap<String, String> dependent_group = []
+                ArrayList schema_components = ex_json['schemaLocation'].split('/')
+                // try to check the schema to get the dependencies
+                try {
+                    dependent_group = getSchemaDependencies(raw_schema, schema_components, m[0][1])
+                }
+                catch (groovy.lang.MissingPropertyException e) {}
+                if (dependent_group.size() == 0) {
+                    // fall back to old error if we cannot read the schema
+                    log.error "* Missing required parameter: --${m[0][1]}"
+                    return
+                }
+                // we don't actually get told which parameter triggered the dependency
+                //   only the name of the parameter that is actually missing.
+                // dependent_group will contain a list of triggering keys and their
+                // deps so we can print them all out for the user.
+                for (trigger_key in dependent_group.keySet()) {
+                    String schema_error_location = "${ex_json.schemaLocation}/dependencies/${trigger_key}"
+                    if (schema_error_location in observed_exceptions){
+                        return
+                    }
+                    String stderr_msg_prefix = "* Missing parameter(s) that are required by --${trigger_key}:"
+                    ArrayList stderr_msg = [stderr_msg_prefix]
+                    // list dependencies of trigger_key that are not in the params_json
+                    for (dep_key in dependent_group[trigger_key]) {
+                        if (!params_json.has(dep_key)) {
+                            stderr_msg += "    --${dep_key}"
+                        }
+                    }
+                    log.error stderr_msg.join('\n')
+                    // suppress similar warnings
+                    observed_exceptions.add(schema_error_location)
+                }
             }
             // Other base-level error
             else if (ex_json['pointerToViolation'] == '#') {
@@ -381,11 +515,11 @@ class NfcoreSchema {
                 def param = ex_json['pointerToViolation'] - ~/^#\//
                 def param_val = params_json[param].toString()
                 if (ex_json['message'] =~ /.*not a valid enum value.*/) {
-                    String stderr_msg = "* --${param}: ${param_val} is not a valid choice, pick one of:\n"
+                    ArrayList stderr_msg = ["* --${param}: ${param_val} is not a valid choice, pick one of:"]
                     for (choice in enums[param]) {
-                        stderr_msg += "    - ${choice}\n"
+                        stderr_msg += "    - ${choice}"
                     }
-                    log.error stderr_msg // avoid silly new lines between choices
+                    log.error stderr_msg.join('\n') // avoid silly new lines between choices by printing one string
                 }
                 else {
                     log.error "* --${param}: ${ex_json['message']} (${param_val})"
@@ -393,7 +527,7 @@ class NfcoreSchema {
             }
         }
         for (ex in causingExceptions) {
-            printExceptions(ex, params_json, log, enums)
+            printExceptions(ex, params_json, log, enums, raw_schema, observed_exceptions)
         }
     }
 

main.nf

@@ -545,6 +545,7 @@ workflow pipeline {
                 // So map this reference to all sample_ids
                 seq_for_transcriptome_build = sample_ids.flatten().combine(ref_genome)
             }
+
             get_transcriptome(
                 merge_gff_bundles.out.gff
                 .join(run_gffcompare.out.gffcmp_dir)

nextflow.config

@@ -114,7 +114,7 @@ manifest {
     homePage        = 'https://github.com/epi2me-labs/wf-transcriptomes'
     description     = 'Transcriptome analysis including gene fusions, differential expression as well as assembly and annotation of cDNA and direct RNA sequencing data.'
     mainScript      = 'main.nf'
-    nextflowVersion = '>=22.10.8'
+    nextflowVersion = '>=23.04.2'
     version         = 'v0.2.1'
 }
 

subworkflows/differential_expression.nf

@@ -162,7 +162,8 @@ workflow differential_expression {
     main:
         checkSampleSheetCondition(sample_sheet)
         t_index = build_minimap_index_transcriptome(ref_transcriptome)
-        mapped = map_transcriptome(full_len_reads.combine(t_index))
+        mapped = map_transcriptome(full_len_reads.combine(t_index)
+        .map{meta, fastq, reference, transcriptome -> tuple(meta, fastq, reference) })
         count_transcripts(mapped.bam.combine(t_index.map{ mmi, reference -> reference}))
         merged = mergeCounts(count_transcripts.out.counts.collect())
         merged_TPM = mergeTPM(count_transcripts.out.counts.collect())