From b6159b2c2689fb71c2e98e9348a4896e2db73cc6 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Thu, 3 Aug 2023 15:54:39 +0000 Subject: [PATCH] nf version --- CHANGELOG.md | 4 + lib/NfcoreSchema.groovy | 158 ++++++++++++++++++++++-- main.nf | 1 + nextflow.config | 2 +- subworkflows/differential_expression.nf | 3 +- 5 files changed, 154 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f140bd..ae25ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [updated] +### Changed +- Nextflow minimum required version to 23.04.2 + ## [v0.2.1] ### Changed - Any sample aliases that contain spaces will be replaced with underscores. diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 16f1681..3b29be1 100644 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -182,8 +182,8 @@ class NfcoreSchema { println '' log.error 'ERROR: Validation of pipeline parameters failed!' JSONObject exceptionJSON = e.toJSON() - printExceptions(exceptionJSON, params_json, log, enums) - println '' + HashSet observed_exceptions = [] + printExceptions(exceptionJSON, params_json, log, enums, raw_schema, observed_exceptions) has_error = true } @@ -196,7 +196,7 @@ class NfcoreSchema { warn_msg = warn_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}" } log.warn warn_msg - log.info "- ${colors.dim}Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}" + log.info "${colors.dim}- Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}" println '' } @@ -361,16 +361,150 @@ class NfcoreSchema { return output } + // Traverse the raw_schema to resolve a $ref, returning a JSONObject or JSONArray. + private static def getSchemaLocation(raw_schema, schema_components) { + // a quick search did not reveal how to resolve a $ref inside a + // JSONObject so we'll go with traversing it ourselves! + def head = raw_schema + for (component in schema_components) { + if (component == '#') { + continue + } + else if (component.isNumber()) { + // assume numeric components are indexing an array + head = head[component as int] + } + else { + // otherwise grab by key + head = head[component] + } + } + return head + } + + // Return a map of dependencies for a path in a schema, that relate to a key that is known to be missing. + // Each key in the map is a parameter that has been switched on and has a requirement on the missing key. + // The value of each key in the map is a list of all parameters that are required by the corresponding key. + private static HashMap getSchemaDependencies(raw_schema, schema_components, missing_key) { + def head = getSchemaLocation(raw_schema, schema_components[1..-1] + 'dependencies') + HashMap requirements = [] + for (key in head.keySet()) { + if (head[key].contains(missing_key)) { + requirements[key] = head[key] + } + } + return requirements + } + + // Return a list of all required parameters that are adjacent to a given path in a schema + private static ArrayList getSchemaRequirements(raw_schema, schema_components) { + def head = getSchemaLocation(raw_schema, schema_components[1..-2]) + ArrayList requirements = [] + for (entry in head) { + // this could be an allOf with more allOf/oneOf/anyOf inside + // we'll ignore anything that isn't a required key + if (entry.has("required")) { + requirements += entry["required"] + } + } + return requirements + } + // // Loop over nested exceptions and print the causingException // - private static void printExceptions(ex_json, params_json, log, enums) { + private static void printExceptions(ex_json, params_json, log, enums, raw_schema, observed_exceptions) { def causingExceptions = ex_json['causingExceptions'] if (causingExceptions.length() == 0) { - def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/ - // Missing required param - if (m.matches()) { - log.error "* Missing required parameter: --${m[0][1]}" + // Missing required param... + if (ex_json["keyword"] == "required") { + def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/ + // ...but what sort of required param + ArrayList schema_components = ex_json['schemaLocation'].split('/') + if (schema_components[-1].isNumber()) { + // schemaLocation ends in array position, indicating use of + // xOf in the schema. this is some sort of complex definition + String schema_error_location = schema_components[1..-2].join('.') + String composition_type = schema_components[-2] + if (schema_error_location in observed_exceptions){ + return + } + // try to check the schema to get the adjacent requirements + ArrayList requirement_group = [] + try { + requirement_group = getSchemaRequirements(raw_schema, schema_components) + } + catch (groovy.lang.MissingPropertyException e) {} + if (requirement_group.size() == 0) { + // fall back to old error if we cannot read the schema + log.error "* Missing required parameter: --${m[0][1]}" + return + } + else if (composition_type == "allOf") { + // similarly, use the old error for all members of an allOf + for (choice in requirement_group) { + log.error "* Missing required parameter: --${choice}" + } + return + } + // construct an appropriate message for the other types of composition + String stderr_msg_prefix = "* Bad parameter configuration. " + if (composition_type == "oneOf") { + stderr_msg_prefix += "You must select only one option of:" + } + else if (composition_type == "anyOf") { + stderr_msg_prefix += "You must select at least one of:" + } + ArrayList stderr_msg = [stderr_msg_prefix] + // glue choices to error + for (choice in requirement_group) { + stderr_msg += " --${choice}" + } + log.error stderr_msg.join('\n') + // suppress similar warnings + observed_exceptions.add(schema_error_location) + } + else { + // boring simple required + log.error "* Missing required parameter: --${m[0][1]}" + } + } + // Missing dependent param + else if (ex_json['keyword'] == "dependencies") { + def m = ex_json['message'] =~ /property \[([^\]]+)\] is required/ + HashMap dependent_group = [] + ArrayList schema_components = ex_json['schemaLocation'].split('/') + // try to check the schema to get the dependencies + try { + dependent_group = getSchemaDependencies(raw_schema, schema_components, m[0][1]) + } + catch (groovy.lang.MissingPropertyException e) {} + if (dependent_group.size() == 0) { + // fall back to old error if we cannot read the schema + log.error "* Missing required parameter: --${m[0][1]}" + return + } + // we don't actually get told which parameter triggered the dependency + // only the name of the parameter that is actually missing. + // dependent_group will contain a list of triggering keys and their + // deps so we can print them all out for the user. + for (trigger_key in dependent_group.keySet()) { + String schema_error_location = "${ex_json.schemaLocation}/dependencies/${trigger_key}" + if (schema_error_location in observed_exceptions){ + return + } + String stderr_msg_prefix = "* Missing parameter(s) that are required by --${trigger_key}:" + ArrayList stderr_msg = [stderr_msg_prefix] + // list dependencies of trigger_key that are not in the params_json + for (dep_key in dependent_group[trigger_key]) { + if (!params_json.has(dep_key)) { + stderr_msg += " --${dep_key}" + } + } + log.error stderr_msg.join('\n') + // suppress similar warnings + observed_exceptions.add(schema_error_location) + } } // Other base-level error else if (ex_json['pointerToViolation'] == '#') { @@ -381,11 +515,11 @@ class NfcoreSchema { def param = ex_json['pointerToViolation'] - ~/^#\// def param_val = params_json[param].toString() if (ex_json['message'] =~ /.*not a valid enum value.*/) { - String stderr_msg = "* --${param}: ${param_val} is not a valid choice, pick one of:\n" + ArrayList stderr_msg = ["* --${param}: ${param_val} is not a valid choice, pick one of:"] for (choice in enums[param]) { - stderr_msg += " - ${choice}\n" + stderr_msg += " - ${choice}" } - log.error stderr_msg // avoid silly new lines between choices + log.error stderr_msg.join('\n') // avoid silly new lines between choices by printing one string } else { log.error "* --${param}: ${ex_json['message']} (${param_val})" @@ -393,7 +527,7 @@ class NfcoreSchema { } } for (ex in causingExceptions) { - printExceptions(ex, params_json, log, enums) + printExceptions(ex, params_json, log, enums, raw_schema, observed_exceptions) } } diff --git a/main.nf b/main.nf index 103dd8a..b04eef1 100644 --- a/main.nf +++ b/main.nf @@ -545,6 +545,7 @@ workflow pipeline { // So map this reference to all sample_ids seq_for_transcriptome_build = sample_ids.flatten().combine(ref_genome) } + get_transcriptome( merge_gff_bundles.out.gff .join(run_gffcompare.out.gffcmp_dir) diff --git a/nextflow.config b/nextflow.config index 677ed0f..a72d262 100644 --- a/nextflow.config +++ b/nextflow.config @@ -114,7 +114,7 @@ manifest { homePage = 'https://github.com/epi2me-labs/wf-transcriptomes' description = 'Transcriptome analysis including gene fusions, differential expression as well as assembly and annotation of cDNA and direct RNA sequencing data.' mainScript = 'main.nf' - nextflowVersion = '>=22.10.8' + nextflowVersion = '>=23.04.2' version = 'v0.2.1' } diff --git a/subworkflows/differential_expression.nf b/subworkflows/differential_expression.nf index 39c34fe..439ddf0 100644 --- a/subworkflows/differential_expression.nf +++ b/subworkflows/differential_expression.nf @@ -162,7 +162,8 @@ workflow differential_expression { main: checkSampleSheetCondition(sample_sheet) t_index = build_minimap_index_transcriptome(ref_transcriptome) - mapped = map_transcriptome(full_len_reads.combine(t_index)) + mapped = map_transcriptome(full_len_reads.combine(t_index) + .map{meta, fastq, reference, transcriptome -> tuple(meta, fastq, reference) }) count_transcripts(mapped.bam.combine(t_index.map{ mmi, reference -> reference})) merged = mergeCounts(count_transcripts.out.counts.collect()) merged_TPM = mergeTPM(count_transcripts.out.counts.collect())