kfukuta/wf-transcriptomes-v202
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support s3 input paths

Browse Source
This commit is contained in:
Chris Wright 2021-09-28 17:11:33 +01:00
parent 0d230e3930
commit 038d399b12
1 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/fastqingress.nf
View File

@ -1,3 +1,4 @@
process checkSampleSheet { process checkSampleSheet {
label "artic" label "artic"
cpus 1 cpus 1
@ -33,7 +34,7 @@ def check_sample_sheet(samples)
* Find fastq data using various globs. Wrapper around Nextflow `file` * Find fastq data using various globs. Wrapper around Nextflow `file`
* method. * method.
* *
* @param patten glob pattern for top level input folder. * @param pattern file object corresponding to top level input folder.
* @param maxdepth maximum depth to traverse * @param maxdepth maximum depth to traverse
* @return list of files. * @return list of files.
*/ */
@ -42,7 +43,7 @@ def find_fastq(pattern, maxdepth)
files = [] files = []
extensions = ["fastq", "fastq.gz", "fq", "fq.gz"] extensions = ["fastq", "fastq.gz", "fq", "fq.gz"]
for (ext in extensions) { for (ext in extensions) {
files += file("${pattern}/*.${ext}", type: 'file', maxdepth: maxdepth) files += file(pattern.resolve("*.${ext}"), type: 'file', maxdepth: maxdepth)
} }
return files return files
} }
@ -54,17 +55,17 @@ def find_fastq(pattern, maxdepth)
* subdirectories ready for processing. * subdirectories ready for processing.
* *
* @param input_folder Top-level input directory. * @param input_folder Top-level input directory.
* @param output_folder Top-level output_directory. * @param staging Top-level output_directory.
* @return A File object representating the staging directory created * @return A File object representating the staging directory created
* under output_folder * under output_folder
*/ */
def sanitize_fastq(input_folder, output_folder) def sanitize_fastq(input_folder, staging)
{ {
// TODO: this fails if input_folder is an S3 path
println("Running sanitization.") println("Running sanitization.")
println(" - Moving files: ${input_folder} -> ${output_folder}") println(" - Moving files: ${input_folder} -> ${staging}")
staging = new File(output_folder)
staging.mkdirs() staging.mkdirs()
files = find_fastq("${input_folder}/**/", 1) files = find_fastq(input_folder.resolve("**"), 1)
for (fastq in files) { for (fastq in files) {
fname = fastq.getFileName() fname = fastq.getFileName()
// find barcode // find barcode
@ -72,11 +73,11 @@ def sanitize_fastq(input_folder, output_folder)
matcher = fname =~ pattern matcher = fname =~ pattern
if (!matcher.find()) { if (!matcher.find()) {
// not barcoded - leave alone // not barcoded - leave alone
fastq.renameTo("${staging}/${fname}") fastq.renameTo(staging.resolve(fname))
} else { } else {
bc_dir = new File("${staging}/${matcher[0]}") bc_dir = file(staging.resolve(matcher[0]))
bc_dir.mkdirs() bc_dir.mkdirs()
fastq.renameTo("${staging}/${matcher[0]}/${fname}") fastq.renameTo(staging.resolve("${matcher[0]}/${fname}"))
} }
} }
println(" - Finished sanitization.") println(" - Finished sanitization.")
@ -98,7 +99,7 @@ def resolve_barcode_structure(input_folder, sample_sheet)
{ {
println("Checking input directory structure.") println("Checking input directory structure.")
barcode_dirs = file("$input_folder/barcode*", type: 'dir', maxdepth: 1) barcode_dirs = file("$input_folder/barcode*", type: 'dir', maxdepth: 1)
not_barcoded = find_fastq("$input_folder/", 1) not_barcoded = find_fastq(file(input_folder), 1)
samples = null samples = null
if (barcode_dirs) { if (barcode_dirs) {
println(" - Found barcode directories") println(" - Found barcode directories")
@ -155,8 +156,8 @@ def fastq_ingress(input_folder, output_folder, samples, sanitize)
{ {
// EPI2ME harness // EPI2ME harness
if (sanitize) { if (sanitize) {
staging = "${output_folder}/staging" staging = file(output_folder).resolve("staging")
input_folder = sanitize_fastq(input_folder, staging) input_folder = sanitize_fastq(file(input_folder), staging)
} }
// check sample sheet // check sample sheet
sample_sheet = null sample_sheet = null