Skip to content

Commit

Permalink
Merge pull request #845 from mgcam/update_rf_test_data
Browse files Browse the repository at this point in the history
Update tests
jmtcsngr authored May 30, 2024
2 parents e7418d2 + 1297dc0 commit 9351315
Showing 31 changed files with 515 additions and 600 deletions.
20 changes: 4 additions & 16 deletions MANIFEST
Original file line number Diff line number Diff line change
@@ -124,10 +124,8 @@ t/50-npg_pipeline-daemon-archival.t
t/bin/bkill
t/bin/bresume
t/bin/bsub
t/data/bam_flagstats/1234_1_bam_flagstats.json
t/data/bam_flagstats/1234_3_bam_flagstats.json
t/data/bam_flagstats/1234_3_phix_bam_flagstats.json
t/data/bam_flagstats/1234_4_bam_flagstats.json
t/data/bam_flagstats/47995_3_bam_flagstats.json
t/data/bam_flagstats/47995_3_phix_bam_flagstats.json
t/data/barcodes/samplesheet_batch2015.csv
t/data/barcodes/samplesheet_batch42225.csv
t/data/barcodes/samplesheet_batch42225_amended1.csv
@@ -229,6 +227,7 @@ t/data/miseq/16850_runParameters.xml
t/data/miseq/16866_RunInfo.xml
t/data/miseq/20990_RunInfo.xml
t/data/miseq/24135_RunInfo.xml
t/data/miseq/24347_RunInfo.xml
t/data/miseq/samplesheet_16850.csv
t/data/miseq/samplesheet_16866.csv
t/data/miseq/samplesheet_20990.csv
@@ -1011,7 +1010,7 @@ t/data/novaseqx/47539/samplesheet_47539.csv
t/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3/RunInfo.xml
t/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3/RunParameters.xml
t/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3/samplesheet_47995.csv
t/data/p4_stage1_analysis/1234_samplesheet.csv
t/data/p4_stage1_analysis/samplesheet.csv
t/data/p4_stage1_analysis/TileMetricsOut.bin
t/data/portable_pipelines/ncov2019-artic-nf/cf01166c42a/product_release.yml
t/data/portable_pipelines/ncov2019-artic-nf/cf01166c42a/product_release_no_pp.yml
@@ -1061,17 +1060,6 @@ t/data/run_params/runParameters.hiseqx.upgraded.xml
t/data/run_params/runParameters.miseq.xml
t/data/run_params/runParameters.novaseq.xml
t/data/run_params/RunParameters.novaseqx.xml
t/data/runfolder/archive/1234_2.bam
t/data/runfolder/archive/1234_2_human.bam
t/data/runfolder/archive/1234_3.bam
t/data/runfolder/archive/lane1/1234_1#15.bam
t/data/runfolder/archive/lane1/1234_1#15.cram
t/data/runfolder/archive/lane1/1234_1#15.seqchksum
t/data/runfolder/archive/lane1/1234_1#15_human.bam
t/data/runfolder/archive/lane1/1234_1#15_phix.bam
t/data/runfolder/archive/lane4/1234_4#16.bam
t/data/runfolder/archive/lane4/1234_4#32.bam
t/data/runfolder/Data/RunInfo.xml
t/data/samplesheet_1234.csv
t/data/samplesheet_8747.csv
t/data/samplesheet_33990.csv
116 changes: 59 additions & 57 deletions t/10-pluggable-central.t
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
use strict;
use warnings;
use Test::More tests => 22;
use Test::More tests => 4;
use Test::Exception;
use Log::Log4perl qw(:levels);
use File::Copy qw(cp);
use File::Path qw(make_path);
use File::Temp qw(tempdir);

use t::util;

@@ -18,6 +19,7 @@ foreach my $tool (@tools) {
}
chmod 0755, @tools;
local $ENV{'PATH'} = join q[:], $tdir, $ENV{'PATH'};
local $ENV{'HOME'} = 't';

my $product_config = q[t/data/release/config/archive_on/product_release.yml];

@@ -26,74 +28,82 @@ Log::Log4perl->easy_init({layout => '%d %-5p %c - %m%n',
file => join(q[/], $tdir, 'logfile'),
utf8 => 1});

my $test_data_dir_47995 = 't/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3';
sub _setup_runfolder_47995 {
my $tmp_dir = tempdir(CLEANUP => 1);
my @dirs = split q[/], $test_data_dir_47995;
my $rf_name = pop @dirs;
my $rf_info = $util->create_runfolder($tmp_dir, {'runfolder_name' => $rf_name});
my $rf = $rf_info->{'runfolder_path'};
for my $file (qw(RunInfo.xml RunParameters.xml)) {
if (cp("$test_data_dir_47995/$file", "$rf/$file") == 0) {
die "Failed to copy $file";
}
}
return $rf_info;
}


my $central = q{npg_pipeline::pluggable::central};
use_ok($central);

my $runfolder_path = $util->analysis_runfolder_path();
subtest 'test object creation' => sub {
plan tests => 4;

{
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q[t/data/samplesheet_1234.csv];
$util->create_analysis();
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = "$test_data_dir_47995/samplesheet_47995.csv";
my $pipeline;
lives_ok {
$pipeline = $central->new(
runfolder_path => $runfolder_path,
runfolder_path => $tdir,
);
} q{no croak creating new object};
} q{no error creating new object};
isa_ok($pipeline, $central);
}

{
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q[t/data/samplesheet_1234.csv];
my $pb;
lives_ok {
$pb = $central->new(
$pipeline = $central->new(
function_order => [qw(qc_qX_yield qc_insert_size)],
runfolder_path => $runfolder_path,
runfolder_path => $tdir,
);
} q{no croak on creation};
$util->create_analysis();
is(join(q[ ], @{$pb->function_order()}), 'qc_qX_yield qc_insert_size',
} q{no error on creation};
is(join(q[ ], @{$pipeline->function_order()}), 'qc_qX_yield qc_insert_size',
'function_order set on creation');
}
};

{
local $ENV{CLASSPATH} = undef;
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q[t/data/samplesheet_1234.csv];
my $pb;
$util->create_analysis();
cp 't/data/run_params/runParameters.hiseq.xml',
join(q[/], $runfolder_path, 'runParameters.xml');
subtest 'execute main()' => sub {
plan tests => 2;

$util->create_run_info();
local $ENV{CLASSPATH} = undef;
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = "$test_data_dir_47995/samplesheet_47995.csv";
my $rf_info = _setup_runfolder_47995();
my $config_dir = 'data/config_files';
my $init = {
function_order => [qw{qc_qX_yield qc_adapter update_ml_warehouse qc_insert_size}],
lanes => [4],
runfolder_path => $runfolder_path,
function_list => "$config_dir/function_list_central.json",
id_flowcell_lims => 2015,
no_bsub => 1,
repository => 't/data/sequence',
spider => 0,
no_sf_resource => 1,
product_conf_file_path => $product_config,
};

lives_ok { $pb = $central->new($init); } q{no croak on new creation};
mkdir $pb->archive_path;
my $pb;
lives_ok { $pb = $central->new(
id_run => 47995,
function_order => [qw{qc_qX_yield qc_adapter update_ml_warehouse qc_insert_size}],
lanes => [4],
run_folder => $rf_info->{'runfolder_name'},
runfolder_path => $rf_info->{'runfolder_path'},
function_list => "$config_dir/function_list_central.json",
id_flowcell_lims => 17089,
no_bsub => 1,
repository => 't/data/sequence',
spider => 0,
product_conf_file_path => $product_config,
); } q{no croak on new creation};

lives_ok { $pb->main() } q{no croak running qc->main()};
}
};

subtest 'execute prepare()' => sub {
plan tests => 12;

{
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q[t/data/samplesheet_1234.csv];
my $rf = join q[/], $tdir, 'myfolder';
mkdir $rf;
cp 't/data/run_params/runParameters.hiseq.xml',
join(q[/], $rf, 'runParameters.xml');
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = "$test_data_dir_47995/samplesheet_47995.csv";
my $rf_info = _setup_runfolder_47995();
my $rf = $rf_info->{'runfolder_path'};
my $init = {
id_run => 1234,
run_folder => 'myfolder',
id_run => 47995,
run_folder => $rf_info->{'runfolder_name'},
runfolder_path => $rf,
timestamp => '22-May',
spider => 0,
@@ -103,14 +113,6 @@ my $runfolder_path = $util->analysis_runfolder_path();
my $pb = $central->new($init);
is ($pb->intensity_path, "$rf/Data/Intensities", 'intensities path');
is ($pb->basecall_path, "$rf/Data/Intensities/BaseCalls", 'basecalls path');
throws_ok { $pb->prepare() }
qr/does not exist, either bam_basecall_path or analysis_path should be given/,
q{error scaffolding the run folder};

make_path "$rf/Data/Intensities";
$pb = $central->new($init);
is ($pb->intensity_path, "$rf/Data/Intensities", 'intensities path');
is ($pb->basecall_path, "$rf/Data/Intensities/BaseCalls", 'basecalls path');
lives_ok { $pb->prepare() } 'prepare runs fine';
my $expected_pb_cal = join q[/],$rf,q{Data/Intensities/BAM_basecalls_22-May};
is ($pb->bam_basecall_path, $expected_pb_cal, 'bam basecall path is set');
@@ -137,6 +139,6 @@ my $runfolder_path = $util->analysis_runfolder_path();
$pb = $central->new($init);
$pb->prepare();
is ($pb->bam_basecall_path, $expected_pb_cal, 'bam basecall path is set');
}
};

1;
1 change: 1 addition & 0 deletions t/10-pluggable.t
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@ foreach my $tool (@tools) {
}
chmod 0755, @tools;
local $ENV{'PATH'} = join q[:], $test_bin, $ENV{'PATH'};
local $ENV{'HOME'} = 't';

Log::Log4perl->easy_init({layout => '%d %-5p %c - %m%n',
level => $DEBUG,
127 changes: 79 additions & 48 deletions t/15-pipeline_launcher_scripts.t
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
use strict;
use warnings;
use English qw{-no_match_vars};
use Test::More tests => 10;
use English qw(-no_match_vars);
use Test::More tests => 4;
use Test::Exception;
use File::Copy;
use Cwd;
use File::Temp qw(tempdir);
use File::Path qw(make_path);

use t::util;

@@ -20,69 +22,98 @@ foreach my $tool (@tools) {
}
chmod 0755, @tools;

local $ENV{'PATH'} = join q[:], $tmp_dir, $bin, $ENV{'PATH'};
local $ENV{'http_proxy'} = q[http://wibble];
local $ENV{'no_proxy'} = q[];
local $ENV{'PATH'} = join q[:], $tmp_dir, $bin, $ENV{'PATH'};
local $ENV{'HOME'} = 't';

my $rf = $util->analysis_runfolder_path;
my $bbp = "$rf/bam_basecall_path";
my $product_config = q[t/data/release/config/archive_on/product_release.yml];
my $test_data_dir_47995 = 't/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3';

sub _setup_runfolder_47995 {
my $tmp_dir = tempdir(CLEANUP => 1);
my @dirs = split q[/], $test_data_dir_47995;
my $rf_name = pop @dirs;
my $rf_info = $util->create_runfolder($tmp_dir, {'runfolder_name' => $rf_name});
my $rf = $rf_info->{'runfolder_path'};
for my $file (qw(RunInfo.xml RunParameters.xml)) {
if (copy("$test_data_dir_47995/$file", "$rf/$file") == 0) {
die "Failed to copy $file";
}
}
return $rf_info;
}

{
my $rf_info = _setup_runfolder_47995();
my $rf = $rf_info->{'runfolder_path'};
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q{/does/not/exist.csv};
$util->create_analysis();
my $out = `$bin/npg_pipeline_central --product_conf_file_path $product_config --spider --no_bsub --no_sf_resource --runfolder_path $rf --function_order dodo 2>&1`;
like($out,
qr/Error initializing pipeline: Error while spidering/,
'error in spidering when pre-set samplesheet does not exist');
my $command = "$bin/npg_pipeline_central " .
"--product_conf_file_path $product_config --spider --no_bsub " .
"--runfolder_path $rf --function_order dodo 2>&1";
note "Executing $command";
like(`$command`, qr/Error initializing pipeline: Error while spidering/,
'error in spidering when pre-set samplesheet does not exist');
}

{
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q{t/data/samplesheet_1234.csv};
$util->create_analysis();
$util->create_run_info();

my $out = `$bin/npg_pipeline_central --product_conf_file_path $product_config --no-spider --no_bsub --no_sf_resource --runfolder_path $rf --bam_basecall_path $bbp --function_order dodo 2>&1`;
like($out,
qr/Function dodo cannot be found in the graph/,
'error when function does not exist');
}
subtest 'test analysis and archival pipeline scripts' => sub {
plan tests => 5;

# A full run folder is scaffolded by the analysis pipeline.
# The archival pipeline is using teh same run folder.

my $rf_info = _setup_runfolder_47995();
my $rf = $rf_info->{'runfolder_path'};
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = "$test_data_dir_47995/samplesheet_47995.csv";
my $command = "$bin/npg_pipeline_central " .
"--product_conf_file_path $product_config --no-spider --no_bsub " .
"--runfolder_path $rf --function_order create_summary_link_analysis " .
"--function_order dodo 2>&1";
note "Executing $command";
like(`$command`, qr/Function dodo cannot be found in the graph/,
'error when function does not exist');

{
local $ENV{NPG_CACHED_SAMPLESHEET_FILE} = q{t/data/samplesheet_1234.csv};
my $config_dir = join q[/], $tmp_dir, 'config';
mkdir $config_dir;
make_path($config_dir);
my @files = glob 'data/config_files/*.{json,ini}';
push @files, 't/data/release/config/archive_on/product_release.yml';
for (@files) {
copy $_, $config_dir;
}

lives_ok { qx{
$bin/npg_pipeline_post_qc_review --no_bsub --no_sf_resource --runfolder_path $rf --bam_basecall_path $bbp --conf_path $config_dir};}
q{ran bin/npg_pipeline_post_qc_review};
ok(!$CHILD_ERROR, qq{Return code of $CHILD_ERROR});

lives_ok { qx{
$bin/npg_pipeline_post_qc_review --no_bsub --no_sf_resource --runfolder_path $rf --bam_basecall_path $bbp --function_list some --conf_path $config_dir}; }
q{ran bin/npg_pipeline_post_qc_review with non-exisiting function list};
ok($CHILD_ERROR, qq{Child error $CHILD_ERROR});
}

{
$util->create_analysis();
$util->create_run_info();

lives_ok { qx{$bin/npg_pipeline_seqchksum_comparator --id_run=1234 --archive_path=$rf/Data/Intensities/BAM_basecalls_20140815-114817/no_cal/archive --bam_basecall_path=$rf/Data/Intensities/BAM_basecalls_20140815-114817 --lanes=1 };} q{ran bin/npg_pipeline_seqchksum_comparator with analysis and bam_basecall_path};
$command = "$bin/npg_pipeline_post_qc_review --no_bsub --runfolder_path $rf " .
"--conf_path $config_dir";
note "Executing $command";
lives_ok { `$command` } 'ran bin/npg_pipeline_post_qc_review';
ok(!$CHILD_ERROR, 'No error running command');

$command = "$bin/npg_pipeline_post_qc_review --no_bsub --runfolder_path $rf " .
"--conf_path $config_dir --function_list some";
note "Executing $command";
lives_ok { `$command` }
'ran bin/npg_pipeline_post_qc_review with non-exisiting function list';
ok($CHILD_ERROR, "Child error $CHILD_ERROR");
};

subtest 'test npg_pipeline_seqchksum_comparator script' => sub {
plan tests => 2;
my $rf_info = _setup_runfolder_47995();
my $rf = $rf_info->{'runfolder_path'};
my $bbc = "$rf/Data/Intensities/BAM_basecalls";
my $apath = "$bbc/no_cal/archive";
make_path($apath);
my $command = "$bin/npg_pipeline_seqchksum_comparator --id_run=1234 " .
"--archive_path=$apath --bam_basecall_path=$bbc --lanes=1";
note "Executing $command";
lives_ok { `$command` }
'ran npg_pipeline_seqchksum_comparator with analysis and bam_basecall_path';
ok($CHILD_ERROR, qq{Return code of $CHILD_ERROR as no files found});
}
};

{
subtest 'test npg_pipeline_preexec_references script' => sub {
plan tests => 2;
`bin/npg_pipeline_preexec_references --repository t/data/sequence/refs 2>/dev/null`;
ok( $CHILD_ERROR, qq{failed as could not locate references directory - $CHILD_ERROR} );

qx{bin/npg_pipeline_preexec_references --repository t/data/sequence};
ok( ! $CHILD_ERROR, q{script runs OK} );
}
ok($CHILD_ERROR, "failed as could not locate references directory - $CHILD_ERROR");
`bin/npg_pipeline_preexec_references --repository t/data/sequence`;
ok(! $CHILD_ERROR, 'script runs OK');
};

1;
Loading

0 comments on commit 9351315

Please sign in to comment.