1
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 4;
|
||||
|
||||
ok($> == 0, "Running as 'root'") or
|
||||
BAIL_OUT("Rerun tests as 'root'");
|
||||
|
||||
my $message = "A username must be supplied for testing job submission.\n"
|
||||
. "Do one of the following:\n"
|
||||
. "- Set the username using the -u commandline parameter\n"
|
||||
. " torque.test -u <USERNAME>\n"
|
||||
. "- Set the \$TORQUE_TEST_USER environment variable.\n";
|
||||
|
||||
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
|
||||
do
|
||||
{
|
||||
diag($message);
|
||||
BAIL_OUT('Set test username');
|
||||
};
|
||||
|
||||
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
|
||||
do
|
||||
{
|
||||
diag($message);
|
||||
BAIL_OUT('Set test username');
|
||||
};
|
||||
|
||||
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
do
|
||||
{
|
||||
diag($message);
|
||||
BAIL_OUT('Set test username');
|
||||
};
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 7;
|
||||
|
||||
# pbs_server - TORQUE Install
|
||||
my $pbs_server = `which pbs_server` || undef;
|
||||
ok(defined $pbs_server, 'TORQUE Install') or
|
||||
BAIL_OUT('Cannot locate TORQUE server daemon - install TORQUE or add TORQUE install directory to $PATH');
|
||||
|
||||
# qstat - Client Tools
|
||||
my $qstat = `which qstat` || undef;
|
||||
ok(defined $qstat, 'Client Tools') or
|
||||
BAIL_OUT('Cannot locate TORQUE clients, install TORQUE or add TORQUE install directory to $PATH');
|
||||
|
||||
# version - TORQUE version
|
||||
my $version = `qstat --version 2>&1`;
|
||||
$version =~ /^version:\s*([\d.]+)\s*$/i;
|
||||
$version = $1 || undef;
|
||||
ok(defined $version, 'TORQUE Version') or
|
||||
BAIL_OUT('Cannot determine TORQUE version');
|
||||
|
||||
# sbin/ - Install Location
|
||||
my $sbindir = $pbs_server;
|
||||
$sbindir =~ s/pbs_server\s*$//;
|
||||
ok($sbindir ne '', 'Determine sbin') or
|
||||
BAIL_OUT('Cannot determine SBIN directory');
|
||||
ok(-d $sbindir,'sbin Exists') or
|
||||
BAIL_OUT("$sbindir does not seem to exist");
|
||||
|
||||
# PID - TORQUE Currently Running
|
||||
my $pid = `ps -ef | grep pbs_server | grep -v grep`;
|
||||
$pid =~ /^\S+\s+(\d+)\s+/;
|
||||
$pid = $1 || undef;
|
||||
ok(defined $pid, 'TORQUE Currently Running') or
|
||||
BAIL_OUT('pbs_server not running, start pbs_server and rerun test - see TORQUE docs, Section 1.2.4');
|
||||
|
||||
# qmgr - TORQUE Database Created
|
||||
my $qmgr = `qmgr -c 'p s'` || undef;
|
||||
ok(defined $qmgr, 'Database Created') or
|
||||
BAIL_OUT("TORQUE database not created, restart pbs_server using 'pbs_server -t create' - see TORQUE docs, Section 1.2.1");
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 8;
|
||||
|
||||
# pbs_server --about - Directory Information
|
||||
my @pbs_server = `pbs_server --about`;
|
||||
ok(scalar @pbs_server, 'Directory Information') or
|
||||
BAIL_OUT('Cannot get directory infromation from pbs_server');
|
||||
|
||||
# TORQUE Home Directory
|
||||
my $torquehome = undef;
|
||||
foreach my $line (@pbs_server)
|
||||
{
|
||||
if ($line =~ /^serverhome:\s+(.+)\s*$/i)
|
||||
{
|
||||
$torquehome = $1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
ok(defined $torquehome, 'TORQUE Home Directory') or
|
||||
BAIL_OUT('Cannot determine server home directory from pbs_server');
|
||||
ok(-d $torquehome, 'TORQUE Home Directory Exists') or
|
||||
BAIL_OUT("The server home directory ($torquehome) does not exist");
|
||||
|
||||
# pbsnodes - Node List
|
||||
my $nodesfile = "$torquehome/server_priv/nodes";
|
||||
$nodesfile =~ s#//#/#g;
|
||||
ok(-T $nodesfile, 'Nodes File Exists') or
|
||||
BAIL_OUT("TORQUE 'nodes' file not created, setup $nodesfile and restart pbs_server - see TORQUE docs, Section 1.2.2");
|
||||
my @pbsnodes = map { s/\s+$//g; $_ }
|
||||
grep { /^\S/ }
|
||||
`pbsnodes -a`;
|
||||
ok(scalar @pbsnodes, 'pbsnodes Reports Nodes') or
|
||||
BAIL_OUT("TORQUE 'nodes' file not created, setup $nodesfile and restart pbs_server - see TORQUE docs, Section 1.2.2");
|
||||
ok($pbsnodes[0] ne 'no nodes', 'pbsnodes Lists Nodes') or
|
||||
BAIL_OUT("TORQUE 'nodes' file not created, setup $nodesfile and restart pbs_server - see TORQUE docs, Section 1.2.2");
|
||||
|
||||
# Node Count
|
||||
my $nodecount = scalar @pbsnodes;
|
||||
ok($nodecount, 'Node Count') or
|
||||
BAIL_OUT('TORQUE reporting zero nodes available');
|
||||
|
||||
# Active Nodes
|
||||
my @upnodes = grep { /down|unknown/i }
|
||||
grep { /^\s+state\s+=\s+/i }
|
||||
`pbsnodes -a`;
|
||||
ok(!scalar(@upnodes), 'Nodes Have Connected to Server') or
|
||||
do {
|
||||
diag
|
||||
(
|
||||
"Compute nodes have not contacted pbs_server, verify the following:\n"
|
||||
. "- pbs_mom daemons are running on hosts\n"
|
||||
. "- compute hostnames match names listed in $torquehome/server_priv/nodes\n"
|
||||
. "- compute hosts are network-accessible from the head node\n"
|
||||
. "- the command 'momctl -d3 -h $pbsnodes[0]' does not report any errors\n"
|
||||
);
|
||||
BAIL_OUT('Compute nodes have not contacted pbs_server');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More qw(no_plan);
|
||||
|
||||
# Queue List
|
||||
my (@xqueues, %queues);
|
||||
# - Build first-level entries for queue hash table
|
||||
map
|
||||
{
|
||||
/^create queue (\w+)/i;
|
||||
$queues{$1} = ();
|
||||
}
|
||||
grep
|
||||
{
|
||||
/^create queue \w+/i
|
||||
}
|
||||
`qmgr -c 'p s'`;
|
||||
# - Populate additional entries for each queue in the hash
|
||||
foreach my $queue (keys %queues)
|
||||
{
|
||||
my $data = `qmgr -c 'p queue $queue'`;
|
||||
push(@xqueues, $queue)
|
||||
if $data =~ /^set queue $queue queue_type\s+=\s+Execution/im;
|
||||
foreach my $line (split /[\r\n]+/, $data)
|
||||
{
|
||||
next unless $line =~ /^set queue $queue (\S+) = (\S+)/;
|
||||
$queues{$queue}{$1} = $2;
|
||||
}
|
||||
}
|
||||
# - Verify that queues exist
|
||||
ok(scalar (keys %queues), 'Configured Queues') or
|
||||
BAIL_OUT('No TORQUE queues are configured - run torque.setup or see TORQUE docs, Sections 1.2.1 & 4.1');
|
||||
ok(scalar @xqueues, 'Execution Queues') or
|
||||
BAIL_OUT('No execution queues are configured - run torque.setup or see TORQUE docs, Sections 1.2.1 & 4.1');
|
||||
|
||||
# Queue Check
|
||||
my $queuesOK = 1;
|
||||
foreach my $queue (keys %queues)
|
||||
{
|
||||
ok($queues{$queue}{'enabled'} =~ /^True$/i, "Queue '$queue' Enabled") or
|
||||
do
|
||||
{
|
||||
diag("TORQUE queue '$queue' has not been enabled - see TORQUE docs, Section 4.1");
|
||||
$queuesOK = 0;
|
||||
};
|
||||
ok($queues{$queue}{'started'} =~ /^True$/i, "Queue '$queue' Started") or
|
||||
do
|
||||
{
|
||||
diag("TORQUE queue '$queue' has not been started - see TORQUE docs, Section 4.1");
|
||||
$queuesOK = 0;
|
||||
};
|
||||
}
|
||||
ok($queuesOK, 'TORQUE Queues Configured') or
|
||||
BAIL_OUT("TORQUE queues have not been correctly configured - run torque.setup or see section 4.1");
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 2;
|
||||
|
||||
use constant MAUI => 'maui';
|
||||
use constant MOAB => 'moab';
|
||||
use constant FIFO => 'fifo';
|
||||
|
||||
my %schedulers;
|
||||
|
||||
# Installed Schedulers
|
||||
{
|
||||
# Check Maui
|
||||
$schedulers{MAUI} = `which maui 2>/dev/null` || undef;
|
||||
delete $schedulers{MAUI}
|
||||
unless defined $schedulers{MAUI};
|
||||
|
||||
# Check Moab
|
||||
$schedulers{MOAB} = `which moab 2>/dev/null` || undef;
|
||||
delete $schedulers{MOAB}
|
||||
unless defined $schedulers{MOAB};
|
||||
|
||||
# Check FIFO
|
||||
$schedulers{FIFO} = `which pbs_sched 2>/dev/null` || undef;
|
||||
delete $schedulers{FIFO}
|
||||
unless defined $schedulers{FIFO};
|
||||
}
|
||||
ok(scalar (keys %schedulers), 'Installed Scheduler') or
|
||||
diag("No recognized scheduler is installed - install Moab, Maui, or pbs_sched and rerun test - see TORQUE docs, Section 5.1");
|
||||
|
||||
# Active Schedulers
|
||||
{
|
||||
# Check Maui
|
||||
if (exists $schedulers{MAUI})
|
||||
{
|
||||
$schedulers{MAUI} = `ps aux | grep maui | grep -v grep` || undef;
|
||||
delete $schedulers{MAUI}
|
||||
unless defined $schedulers{MAUI};
|
||||
}
|
||||
|
||||
# Check Moab
|
||||
if (exists $schedulers{MOAB})
|
||||
{
|
||||
$schedulers{MOAB} = `ps aux | grep moab | grep -v grep` || undef;
|
||||
delete $schedulers{MOAB}
|
||||
unless defined $schedulers{MOAB};
|
||||
}
|
||||
|
||||
# Check FIFO
|
||||
if (exists $schedulers{FIFO})
|
||||
{
|
||||
$schedulers{FIFO} = `ps aux | grep pbs_sched | grep -v grep` || undef;
|
||||
delete $schedulers{FIFO}
|
||||
unless defined $schedulers{FIFO};
|
||||
}
|
||||
}
|
||||
ok(scalar (keys %schedulers), 'Active Scheduler') or
|
||||
diag("No recognized scheduler is running - jobs will not start, start Moab, Maui, or pbs_sched and rerun test - see TORQUE docs, Section 5.1");
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 10;
|
||||
|
||||
diag('Testing serial jobs on TORQUE - this may take several minutes');
|
||||
|
||||
# Check Test User
|
||||
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
|
||||
BAIL_OUT('Test user does not exist');
|
||||
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
BAIL_OUT('Invalid test user');
|
||||
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
BAIL_OUT('Invalid test user');
|
||||
my $testuser = $ENV{'TORQUE_TEST_USER'};
|
||||
|
||||
# Clear Queue
|
||||
my $joblist = `qstat` || undef;
|
||||
ok(!defined($joblist), 'Queue Empty') or
|
||||
BAIL_OUT("Queue must be empty to run job submission tests - cancel all jobs and rerun test");
|
||||
|
||||
# Submit Job
|
||||
my $joblength = 10; # seconds
|
||||
my $walltime = 2 * $joblength;
|
||||
my $job = `su $testuser -c 'echo "sleep $joblength" | qsub -l nodes=1,walltime=$walltime'` || undef;
|
||||
ok(defined $job, 'Job Submission') or
|
||||
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
|
||||
ok($job =~ /^\d+\S*\s*$/, 'Job Submission') or
|
||||
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
|
||||
|
||||
# Job In Queue
|
||||
$job =~ s/\D//g;
|
||||
sleep 1;
|
||||
my $qstat = `qstat | grep $job` || undef;
|
||||
ok(defined $qstat, 'Job in Queue') or
|
||||
BAIL_OUT('Submitted job does not appear in the TORQUE queue');
|
||||
|
||||
# Job Running
|
||||
my $running = 0;
|
||||
my $timer = 0;
|
||||
my $waittime = 45;
|
||||
for (1..$waittime) # Wait up to $waittime seconds for job to start
|
||||
{
|
||||
my $line = `qstat | grep $job` || undef;
|
||||
unless (defined $line)
|
||||
{
|
||||
diag("Submitted job has disappeared from the TORQUE queue");
|
||||
last;
|
||||
}
|
||||
if ($line =~ /\sR\s/)
|
||||
{
|
||||
$running = 1;
|
||||
last;
|
||||
}
|
||||
sleep 1;
|
||||
$timer++;
|
||||
}
|
||||
ok($running, 'Job Running') or
|
||||
BAIL_OUT("Submitted job has failed to start within $waittime seconds - check scheduler - see TORQUE docs, Section 5.1");
|
||||
|
||||
# Check Output Files
|
||||
my $remaining = $joblength - $timer + 1;
|
||||
my $outputfile = "STDIN.o$job";
|
||||
my $errorfile = "STDIN.e$job";
|
||||
sleep $remaining if ($remaining > 0);
|
||||
ok(-T $outputfile, 'Output File') or
|
||||
BAIL_OUT('Submitted job output file not copied - check data staging - see TORQUE docs, Section 6');
|
||||
ok(-T $errorfile, 'Error File') or
|
||||
BAIL_OUT('Submitted job error file not copied - check data staging - see TORQUE docs, Section 6');
|
||||
unlink $outputfile if -T $outputfile;
|
||||
unlink $errorfile if -T $errorfile;
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 17;
|
||||
use subs qw(submit_job job_in_queue job_running cleanup_job);
|
||||
our $Joblength = 20; # seconds
|
||||
|
||||
diag('Testing parallel jobs on TORQUE - this may take several minutes');
|
||||
|
||||
# Check Test User
|
||||
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
|
||||
BAIL_OUT('Test user does not exist');
|
||||
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
BAIL_OUT('Invalid test user');
|
||||
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
BAIL_OUT('Invalid test user');
|
||||
my $testuser = $ENV{'TORQUE_TEST_USER'};
|
||||
|
||||
# Determine Number of Procs
|
||||
my %nodes = ();
|
||||
my $node = undef;
|
||||
my $proccount = 0;
|
||||
my $pbsnodes = `pbsnodes` || undef;
|
||||
ok(defined $pbsnodes, 'Node Information from pbsnodes') or
|
||||
BAIL_OUT('Unable to gather node information from pbsnodes');
|
||||
foreach my $line (split /[\r\n]+/, $pbsnodes)
|
||||
{
|
||||
if ($line =~ /^(\S+)/)
|
||||
{
|
||||
$node = $1;
|
||||
$nodes{$node} = 1;
|
||||
next;
|
||||
}
|
||||
$nodes{$node} += $1 - 1
|
||||
if ($line =~ /^\s+np = (\d+)/) and defined $node;
|
||||
}
|
||||
map { $proccount += $_ } (values %nodes);
|
||||
ok($proccount, 'Processor Count') or
|
||||
BAIL_OUT('TORQUE reported 0 processors');
|
||||
|
||||
# Submit Jobs
|
||||
my $jobid;
|
||||
|
||||
$jobid = submit_job(2);
|
||||
job_in_queue($jobid);
|
||||
job_running($jobid);
|
||||
cleanup_job($jobid);
|
||||
|
||||
$jobid = submit_job($proccount);
|
||||
job_in_queue($jobid);
|
||||
sleep 10;
|
||||
job_running($jobid);
|
||||
cleanup_job($jobid);
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub submit_job ($)
|
||||
{
|
||||
my $procs = $_[0] || 2;
|
||||
my $walltime = 2 * $Joblength;
|
||||
my $job = `su $testuser -c 'echo "sleep $Joblength" | qsub -l nodes=$procs,walltime=$walltime'` || undef;
|
||||
ok(defined $job, 'Job Submission') or
|
||||
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
|
||||
ok($job =~ /^\d+\S*\s*$/, 'Job Submission') or
|
||||
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
|
||||
$job =~ s/\D//g;
|
||||
return $job;
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub job_in_queue ($)
|
||||
{
|
||||
my $job = $_[0];
|
||||
sleep 1;
|
||||
my $qstat = `qstat | grep $job` || undef;
|
||||
ok(defined $qstat, 'Job in Queue') or
|
||||
BAIL_OUT('Submitted job does not appear in the TORQUE queue');
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub job_running ($)
|
||||
{
|
||||
my $job = $_[0];
|
||||
my $running = 0;
|
||||
my $timer = 0;
|
||||
my $waittime = 45;
|
||||
for (1..$waittime) # Wait up to $waittime seconds for job to start
|
||||
{
|
||||
my $line = `qstat | grep $job` || undef;
|
||||
unless (defined $line)
|
||||
{
|
||||
diag("Submitted job has disappeared from the TORQUE queue");
|
||||
last;
|
||||
}
|
||||
if ($line =~ /\sR\s/)
|
||||
{
|
||||
$running = 1;
|
||||
last;
|
||||
}
|
||||
sleep 1;
|
||||
$timer++;
|
||||
}
|
||||
ok($running, 'Job Running') or
|
||||
BAIL_OUT("Submitted job has failed to start within $waittime seconds - check scheduler - see TORQUE docs, Section 5.1");
|
||||
my $remaining = $Joblength - $timer + 1;
|
||||
sleep $remaining if ($remaining > 0);
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub cleanup_job ($)
|
||||
{
|
||||
my $job = $_[0];
|
||||
my $outputfile = "STDIN.o$job";
|
||||
my $errorfile = "STDIN.e$job";
|
||||
ok(-T $outputfile, 'Output File') or
|
||||
BAIL_OUT('Submitted job output file not copied - check data staging - see TORQUE docs, Section 6');
|
||||
ok(-T $errorfile, 'Error File') or
|
||||
BAIL_OUT('Submitted job error file not copied - check data staging - see TORQUE docs, Section 6');
|
||||
unlink $outputfile if -T $outputfile;
|
||||
unlink $errorfile if -T $errorfile;
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More qw(no_plan);
|
||||
use subs qw(check_jobs cancel_jobs qstat_info);
|
||||
our @Jobs = ();
|
||||
our $MAX_CANCEL_TRIES = 5;
|
||||
our $Jobcount = 0;
|
||||
our $Joblength = 1200; # seconds (Default)
|
||||
our $Testuser = undef;
|
||||
|
||||
diag('Testing job arrays on TORQUE - this may take several minutes');
|
||||
|
||||
# Check Test User
|
||||
{
|
||||
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
|
||||
BAIL_OUT('Test user does not exist');
|
||||
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
BAIL_OUT('Invalid test user');
|
||||
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
|
||||
BAIL_OUT('Invalid test user');
|
||||
$Testuser = $ENV{'TORQUE_TEST_USER'};
|
||||
}
|
||||
|
||||
# Determine TORQUE version
|
||||
my $version = 0;
|
||||
{
|
||||
my $qstat = `qstat --about 2>&1`;
|
||||
ok(defined $qstat, 'TORQUE Information from qstat') or
|
||||
BAIL_OUT('Unable to gather TORQUE information from qstat');
|
||||
|
||||
$version = $1 if $qstat =~ /Version:\s+(\d+\.\d+\.\d+)/;
|
||||
}
|
||||
|
||||
SKIP:
|
||||
{
|
||||
|
||||
my $versionMM = ($version =~ /^(\d+\.\d+)\.\d+$/) ? $1 : 0;
|
||||
skip "TORQUE not version 2.2.0 or higher ($version)", 1 if $versionMM < 2.2;
|
||||
|
||||
# Determine Number of Jobs
|
||||
{
|
||||
my %nodes = ();
|
||||
my $node = undef;
|
||||
my $proccount = 0;
|
||||
my $pbsnodes = `pbsnodes` || undef;
|
||||
ok(defined $pbsnodes, 'Node Information from pbsnodes') or
|
||||
BAIL_OUT('Unable to gather node information from pbsnodes');
|
||||
|
||||
foreach my $line (split /[\r\n]+/, $pbsnodes)
|
||||
{
|
||||
if ($line =~ /^(\S+)/)
|
||||
{
|
||||
$node = $1;
|
||||
$nodes{$node} = 1;
|
||||
next;
|
||||
}
|
||||
$nodes{$node} += $1 - 1
|
||||
if ($line =~ /^\s+np = (\d+)/) and defined $node;
|
||||
}
|
||||
|
||||
map { $proccount += $_ } (values %nodes);
|
||||
$Jobcount = 2 * $proccount;
|
||||
ok($proccount, 'Processor Count') or
|
||||
BAIL_OUT('TORQUE reported 0 processors');
|
||||
|
||||
$Joblength = 300 + ($proccount * 30); # seconds
|
||||
}
|
||||
|
||||
# Submit Jobs
|
||||
{
|
||||
my $walltime = 1.1 * $Joblength;
|
||||
my $baseid = `su $Testuser -c 'echo "sleep $Joblength" | qsub -k oe -l nodes=1,walltime=$walltime -t 0-$Jobcount'` || undef;
|
||||
$baseid =~ s/\D//g if defined $baseid;
|
||||
ok(defined $baseid, "Job Submission") or
|
||||
BAIL_OUT("Unable to submit job to TORQUE as '$Testuser' - see TORQUE docs, Section 2.1");
|
||||
ok($baseid =~ /^\d+\S*\s*$/, "Job Submission") or
|
||||
BAIL_OUT("Unable to submit job to TORQUE as '$Testuser' - see TORQUE docs, Section 2.1");
|
||||
@Jobs = map { "$baseid-$_" } (0..$Jobcount);
|
||||
}
|
||||
|
||||
# Jobs In Queue
|
||||
{
|
||||
sleep 5;
|
||||
my %data = qstat_info;
|
||||
for my $i (0..$Jobcount)
|
||||
{
|
||||
ok(exists $data{$Jobs[$i]}, "Job in Queue ($Jobs[$i])") or
|
||||
BAIL_OUT("Submitted job ($Jobs[$i]) does not appear in the TORQUE queue");
|
||||
}
|
||||
}
|
||||
|
||||
# Job Set 1 (first half)
|
||||
{
|
||||
check_jobs 0, int($Jobcount / 2) - 1;
|
||||
cancel_jobs 0, int($Jobcount / 2) - 1;
|
||||
}
|
||||
|
||||
# Job Set 2 (second half)
|
||||
{
|
||||
check_jobs int($Jobcount / 2), $Jobcount - 1;
|
||||
cancel_jobs int($Jobcount / 2), $Jobcount - 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub check_jobs ($$)
|
||||
{
|
||||
my ($first, $last, undef) = @_;
|
||||
my $waittime = int($Joblength * .25) + 1;
|
||||
$waittime = 180 if $waittime < 180;
|
||||
my %complete = ();
|
||||
|
||||
# Wait up to $waittime seconds for job to start
|
||||
ROUND: for my $second (0..($waittime - 1))
|
||||
{
|
||||
sleep 1;
|
||||
my %data = qstat_info;
|
||||
# Check Information
|
||||
my $allcomplete = 1;
|
||||
for my $i ($first..$last)
|
||||
{
|
||||
my $id = $Jobs[$i];
|
||||
$complete{$id} = ((exists $data{$id}) and ($data{$id} =~ /^[RC]$/)) ? 1 : 0;
|
||||
unless ($complete{$id})
|
||||
{
|
||||
$allcomplete = 0;
|
||||
next ROUND;
|
||||
}
|
||||
}
|
||||
last ROUND if $allcomplete;
|
||||
|
||||
}
|
||||
|
||||
for my $i ($first..$last)
|
||||
{
|
||||
ok($complete{$Jobs[$i]}, "Job Running ($Jobs[$i])") or
|
||||
BAIL_OUT("Submitted job ($Jobs[$i]) has failed to start within $waittime seconds - check scheduler - see TORQUE docs, Section 5.1");
|
||||
}
|
||||
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub cancel_jobs ($$)
|
||||
{
|
||||
my ($first, $last, undef) = @_;
|
||||
my $tries = 0;
|
||||
CANCEL: for my $i ($first..$last)
|
||||
{
|
||||
my $qdel = `qdel $Jobs[$i] 2>&1` || undef;
|
||||
$qdel =~ s/[\r\n]//g if defined $qdel;
|
||||
if ((defined $qdel) && ($qdel =~ /MSG=invalid state for job/i))
|
||||
{
|
||||
$qdel = undef if $qdel =~ /MSG=invalid state for job/i;
|
||||
}
|
||||
if ((defined $qdel) and ($tries < $MAX_CANCEL_TRIES))
|
||||
{
|
||||
$tries++;
|
||||
sleep 1;
|
||||
diag("Failed to cancel job ($Jobs[$i]) - Retry $tries/$MAX_CANCEL_TRIES - [$qdel]");
|
||||
redo CANCEL;
|
||||
}
|
||||
ok(!defined $qdel, "Cancel Job ($Jobs[$i])") or
|
||||
BAIL_OUT("Submitted job ($Jobs[$i]) could not be cancelled");
|
||||
$tries = 0;
|
||||
}
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
#
|
||||
sub qstat_info
|
||||
{
|
||||
my %data;
|
||||
# Gather Information
|
||||
my $qstat = `qstat` || undef;
|
||||
ok(defined $qstat, 'qstat Results') or
|
||||
BAIL_OUT('Cannot gather information from qstat');
|
||||
foreach my $line (split /[\r\n]+/, $qstat)
|
||||
{
|
||||
next unless $line =~
|
||||
/^
|
||||
(\d+-\d+)\S*\s+ # Job ID
|
||||
\S+\s+ # Name
|
||||
\S+\s+ # User
|
||||
\S+\s+ # Time Use
|
||||
(\w)\s+ # State
|
||||
\S+\s* # Queue
|
||||
$/x;
|
||||
$data{$1} = $2 if defined($1) and defined($2);
|
||||
}
|
||||
return %data;
|
||||
}
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 1;
|
||||
sub warning ($) { diag "WARNING: $_[0]" };
|
||||
|
||||
##
|
||||
#
|
||||
# This set of tests is for providing warnings to the users. It should not be
|
||||
# used for checking for actual errors. Error checking should be done in an
|
||||
# actual test file. This is the final set of checks that occurs when testing
|
||||
# the TORQUE installation.
|
||||
#
|
||||
##
|
||||
|
||||
# Default Queue & Queue Defaults
|
||||
my $qmgr = `qmgr -c 'p s'`;
|
||||
if (defined($qmgr) and $qmgr)
|
||||
{
|
||||
|
||||
# - Default Queue
|
||||
warning('No default_queue set - see TORQUE docs, Section 4.1.3')
|
||||
unless ($qmgr =~ /^set server default_queue = \S+/mi);
|
||||
|
||||
# - Queue Defaults
|
||||
my $queue = undef;
|
||||
my %queues;
|
||||
# - - Parse Queue Information
|
||||
foreach my $line (split /[\r\n]+/, $qmgr)
|
||||
{
|
||||
$queue = $1 if $line =~ /^create queue (\S+)/;
|
||||
next unless $line =~ /^set queue batch (\S+) = (\S+)/;
|
||||
$queues{$queue}{$1} = $2 if defined $queue;
|
||||
}
|
||||
# - - Check Queue Information
|
||||
foreach $queue (sort keys %queues)
|
||||
{
|
||||
warning("No default nodes for queue '$queue' set - see TORQUE docs, Section 4.1.2")
|
||||
unless exists $queues{$queue}{'resources_default.nodes'};
|
||||
warning("No default walltime for queue '$queue' set - see TORQUE docs, Section 4.1.2.")
|
||||
unless exists $queues{$queue}{'resources_default.walltime'};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ok($qmgr,'Warnings Check') or
|
||||
BAIL_OUT('Cannot communicate with TORQUE through qmgr');
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Test::More tests => 1;
|
||||
|
||||
ok(1,'');
|
||||
|
||||
__END__
|
||||
|
||||
# run multiple jobs
|
||||
|
||||
# verify default queue
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user