This commit is contained in:
ruoyunbai
2021-09-29 21:06:16 +08:00
parent b200bb2f20
commit 2bb9621e30
2960 changed files with 642536 additions and 1 deletions
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 4;
ok($> == 0, "Running as 'root'") or
BAIL_OUT("Rerun tests as 'root'");
my $message = "A username must be supplied for testing job submission.\n"
. "Do one of the following:\n"
. "- Set the username using the -u commandline parameter\n"
. " torque.test -u <USERNAME>\n"
. "- Set the \$TORQUE_TEST_USER environment variable.\n";
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
do
{
diag($message);
BAIL_OUT('Set test username');
};
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
do
{
diag($message);
BAIL_OUT('Set test username');
};
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
do
{
diag($message);
BAIL_OUT('Set test username');
};
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 7;
# pbs_server - TORQUE Install
my $pbs_server = `which pbs_server` || undef;
ok(defined $pbs_server, 'TORQUE Install') or
BAIL_OUT('Cannot locate TORQUE server daemon - install TORQUE or add TORQUE install directory to $PATH');
# qstat - Client Tools
my $qstat = `which qstat` || undef;
ok(defined $qstat, 'Client Tools') or
BAIL_OUT('Cannot locate TORQUE clients, install TORQUE or add TORQUE install directory to $PATH');
# version - TORQUE version
my $version = `qstat --version 2>&1`;
$version =~ /^version:\s*([\d.]+)\s*$/i;
$version = $1 || undef;
ok(defined $version, 'TORQUE Version') or
BAIL_OUT('Cannot determine TORQUE version');
# sbin/ - Install Location
my $sbindir = $pbs_server;
$sbindir =~ s/pbs_server\s*$//;
ok($sbindir ne '', 'Determine sbin') or
BAIL_OUT('Cannot determine SBIN directory');
ok(-d $sbindir,'sbin Exists') or
BAIL_OUT("$sbindir does not seem to exist");
# PID - TORQUE Currently Running
my $pid = `ps -ef | grep pbs_server | grep -v grep`;
$pid =~ /^\S+\s+(\d+)\s+/;
$pid = $1 || undef;
ok(defined $pid, 'TORQUE Currently Running') or
BAIL_OUT('pbs_server not running, start pbs_server and rerun test - see TORQUE docs, Section 1.2.4');
# qmgr - TORQUE Database Created
my $qmgr = `qmgr -c 'p s'` || undef;
ok(defined $qmgr, 'Database Created') or
BAIL_OUT("TORQUE database not created, restart pbs_server using 'pbs_server -t create' - see TORQUE docs, Section 1.2.1");
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 8;
# pbs_server --about - Directory Information
my @pbs_server = `pbs_server --about`;
ok(scalar @pbs_server, 'Directory Information') or
BAIL_OUT('Cannot get directory infromation from pbs_server');
# TORQUE Home Directory
my $torquehome = undef;
foreach my $line (@pbs_server)
{
if ($line =~ /^serverhome:\s+(.+)\s*$/i)
{
$torquehome = $1;
last;
}
}
ok(defined $torquehome, 'TORQUE Home Directory') or
BAIL_OUT('Cannot determine server home directory from pbs_server');
ok(-d $torquehome, 'TORQUE Home Directory Exists') or
BAIL_OUT("The server home directory ($torquehome) does not exist");
# pbsnodes - Node List
my $nodesfile = "$torquehome/server_priv/nodes";
$nodesfile =~ s#//#/#g;
ok(-T $nodesfile, 'Nodes File Exists') or
BAIL_OUT("TORQUE 'nodes' file not created, setup $nodesfile and restart pbs_server - see TORQUE docs, Section 1.2.2");
my @pbsnodes = map { s/\s+$//g; $_ }
grep { /^\S/ }
`pbsnodes -a`;
ok(scalar @pbsnodes, 'pbsnodes Reports Nodes') or
BAIL_OUT("TORQUE 'nodes' file not created, setup $nodesfile and restart pbs_server - see TORQUE docs, Section 1.2.2");
ok($pbsnodes[0] ne 'no nodes', 'pbsnodes Lists Nodes') or
BAIL_OUT("TORQUE 'nodes' file not created, setup $nodesfile and restart pbs_server - see TORQUE docs, Section 1.2.2");
# Node Count
my $nodecount = scalar @pbsnodes;
ok($nodecount, 'Node Count') or
BAIL_OUT('TORQUE reporting zero nodes available');
# Active Nodes
my @upnodes = grep { /down|unknown/i }
grep { /^\s+state\s+=\s+/i }
`pbsnodes -a`;
ok(!scalar(@upnodes), 'Nodes Have Connected to Server') or
do {
diag
(
"Compute nodes have not contacted pbs_server, verify the following:\n"
. "- pbs_mom daemons are running on hosts\n"
. "- compute hostnames match names listed in $torquehome/server_priv/nodes\n"
. "- compute hosts are network-accessible from the head node\n"
. "- the command 'momctl -d3 -h $pbsnodes[0]' does not report any errors\n"
);
BAIL_OUT('Compute nodes have not contacted pbs_server');
}
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More qw(no_plan);
# Queue List
my (@xqueues, %queues);
# - Build first-level entries for queue hash table
map
{
/^create queue (\w+)/i;
$queues{$1} = ();
}
grep
{
/^create queue \w+/i
}
`qmgr -c 'p s'`;
# - Populate additional entries for each queue in the hash
foreach my $queue (keys %queues)
{
my $data = `qmgr -c 'p queue $queue'`;
push(@xqueues, $queue)
if $data =~ /^set queue $queue queue_type\s+=\s+Execution/im;
foreach my $line (split /[\r\n]+/, $data)
{
next unless $line =~ /^set queue $queue (\S+) = (\S+)/;
$queues{$queue}{$1} = $2;
}
}
# - Verify that queues exist
ok(scalar (keys %queues), 'Configured Queues') or
BAIL_OUT('No TORQUE queues are configured - run torque.setup or see TORQUE docs, Sections 1.2.1 & 4.1');
ok(scalar @xqueues, 'Execution Queues') or
BAIL_OUT('No execution queues are configured - run torque.setup or see TORQUE docs, Sections 1.2.1 & 4.1');
# Queue Check
my $queuesOK = 1;
foreach my $queue (keys %queues)
{
ok($queues{$queue}{'enabled'} =~ /^True$/i, "Queue '$queue' Enabled") or
do
{
diag("TORQUE queue '$queue' has not been enabled - see TORQUE docs, Section 4.1");
$queuesOK = 0;
};
ok($queues{$queue}{'started'} =~ /^True$/i, "Queue '$queue' Started") or
do
{
diag("TORQUE queue '$queue' has not been started - see TORQUE docs, Section 4.1");
$queuesOK = 0;
};
}
ok($queuesOK, 'TORQUE Queues Configured') or
BAIL_OUT("TORQUE queues have not been correctly configured - run torque.setup or see section 4.1");
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 2;
use constant MAUI => 'maui';
use constant MOAB => 'moab';
use constant FIFO => 'fifo';
my %schedulers;
# Installed Schedulers
{
# Check Maui
$schedulers{MAUI} = `which maui 2>/dev/null` || undef;
delete $schedulers{MAUI}
unless defined $schedulers{MAUI};
# Check Moab
$schedulers{MOAB} = `which moab 2>/dev/null` || undef;
delete $schedulers{MOAB}
unless defined $schedulers{MOAB};
# Check FIFO
$schedulers{FIFO} = `which pbs_sched 2>/dev/null` || undef;
delete $schedulers{FIFO}
unless defined $schedulers{FIFO};
}
ok(scalar (keys %schedulers), 'Installed Scheduler') or
diag("No recognized scheduler is installed - install Moab, Maui, or pbs_sched and rerun test - see TORQUE docs, Section 5.1");
# Active Schedulers
{
# Check Maui
if (exists $schedulers{MAUI})
{
$schedulers{MAUI} = `ps aux | grep maui | grep -v grep` || undef;
delete $schedulers{MAUI}
unless defined $schedulers{MAUI};
}
# Check Moab
if (exists $schedulers{MOAB})
{
$schedulers{MOAB} = `ps aux | grep moab | grep -v grep` || undef;
delete $schedulers{MOAB}
unless defined $schedulers{MOAB};
}
# Check FIFO
if (exists $schedulers{FIFO})
{
$schedulers{FIFO} = `ps aux | grep pbs_sched | grep -v grep` || undef;
delete $schedulers{FIFO}
unless defined $schedulers{FIFO};
}
}
ok(scalar (keys %schedulers), 'Active Scheduler') or
diag("No recognized scheduler is running - jobs will not start, start Moab, Maui, or pbs_sched and rerun test - see TORQUE docs, Section 5.1");
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 10;
diag('Testing serial jobs on TORQUE - this may take several minutes');
# Check Test User
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
BAIL_OUT('Test user does not exist');
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
BAIL_OUT('Invalid test user');
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
BAIL_OUT('Invalid test user');
my $testuser = $ENV{'TORQUE_TEST_USER'};
# Clear Queue
my $joblist = `qstat` || undef;
ok(!defined($joblist), 'Queue Empty') or
BAIL_OUT("Queue must be empty to run job submission tests - cancel all jobs and rerun test");
# Submit Job
my $joblength = 10; # seconds
my $walltime = 2 * $joblength;
my $job = `su $testuser -c 'echo "sleep $joblength" | qsub -l nodes=1,walltime=$walltime'` || undef;
ok(defined $job, 'Job Submission') or
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
ok($job =~ /^\d+\S*\s*$/, 'Job Submission') or
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
# Job In Queue
$job =~ s/\D//g;
sleep 1;
my $qstat = `qstat | grep $job` || undef;
ok(defined $qstat, 'Job in Queue') or
BAIL_OUT('Submitted job does not appear in the TORQUE queue');
# Job Running
my $running = 0;
my $timer = 0;
my $waittime = 45;
for (1..$waittime) # Wait up to $waittime seconds for job to start
{
my $line = `qstat | grep $job` || undef;
unless (defined $line)
{
diag("Submitted job has disappeared from the TORQUE queue");
last;
}
if ($line =~ /\sR\s/)
{
$running = 1;
last;
}
sleep 1;
$timer++;
}
ok($running, 'Job Running') or
BAIL_OUT("Submitted job has failed to start within $waittime seconds - check scheduler - see TORQUE docs, Section 5.1");
# Check Output Files
my $remaining = $joblength - $timer + 1;
my $outputfile = "STDIN.o$job";
my $errorfile = "STDIN.e$job";
sleep $remaining if ($remaining > 0);
ok(-T $outputfile, 'Output File') or
BAIL_OUT('Submitted job output file not copied - check data staging - see TORQUE docs, Section 6');
ok(-T $errorfile, 'Error File') or
BAIL_OUT('Submitted job error file not copied - check data staging - see TORQUE docs, Section 6');
unlink $outputfile if -T $outputfile;
unlink $errorfile if -T $errorfile;
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 17;
use subs qw(submit_job job_in_queue job_running cleanup_job);
our $Joblength = 20; # seconds
diag('Testing parallel jobs on TORQUE - this may take several minutes');
# Check Test User
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
BAIL_OUT('Test user does not exist');
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
BAIL_OUT('Invalid test user');
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
BAIL_OUT('Invalid test user');
my $testuser = $ENV{'TORQUE_TEST_USER'};
# Determine Number of Procs
my %nodes = ();
my $node = undef;
my $proccount = 0;
my $pbsnodes = `pbsnodes` || undef;
ok(defined $pbsnodes, 'Node Information from pbsnodes') or
BAIL_OUT('Unable to gather node information from pbsnodes');
foreach my $line (split /[\r\n]+/, $pbsnodes)
{
if ($line =~ /^(\S+)/)
{
$node = $1;
$nodes{$node} = 1;
next;
}
$nodes{$node} += $1 - 1
if ($line =~ /^\s+np = (\d+)/) and defined $node;
}
map { $proccount += $_ } (values %nodes);
ok($proccount, 'Processor Count') or
BAIL_OUT('TORQUE reported 0 processors');
# Submit Jobs
my $jobid;
$jobid = submit_job(2);
job_in_queue($jobid);
job_running($jobid);
cleanup_job($jobid);
$jobid = submit_job($proccount);
job_in_queue($jobid);
sleep 10;
job_running($jobid);
cleanup_job($jobid);
#------------------------------------------------------------------------------
#
sub submit_job ($)
{
my $procs = $_[0] || 2;
my $walltime = 2 * $Joblength;
my $job = `su $testuser -c 'echo "sleep $Joblength" | qsub -l nodes=$procs,walltime=$walltime'` || undef;
ok(defined $job, 'Job Submission') or
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
ok($job =~ /^\d+\S*\s*$/, 'Job Submission') or
BAIL_OUT("Unable to submit job to TORQUE as '$testuser' - see TORQUE docs, Section 2.1");
$job =~ s/\D//g;
return $job;
}
#
#------------------------------------------------------------------------------
#
sub job_in_queue ($)
{
my $job = $_[0];
sleep 1;
my $qstat = `qstat | grep $job` || undef;
ok(defined $qstat, 'Job in Queue') or
BAIL_OUT('Submitted job does not appear in the TORQUE queue');
}
#
#------------------------------------------------------------------------------
#
sub job_running ($)
{
my $job = $_[0];
my $running = 0;
my $timer = 0;
my $waittime = 45;
for (1..$waittime) # Wait up to $waittime seconds for job to start
{
my $line = `qstat | grep $job` || undef;
unless (defined $line)
{
diag("Submitted job has disappeared from the TORQUE queue");
last;
}
if ($line =~ /\sR\s/)
{
$running = 1;
last;
}
sleep 1;
$timer++;
}
ok($running, 'Job Running') or
BAIL_OUT("Submitted job has failed to start within $waittime seconds - check scheduler - see TORQUE docs, Section 5.1");
my $remaining = $Joblength - $timer + 1;
sleep $remaining if ($remaining > 0);
}
#
#------------------------------------------------------------------------------
#
sub cleanup_job ($)
{
my $job = $_[0];
my $outputfile = "STDIN.o$job";
my $errorfile = "STDIN.e$job";
ok(-T $outputfile, 'Output File') or
BAIL_OUT('Submitted job output file not copied - check data staging - see TORQUE docs, Section 6');
ok(-T $errorfile, 'Error File') or
BAIL_OUT('Submitted job error file not copied - check data staging - see TORQUE docs, Section 6');
unlink $outputfile if -T $outputfile;
unlink $errorfile if -T $errorfile;
}
#
#------------------------------------------------------------------------------
+200
View File
@@ -0,0 +1,200 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More qw(no_plan);
use subs qw(check_jobs cancel_jobs qstat_info);
our @Jobs = ();
our $MAX_CANCEL_TRIES = 5;
our $Jobcount = 0;
our $Joblength = 1200; # seconds (Default)
our $Testuser = undef;
diag('Testing job arrays on TORQUE - this may take several minutes');
# Check Test User
{
ok(exists $ENV{'TORQUE_TEST_USER'}, 'Test User Exists') or
BAIL_OUT('Test user does not exist');
ok(defined $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
BAIL_OUT('Invalid test user');
ok(length $ENV{'TORQUE_TEST_USER'}, 'Valid Test User') or
BAIL_OUT('Invalid test user');
$Testuser = $ENV{'TORQUE_TEST_USER'};
}
# Determine TORQUE version
my $version = 0;
{
my $qstat = `qstat --about 2>&1`;
ok(defined $qstat, 'TORQUE Information from qstat') or
BAIL_OUT('Unable to gather TORQUE information from qstat');
$version = $1 if $qstat =~ /Version:\s+(\d+\.\d+\.\d+)/;
}
SKIP:
{
my $versionMM = ($version =~ /^(\d+\.\d+)\.\d+$/) ? $1 : 0;
skip "TORQUE not version 2.2.0 or higher ($version)", 1 if $versionMM < 2.2;
# Determine Number of Jobs
{
my %nodes = ();
my $node = undef;
my $proccount = 0;
my $pbsnodes = `pbsnodes` || undef;
ok(defined $pbsnodes, 'Node Information from pbsnodes') or
BAIL_OUT('Unable to gather node information from pbsnodes');
foreach my $line (split /[\r\n]+/, $pbsnodes)
{
if ($line =~ /^(\S+)/)
{
$node = $1;
$nodes{$node} = 1;
next;
}
$nodes{$node} += $1 - 1
if ($line =~ /^\s+np = (\d+)/) and defined $node;
}
map { $proccount += $_ } (values %nodes);
$Jobcount = 2 * $proccount;
ok($proccount, 'Processor Count') or
BAIL_OUT('TORQUE reported 0 processors');
$Joblength = 300 + ($proccount * 30); # seconds
}
# Submit Jobs
{
my $walltime = 1.1 * $Joblength;
my $baseid = `su $Testuser -c 'echo "sleep $Joblength" | qsub -k oe -l nodes=1,walltime=$walltime -t 0-$Jobcount'` || undef;
$baseid =~ s/\D//g if defined $baseid;
ok(defined $baseid, "Job Submission") or
BAIL_OUT("Unable to submit job to TORQUE as '$Testuser' - see TORQUE docs, Section 2.1");
ok($baseid =~ /^\d+\S*\s*$/, "Job Submission") or
BAIL_OUT("Unable to submit job to TORQUE as '$Testuser' - see TORQUE docs, Section 2.1");
@Jobs = map { "$baseid-$_" } (0..$Jobcount);
}
# Jobs In Queue
{
sleep 5;
my %data = qstat_info;
for my $i (0..$Jobcount)
{
ok(exists $data{$Jobs[$i]}, "Job in Queue ($Jobs[$i])") or
BAIL_OUT("Submitted job ($Jobs[$i]) does not appear in the TORQUE queue");
}
}
# Job Set 1 (first half)
{
check_jobs 0, int($Jobcount / 2) - 1;
cancel_jobs 0, int($Jobcount / 2) - 1;
}
# Job Set 2 (second half)
{
check_jobs int($Jobcount / 2), $Jobcount - 1;
cancel_jobs int($Jobcount / 2), $Jobcount - 1;
}
}
#------------------------------------------------------------------------------
#
sub check_jobs ($$)
{
my ($first, $last, undef) = @_;
my $waittime = int($Joblength * .25) + 1;
$waittime = 180 if $waittime < 180;
my %complete = ();
# Wait up to $waittime seconds for job to start
ROUND: for my $second (0..($waittime - 1))
{
sleep 1;
my %data = qstat_info;
# Check Information
my $allcomplete = 1;
for my $i ($first..$last)
{
my $id = $Jobs[$i];
$complete{$id} = ((exists $data{$id}) and ($data{$id} =~ /^[RC]$/)) ? 1 : 0;
unless ($complete{$id})
{
$allcomplete = 0;
next ROUND;
}
}
last ROUND if $allcomplete;
}
for my $i ($first..$last)
{
ok($complete{$Jobs[$i]}, "Job Running ($Jobs[$i])") or
BAIL_OUT("Submitted job ($Jobs[$i]) has failed to start within $waittime seconds - check scheduler - see TORQUE docs, Section 5.1");
}
}
#
#------------------------------------------------------------------------------
#
sub cancel_jobs ($$)
{
my ($first, $last, undef) = @_;
my $tries = 0;
CANCEL: for my $i ($first..$last)
{
my $qdel = `qdel $Jobs[$i] 2>&1` || undef;
$qdel =~ s/[\r\n]//g if defined $qdel;
if ((defined $qdel) && ($qdel =~ /MSG=invalid state for job/i))
{
$qdel = undef if $qdel =~ /MSG=invalid state for job/i;
}
if ((defined $qdel) and ($tries < $MAX_CANCEL_TRIES))
{
$tries++;
sleep 1;
diag("Failed to cancel job ($Jobs[$i]) - Retry $tries/$MAX_CANCEL_TRIES - [$qdel]");
redo CANCEL;
}
ok(!defined $qdel, "Cancel Job ($Jobs[$i])") or
BAIL_OUT("Submitted job ($Jobs[$i]) could not be cancelled");
$tries = 0;
}
}
#
#------------------------------------------------------------------------------
#
sub qstat_info
{
my %data;
# Gather Information
my $qstat = `qstat` || undef;
ok(defined $qstat, 'qstat Results') or
BAIL_OUT('Cannot gather information from qstat');
foreach my $line (split /[\r\n]+/, $qstat)
{
next unless $line =~
/^
(\d+-\d+)\S*\s+ # Job ID
\S+\s+ # Name
\S+\s+ # User
\S+\s+ # Time Use
(\w)\s+ # State
\S+\s* # Queue
$/x;
$data{$1} = $2 if defined($1) and defined($2);
}
return %data;
}
#
#------------------------------------------------------------------------------
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 1;
sub warning ($) { diag "WARNING: $_[0]" };
##
#
# This set of tests is for providing warnings to the users. It should not be
# used for checking for actual errors. Error checking should be done in an
# actual test file. This is the final set of checks that occurs when testing
# the TORQUE installation.
#
##
# Default Queue & Queue Defaults
my $qmgr = `qmgr -c 'p s'`;
if (defined($qmgr) and $qmgr)
{
# - Default Queue
warning('No default_queue set - see TORQUE docs, Section 4.1.3')
unless ($qmgr =~ /^set server default_queue = \S+/mi);
# - Queue Defaults
my $queue = undef;
my %queues;
# - - Parse Queue Information
foreach my $line (split /[\r\n]+/, $qmgr)
{
$queue = $1 if $line =~ /^create queue (\S+)/;
next unless $line =~ /^set queue batch (\S+) = (\S+)/;
$queues{$queue}{$1} = $2 if defined $queue;
}
# - - Check Queue Information
foreach $queue (sort keys %queues)
{
warning("No default nodes for queue '$queue' set - see TORQUE docs, Section 4.1.2")
unless exists $queues{$queue}{'resources_default.nodes'};
warning("No default walltime for queue '$queue' set - see TORQUE docs, Section 4.1.2.")
unless exists $queues{$queue}{'resources_default.walltime'};
}
}
ok($qmgr,'Warnings Check') or
BAIL_OUT('Cannot communicate with TORQUE through qmgr');
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/perl -w
use strict;
use warnings;
use Test::More tests => 1;
ok(1,'');
__END__
# run multiple jobs
# verify default queue