--- source-trees/gt2-cvs/gram/jobmanager/setup/pbs/pbs.in 7 Aug 2007 16:23:01 -0000 1.18.6.10 +++ source-trees/gt2-cvs/gram/jobmanager/setup/pbs/pbs.in 30 Jan 2008 21:35:26 -0000 @@ -10,10 +10,11 @@ @ISA = qw(Globus::GRAM::JobManager); -my ($mpirun, $mpiexec, $qsub, $qstat, $qdel, $cluster, $cpu_per_node, $remote_shell); +my ($mpisoftenv, $mpirun, $mpiexec, $qsub, $qstat, $qdel, $cluster, $cpu_per_node, $remote_shell, $soft_msc); BEGIN { + $mpisoftenv = 0; # 0=false, 1=true $mpiexec = '@MPIEXEC@'; $mpirun = '@MPIRUN@'; $qsub = '@QSUB@'; @@ -237,7 +238,18 @@ print JOB '#PBS -o ', $description->stdout(), "\n"; print JOB '#PBS -e ', $description->stderr(), "\n"; - if (defined $description->nodes()) +############################################################################### +# TeraGrid -- prews node type/property and node name based selection support +############################################################################### + my @hosttypes = $description->hosttypes(); + my @hostnames = $description->hostnames(); + my @hostxcount = $description->hostxcount(); + my @xcount = $description->xcount(); + if ( (scalar @hosttypes) || (scalar @hostnames) || (scalar @hostxcount) || (scalar @xcount) ) { + handle_prews_nodes($self, $description); + print JOB '#PBS -l nodes=', $description->nodes(), "\n"; + } + elsif(defined $description->nodes()) { #Generated by ExtensionsHandler.pm from resourceAllocationGroup elements print JOB '#PBS -l nodes=', $description->nodes(), "\n"; @@ -304,6 +316,11 @@ $rsh_env .= $tuple->[0] . '="' . $tuple->[1] . "\";\n" . 'export ' . $tuple->[0] . ";\n"; } +############################################################################### +# TeraGrid -- Do not reset library variables +# 06/30/2005, JP Navarro +############################################################################### + if (0) { foreach (keys %library_vars) { if($library_vars{$_} == 0) @@ -314,6 +331,22 @@ . "export $_;\n"; } } + } +############################################################################### + +############################################################################### +# TeraGrid -- Advanced reservation selection +# 06/30/2005, JP Navarro +############################################################################### + if($description->reservation_id() ne '') + { + print JOB '#PBS -W x=FLAGS:ADVRES:' . $description->reservation_id() . "\n"; + } +############################################################################### + +# print JOB '#PBS -v ' . join(',', @new_env) . "\n"; + +############################################################################### print JOB "$rsh_env"; @@ -405,7 +438,7 @@ local(*CMD); if ( open( CMD, ">$cmd_script_name" ) ) { - print CMD "#!/bin/sh\n"; + print CMD "#!/bin/sh -l\n"; ### SoftEnv extension ### if ( ($softenv_dir ne '') @@ -435,41 +468,39 @@ if ($description->jobtype() eq "mpi") { + my $this_count = ($description->totalprocesses() > 0) ? + $description->totalprocesses() : $description->count(); + my $machinefilearg = ($cluster) ? ' -machinefile $PBS_NODEFILE' : ''; + + if ($mpisoftenv) + { + print JOB 'which mpiexec >/dev/null 2>&1' . "\n"; + print JOB 'if [ $? == 0 ]; then' . "\n"; + print JOB " mpiexec $machinefilearg -n " . $this_count; + print JOB " $cmd_script_name < " . $description->stdin() . "\n"; + print JOB 'else' . "\n"; + print JOB ' which mpirun >/dev/null 2>&1' . "\n"; + print JOB ' if [ $? == 0 ]; then' . "\n"; + print JOB " mpirun -np " . $this_count . $machinefilearg; + print JOB " $cmd_script_name < " . $description->stdin() . "\n"; + print JOB ' else' . "\n"; + } if ($mpiexec ne 'no') { - my $machinefilearg = ""; - if ($cluster) - { - $machinefilearg = ' -machinefile $PBS_NODEFILE'; - } - if ($description->totalprocesses() > 0) - { - print JOB "$mpiexec $machinefilearg -n " - . $description->totalprocesses(); - } - else - { - print JOB "$mpiexec $machinefilearg -n " - . $description->count(); - } + print JOB "$mpiexec $machinefilearg -n " . $this_count; + print JOB " $cmd_script_name < " . $description->stdin() . "\n"; } else { - if ($description->totalprocesses() > 0) - { - print JOB "$mpirun -np " . $description->totalprocesses(); - } - else - { - print JOB "$mpirun -np " . $description->count(); - } - if ($cluster) - { - print JOB ' -machinefile $PBS_NODEFILE'; - } + print JOB "$mpirun -np " . $this_count . $machinefilearg; + print JOB " $cmd_script_name < " . $description->stdin() . "\n"; + } + if ($mpisoftenv) + { + print JOB ' fi' . "\n"; + print JOB 'fi' . "\n"; } - print JOB " $cmd_script_name < " . $description->stdin() . "\n"; } else { @@ -633,6 +664,94 @@ } return Globus::GRAM::Error::JOB_CANCEL_FAILED(); +} + +sub handle_prews_nodes { +############################################################################### +# TeraGrid -- prews node type/property and node name based selection support +# 8/2004, 11/2005, JP Navarro +# +# Support for the following RSL parameters: +# 1) host_types [ host_xcount (nodes of each host_types) ] [ xcount (PPN) ] +# 2) host_names [ xcount (PPN) ] +# 3) [ host_xcount (nodes) ] [ xcount (PPN) ] +# +# TODO: normally count is used to compute total CPU time +############################################################################### + my ($self, $description) = (shift, shift); + my @hosttypes = $description->hosttypes(); + my @hostnames = $description->hostnames(); + my @hostxcount = $description->hostxcount(); + my @xcount = $description->xcount(); + my ($this_hostxcount, $this_xcount, $this); + my $pbsnodes; + my $real_count; # Set to value that will supersede $description->count() + + if(@hosttypes) + { + $self->log("host_types = " . scalar(@hosttypes)); + foreach my $this_hosttype (@hosttypes) + { + $self->log("host_type = " . $this_hosttype); + $this_hostxcount = shift @hostxcount; + $this_xcount = shift @xcount; + $this = (($this_hostxcount) ? "$this_hostxcount:" : '1:') . + $this_hosttype . + (($this_xcount) ? ":ppn=$this_xcount" : ''); + $pbsnodes .= ($pbsnodes) ? '+' : ''; + $pbsnodes .= $this; + $real_count += (($this_hostxcount) ? $this_hostxcount : 1) * (($this_xcount) ? $this_xcount : 1); + } + $real_count ||= 1; # initialize + } + elsif(@hostnames) + { + $self->log("host_names = " . scalar(@hostnames)); + foreach my $this_hostname (@hostnames) + { + $self->log("host_name = " . $this_hostname); + $this_xcount = shift @xcount; + $this = $this_hostname . + (($this_xcount) ? ":ppn=$this_xcount" : ''); + $pbsnodes .= ($pbsnodes) ? '+' : ''; + $pbsnodes .= $this; + $real_count += (($this_xcount) ? $this_xcount : 1); + } + $real_count ||= 1; # initialize + } + elsif(@hostxcount) + { + $self->log("host_xcount = " . scalar(@hostxcount)); + foreach my $this_hostxcount (@hostxcount) + { + $self->log("host_xcount = " . $this_hostxcount); + $this_xcount = shift @xcount; + $this = $this_hostxcount . + (($this_xcount) ? ":ppn=$this_xcount" : ''); + $pbsnodes .= ($pbsnodes) ? '+' : ''; + $pbsnodes .= $this; + $real_count += $this_hostxcount * (($this_xcount) ? $this_xcount : 1); + } + } + elsif(@xcount) + { + $self->log("xcount = " . scalar(@xcount)); + foreach my $this_xcount (@xcount) + { + $self->log("xcount = " . $this_xcount); + $this = "1:ppn=$this_xcount"; + $pbsnodes .= ($pbsnodes) ? '+' : ''; + $pbsnodes .= $this; + $real_count += $this_xcount; + } + } + else + { + $real_count = 1; + $pbsnodes = '1'; + } + $description->add('totalprocesses', $real_count); + $description->add('nodes', $pbsnodes); } 1; --- source-trees/gt2-cvs/gram/jobmanager/setup/pbs/setup-globus-job-manager-pbs.pl 12 Mar 2007 13:59:06 -0000 1.4.8.2 +++ source-trees/gt2-cvs/gram/jobmanager/setup/pbs/setup-globus-job-manager-pbs.pl 30 Jan 2008 21:40:00 -0000 @@ -108,6 +108,26 @@ Values: yes no ValidWhen: GLOBUS_GRAM_JOB_SUBMIT +Attribute: xcount +Description: "Defines the number of processes desired. It is a list that is relative to the RSL paramters host_xcount and host_types, or host_names. Only applies to compute clusters." +ValidWhen: GLOBUS_GRAM_JOB_SUBMIT + +Attribute: host_xcount +Description: "Defines the number of nodes desired. It is a list that is relative to the RSL paramters xcount and host_types. Only applies to compute clusters." +ValidWhen: GLOBUS_GRAM_JOB_SUBMIT + +Attribute: host_types +Description: "Defines the types of nodes desired. It is a list that is relative to the RSL paramters xcount and host_xcount. Only applies to compute clusters." +ValidWhen: GLOBUS_GRAM_JOB_SUBMIT + +Attribute: host_names +Description: "Defines the names of nodes desired. It is a list that is relative to the RSL paramters xcount. Only applies to compute clusters." +ValidWhen: GLOBUS_GRAM_JOB_SUBMIT + +Attribute: reservation_id +Description: "Defines the reservation this job should run under" +ValidWhen: GLOBUS_GRAM_JOB_SUBMIT + EOF if($validate_queues)