mpiexec GMPI_SLAVE env.t problem
Bryan Hellyer
brh at unimelb.edu.au
Fri Aug 15 01:19:13 EDT 2003
I tried "mpi -v -v -v -np1 env", etc. as per below
and the only relevant difference in the shell & mpiexec env.t seems to be
env 65 OLDPWD=/home/brh
env 66 GMPI_ID=0
env 67 GMPI_SLAVE=node040
env 68 MPIEXEC_STDIN_PORT=36757
see below for the outputs (except minor edits "to protect the innocent")
I know I have long paths, etc. but somehow I dont think it's shell env.t
limitations.
Thanx
Bryan
[Lots of output follows again, sorry].
========= cat testenv # qsub script
#!/bin/sh
#PBS -q mpi_gm
##PBS -l nodes=4:ppn=2
##PBS -l walltime=5:00
#PBS -l cput=40:00
#PBS -j oe
#PBS -o testenv_oe.out
echo PBS_QUEUE=$PBS_QUEUE PBS_NODEFILE=$PBS_NODEFILE
cat $PBS_NODEFILE
echo "=====Environment========="
env
echo "=============="
cd /data2/users/brh/mpiexec_tst
echo "mpiexec -v -v -v env"
mpiexec -np 1 -v -v -v env > testenv.out 2>&1
echo "qstat -an"
qstat -an
===========
=========== cat testenv_oe.out
Warning: no access to tty (Bad file descriptor).
Thus no job control in this shell.
PBS_QUEUE=mpi_gm PBS_NODEFILE=/usr/spool/PBS/aux/214.genghis
node040
node039
=====Environment=========
PWD=/home/brh
DLOGS=/var/log/dumplogs
DIST=/usr/local/dist
PBS_JOBNAME=testenv
VENDOR=intel
PBS_ENVIRONMENT=PBS_BATCH
HOSTNAME=node040
LD_LIBRARY_PATH=/lib:/usr/lib:/usr/share/lib:/usr/local/lib:/usr/local/globus/lib:/o
pt/csm/lib:/usr/gm/lib:/usr/local/mpich-gm/lib:/opt/intel/compiler70/ia32/lib
HLOG=/home/brh/data/log/node040
GLOBUS_LOCATION=/usr/local/globus
LOG=/var/log/messages
LESSOPEN=|/usr/bin/lesspipe.sh %s
MANPATH=/usr/man:/usr/share/man:/usr/local/man:/usr/local/share/man:/usr/X11R6/man:/
usr/local/maui/man:/usr/pbs/man:/usr/local/src/mpi/man:/usr/local/src/mpich/man:/opt
/csm/man:/usr/gm/man:/opt/intel/compiler70/man
PBS_O_QUEUE=mpi_gm
PBS_QUEUE=mpi_gm
PBS_MOMPORT=15003
USER=brh
LS_COLORS=
MACHTYPE=i386
PBS_JOBID=214.genghis
PBS_O_LOGNAME=brh
MOMLOG=/usr/spool/PBS/mom_logs/20030814
PBS_O_MAIL=/var/mail/brh
PBS_TASKNUM=1
MOMLOGS=/var/spool/PBS/mom_logs
MAUILOG=/usr/local/maui/log/maui.log
LANG=en_US
PBS_O_HOST=alfred
PBS_JOBCOOKIE=0A1551DC61E29498F5AF226D0E8B3B05
HOST=node040
MYLOG=/home/brh/data/log/mgtnode
PBS_NODENUM=0
PBS=/usr/spool/PBS
LOGNAME=brh
SHLVL=2
PBS_NODEFILE=/usr/spool/PBS/aux/214.genghis
ENVIRONMENT=BATCH
GROUP=users
PBS_O_SHELL=/bin/tcsh
SUPPORTED=en_US.iso885915:en_US:en
SHELL=/bin/tcsh
INTEL_LICENSE_FILE=/opt/intel
CFG=/usr/src/config
HOSTTYPE=i386-linux
PBS_O_HOME=/home/brh
OSTYPE=linux
PBS_O_PATH=/bin:/sbin:/usr/bin:/usr/sbin:/etc:/usr/local/bin:/usr/bin/X11:/home/brh/
bin:.:/usr/local/globus/bin:/usr/local/sbin:/usr/local/maui/bin:/usr/pbs/bin:/usr/pb
s/sbin:/usr/local/mpich-gm/bin:/opt/csm/bin:/usr/gm/bin:/opt/intel/compiler70/ia32/b
in
HOME=/home/brh
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/etc:/usr/local/bin:/usr/bin/X11:/home/brh/bin:.:
/usr/local/globus/bin:/usr/local/sbin:/usr/local/maui/bin:/usr/pbs/bin:/usr/pbs/sbin
:/usr/local/mpich-gm/bin:/opt/csm/bin:/usr/gm/bin:/opt/intel/compiler70/ia32/bin
PBSLOG=/usr/spool/PBS/server_logs/20030814
PBS_O_WORKDIR=/data2/users/brh/mpiexec_tst
_=/usr/bin/env
==============
mpiexec -v -v -v env
PWD=/data2/users/brh/mpiexec_tst
DLOGS=/var/log/dumplogs
DIST=/usr/local/dist
GMPI_MAGIC=214
PBS_JOBNAME=testenv
VENDOR=intel
PBS_ENVIRONMENT=PBS_BATCH
HOSTNAME=node040
LD_LIBRARY_PATH=/lib:/usr/lib:/usr/share/lib:/usr/local/lib:/usr/local/globus/lib:/o
pt/csm/lib:/usr/gm/lib:/usr/local/mpich-gm/lib
HLOG=/home/brh/data/log/node040
GLOBUS_LOCATION=/usr/local/globus
GMPI_PORT1=36755
LOG=/var/log/messages
GMPI_PORT2=36756
MPIEXEC_STDOUT_PORT=36758
LESSOPEN=|/usr/bin/lesspipe.sh %s
MANPATH=/usr/man:/usr/share/man:/usr/local/man:/usr/local/share/man:/usr/X11R6/man:/
usr/local/maui/man:/usr/pbs/man:/usr/local/src/mpi/man:/usr/local/src/mpich/man:/opt
/csm/man:/usr/gm/man
PBS_O_QUEUE=mpi_gm
PBS_QUEUE=mpi_gm
PBS_MOMPORT=15003
GMPI_NP=1
MPIEXEC_STDERR_PORT=36759
USER=brh
LS_COLORS=
MACHTYPE=i386
PBS_JOBID=214.genghis
PBS_O_LOGNAME=brh
MOMLOG=/usr/spool/PBS/mom_logs/20030814
PBS_O_MAIL=/var/mail/brh
OLDPWD=/home/brh
PBS_TASKNUM=1
MOMLOGS=/var/spool/PBS/mom_logs
MAUILOG=/usr/local/maui/log/maui.log
LANG=en_US
PBS_O_HOST=alfred
PBS_JOBCOOKIE=0A1551DC61E29498F5AF226D0E8B3B05
HOST=node040
MYLOG=/home/brh/data/log/mgtnode
GMPI_MASTER=node040
PBS_NODENUM=0
GMPI_BOARD=-1
PBS=/usr/spool/PBS
LOGNAME=brh
SHLVL=2
ENVIRONMENT=BATCH
PBS_O_SHELL=/bin/tcsh
GMPI_PORT=36755
PBS_O_HOME=/home/brh
PBS_O_PATH=/bin:/sbin:/usr/bin:/usr/sbin:/etc:/usr/local/bin:/usr/bin/X11:/home/brh/
bin:.:/usr/local/globus/bin:/usr/local/sbin:/usr/local/maui/bin:/usr/pbs/bin:/usr/pb
s/sbin:/usr/local/mpich-gm/bin:/opt/csm/bin:/usr/gm/bin:/opt/intel/compiler70/ia32/b
in
HOME=/home/brh
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/etc:/usr/local/bin:/usr/bin/X11:/home/brh/bin:.:
/usr/local/globus/bin:/usr/local/sbin:/usr/local/maui/bin:/usr/pbs/bin:/usr/pbs/sbin
:/usr/local/mpich-gm/bin:/opt/csm/bin:/usr/gm/bin
PBS_O_WORKDIR=/data2/users/brh/mpiexec_tst
HOSTTYPE=i386-linux
OSTYPE=linux
GROUP=users
SUPPORTED=en_US.iso885915:en_US:en
CFG=/usr/src/config
PBSLOG=/usr/spool/PBS/server_logs/20030814
qstat -an
genghis:
Req'd Req'd
Elap
Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time
--------------- -------- -------- ---------- ------ --- --- ------ ----- -
-----
214.genghis brh mpi_gm testenv 30675 2 -- -- 00:40 R --
node040/0+node039/0
===========
=========== cat testenv.out
stat_exe: testing "/bin/env"
stat_exe: testing "/sbin/env"
stat_exe: testing "/usr/bin/env"
resolve_exe: found exe "env" in path
node 0: name = node040, mpname = node040, cpu = 0
stdio_fork: might listen on one of abort_fd_array (3,4)
stdio_fork: built listener 0 in fd 5 on port 36757
stdio_fork: built listener 1 in fd 6 on port 36758
stdio_fork: built listener 2 in fd 7 on port 36759
command to 0/1
argv 0 /bin/sh
argv 1 -c
argv 2 if test -d "/data2/users/brh/mpiexec_tst"; then cd
"/data2/users/brh/mpiexec
_tst"; fi; exec /bin/tcsh -c 'exec env'
environment to 0/1
env 0 GMPI_MAGIC=214
env 1 GMPI_MASTER=node040
env 2 GMPI_PORT=36755
env 3 GMPI_PORT1=36755
env 4 GMPI_PORT2=36756
env 5 GMPI_NP=1
env 6 GMPI_BOARD=-1
env 7 MPIEXEC_STDOUT_PORT=36758
env 8 MPIEXEC_STDERR_PORT=36759
env 9 PWD=/data2/users/brh/mpiexec_tst
env 10 DLOGS=/var/log/dumplogs
env 11 DIST=/usr/local/dist
env 12 PBS_JOBNAME=testenv
env 13 VENDOR=intel
env 14 PBS_ENVIRONMENT=PBS_BATCH
env 15 HOSTNAME=node040
env 16
LD_LIBRARY_PATH=/lib:/usr/lib:/usr/share/lib:/usr/local/lib:/usr/local/globus
/lib:/opt/csm/lib:/usr/gm/lib:/usr/local/mpich-gm/lib:/opt/intel/compiler70/ia32/lib
env 17 HLOG=/home/brh/data/log/node040
env 18 GLOBUS_LOCATION=/usr/local/globus
env 19 LOG=/var/log/messages
env 20 LOGD=/usr/var/adm/syslog.dated
env 21 LESSOPEN=|/usr/bin/lesspipe.sh %s
env 22
MANPATH=/usr/man:/usr/share/man:/usr/local/man:/usr/local/share/man:/usr/X11R
6/man:/usr/local/maui/man:/usr/pbs/man:/usr/local/src/mpi/man:/usr/local/src/mpich/m
an:/opt/csm/man:/usr/gm/man:/opt/intel/compiler70/man
env 23 PBS_O_QUEUE=mpi_gm
env 24 PBS_QUEUE=mpi_gm
env 25 PBS_MOMPORT=15003
env 26 USER=brh
env 27 LS_COLORS=
env 28 MACHTYPE=i386
env 29 [removed -bh]
env 30 PBS_JOBID=214.genghis
env 31 PBS_O_LOGNAME=brh
env 32 MOMLOG=/usr/spool/PBS/mom_logs/20030814
env 33 PBS_O_MAIL=/var/mail/brh
env 34 PBS_TASKNUM=1
env 35 MOMLOGS=/var/spool/PBS/mom_logs
env 36 MAUILOG=/usr/local/maui/log/maui.log
env 37 LANG=en_US
env 38 PBS_O_HOST=alfred
env 39 PBS_JOBCOOKIE=0A1551DC61E29498F5AF226D0E8B3B05
env 40 HOST=node040
env 41 MYLOG=/home/brh/data/log/mgtnode
env 42 [removed -bh]
env 43 PBS_NODENUM=0
env 44 PBS=/usr/spool/PBS
env 45 LOGNAME=brh
env 46 SHLVL=2
env 47 PBS_NODEFILE=/usr/spool/PBS/aux/214.genghis
env 48 ENVIRONMENT=BATCH
env 49 GROUP=users
env 50 PBS_O_SHELL=/bin/tcsh
env 51 [removed -bh]
env 52 SUPPORTED=en_US.iso885915:en_US:en
env 53 SHELL=/bin/tcsh
env 54 INTEL_LICENSE_FILE=/opt/intel
env 55 CFG=/usr/src/config
env 56 HOSTTYPE=i386-linux
env 57 PBS_O_HOME=/home/brh
env 58 OSTYPE=linux
env 59
PBS_O_PATH=/bin:/sbin:/usr/bin:/usr/sbin:/etc:/usr/local/bin:/usr/bin/X11:/ho
me/brh/bin:.:/usr/local/globus/bin:/usr/local/sbin:/usr/local/maui/bin:/usr/pbs/bin:
/usr/pbs/sbin:/usr/local/mpich-gm/bin:/opt/csm/bin:/usr/gm/bin:/opt/intel/compiler70
/ia32/bin
env 60 HOME=/home/brh
env 61
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/etc:/usr/local/bin:/usr/bin/X11:/home/brh
/bin:.:/usr/local/globus/bin:/usr/local/sbin:/usr/local/maui/bin:/usr/pbs/bin:/usr/p
bs/sbin:/usr/local/mpich-gm/bin:/opt/csm/bin:/usr/gm/bin:/opt/intel/compiler70/ia32/
bin
env 62 PBSLOG=/usr/spool/PBS/server_logs/20030814
env 63 PBS_O_WORKDIR=/data2/users/brh/mpiexec_tst
env 64 _=/usr/local/bin/mpiexec
env 65 OLDPWD=/home/brh
env 66 GMPI_ID=0
env 67 GMPI_SLAVE=node040
env 68 MPIEXEC_STDIN_PORT=36757
wait_one_task_start: evt = 2, task 0 host node040
All 1 task started.
read_gm_startup_ports: waiting for info
read_gm_startup_ports: obit check returns 0
read_gm_startup_ports: obit check returns 3
wait_tasks: numspawned = 1, got evt 3 for tid 2 host node040 status 0
kill_stdio: sent SIGTERM, waiting on 30703
goodbye_from_parent: got signal 15, exiting now
===========
Bryan Hellyer
HPC Systems Programmer
ITS Systems & Infrastructure
University of Melbourne
More information about the mpiexec
mailing list