#!/bin/csh
#
#   9 May 07 - MWS 
#
#  This is a C-shell script to execute GAMESS, by typing
#       rungms JOB VERNO NCPUS >& JOB.log &
#  JOB    is the name of the 'JOB.inp' file to be executed,
#  VERNO  is the number of the executable you chose at 'lked' time,
#  NCPUS  is the number of processors to be used.
#
#  Unfortunately execution is harder to standardize than compiling,
#  so you have to do a bit more than name your machine type here:
#    a) choose the target for execution from: 
#
#       Select "sockets", or perhaps one of the following special cases:
#           compaq-sc, cray-t3e, cray-x1, cray-xt3, ibm64-sp, sgi64
#
#       Select "sockets" if your compile time target was any of these:
#           axp64, cray-pvp, cray-xd1, fuji-pp, hpux32, hpux64,
#           ibm32, ibm64, linux32, linux64, linux-ia64, mac32, macG5,
#           necsx, sgi32, sun32, sun64
#       as all of these systems use TCP/IP sockets.  Do not name your
#       machine, instead, type "sockets".
#
#       Select "mpi" only if you have deliberately chosen to do the hard
#       work to use MPI for some very special reason.  Your hard work
#       is not yet over, as the mpirun below is unlikely to be correct
#       for your system.
#
#           Search on the words typed in capital letters just below
#           in order to find the right place to select each one:
#    b) choose a directory SCR where large temporary files can reside.
#       This should be the fastest possible disk access, very spacious,
#       and almost certainly a local disk.
#       Translation: do not put these files on a slow network file system!
#    c) change the the VERNO default to the version number you chose when
#       running "lked" as the VERNO default, and maybe NCPUS' default.
#    d) perhaps change the location of the PUNCH and IRCDATA files below,
#       these along with the standard output should probably be routed to
#       a central file server (network file systems is quite OK for these)
#    e) make sure that the ERICFMT file name and MCPPATH pathname point to
#       your file server's GAMESS tree, so that all runs can find them.
#       Again, a network file system is quite OK for these two.
#    f) customize the execution section for your target below,
#       each has its own list of further requirements.
#    g) it is unwise to have every user take a copy of this script, as you
#       can *NEVER* update all the copies later on.  Instead, it is better
#       to ask other users to create an alias to point to a common script,
#       such as this in their C-shell .login file,
#             alias gms '/u1/mike/gamess/rungms'
#    h) it is entirely possible to make 'rungms' run in a batch queue,
#       be it PBS, DQS, et cetera.  This is so installation dependent
#       that we leave it to up to you, although you can find an example
#       of using LoadLeveler via a "llgms" front end in ~/gamess/misc.
#
set TARGET=sockets
set SCR=/scr/$USER
#
set JOB=$1      # name of the input file xxx.inp, give only the xxx part
set VERNO=$2    # revision number of the executable created by 'lked' step
set NCPUS=$3    # number of compute processes to be run

# provide defaults if last two arguments are not given to this script
if (null$VERNO == null) set VERNO=00
if (null$NCPUS == null) set NCPUS=1
#
#  ---- the top third of the script is input and other file assignments ----
#
echo ----- GAMESS execution script -----
set master=`hostname`
echo This job is running on host $master
echo under operating system `uname` at `date`
echo "Available scratch disk space (Kbyte units) at beginning of the job is"
df -k $SCR

#        this added as experiment, February 2007
limit stacksize 8192

#  Grab a copy of the input file.
#  In the case of EXAMnn jobs, this file might be in the "tests" subdirectory.
if ($JOB:r.inp == $JOB) set JOB=$JOB:r      # strip off possible .inp
if (-e $JOB.inp) then
   set echo
   cp  $JOB.inp  $SCR/$JOB.F05
   unset echo
else
   if (-e tests/$JOB.inp) then
      set echo
      cp  tests/$JOB.inp  $SCR/$JOB.F05
      unset echo
   else
      echo "Input file $JOB.inp does not exist."
      echo "This job expected the input file to be in directory `pwd`"
      echo "Please fix your file name problem, and resubmit."
      exit 4
   endif
endif

#  file assignments.
#
#  All binary files should be put on a node's local disk ($SCR directory), 
#  for the highest speed access possible.  These .Fxx files are typically
#  not saved for the next run, but they may be big and/or I/O intensive.
#  
#  It is convenient to write both ASCII output files (PUNCH and IRCDATA)
#  by software such as NFS to the user's permanent disk, on your file
#  server.  They are small.
#
#  Some data files may be read by a run, each is read only once, so
#  that storage of one (1) copy on your file server is appropriate.
#  a. The ERICFMT file is the Fm(t) data for ERI computations and is provided
#     with GAMESS.  It is essential for correct functioning of 2e- integrals.
#  b. The MCPPATH is just a path name, the program appends the specific
#     file names inside that path before opening.  The model core potentials
#     and basis set files come as part of GAMESS's normal distribution tree.
#  c. The EXTBAS file is user-supplied basis sets.
#  d. see NEO plug-in code's documentation regarding the NUCBAS file.
#         Note that you must edit a+b, but will probably skip c+d.
#
set echo
#                        ASCII input files (see explanation above)
setenv ERICFMT ~mike/gamess/ericfmt.dat
setenv MCPPATH ~mike/gamess/mcpdata
setenv  EXTBAS /dev/null
setenv  NUCBAS /dev/null
#
setenv  MAKEFP ~$USER/scr/$JOB.efp
setenv IRCDATA ~$USER/scr/$JOB.irc
setenv   INPUT $SCR/$JOB.F05
setenv   PUNCH ~$USER/scr/$JOB.dat
setenv  AOINTS $SCR/$JOB.F08
setenv  MOINTS $SCR/$JOB.F09
setenv DICTNRY $SCR/$JOB.F10
setenv DRTFILE $SCR/$JOB.F11
setenv CIVECTR $SCR/$JOB.F12
setenv CASINTS $SCR/$JOB.F13
setenv  CIINTS $SCR/$JOB.F14
setenv  WORK15 $SCR/$JOB.F15
setenv  WORK16 $SCR/$JOB.F16
setenv CSFSAVE $SCR/$JOB.F17
setenv FOCKDER $SCR/$JOB.F18
setenv  WORK19 $SCR/$JOB.F19
setenv  DASORT $SCR/$JOB.F20
setenv DFTINTS $SCR/$JOB.F21
setenv DFTGRID $SCR/$JOB.F22
setenv  JKFILE $SCR/$JOB.F23
setenv  ORDINT $SCR/$JOB.F24
setenv  EFPIND $SCR/$JOB.F25
setenv PCMDATA $SCR/$JOB.F26                                     
setenv PCMINTS $SCR/$JOB.F27
setenv SVPWRK1 $SCR/$JOB.F26
setenv SVPWRK2 $SCR/$JOB.F27
setenv  MLTPL  $SCR/$JOB.F28
setenv  MLTPLT $SCR/$JOB.F29
setenv  DAFL30 $SCR/$JOB.F30
setenv  SOINTX $SCR/$JOB.F31
setenv  SOINTY $SCR/$JOB.F32
setenv  SOINTZ $SCR/$JOB.F33
setenv  SORESC $SCR/$JOB.F34
setenv   SIMEN ~$USER/scr/$JOB.simen
setenv  SIMCOR ~$USER/scr/$JOB.simcor
setenv GCILIST $SCR/$JOB.F37
setenv HESSIAN $SCR/$JOB.F38
setenv SOCCDAT $SCR/$JOB.F40
setenv  AABB41 $SCR/$JOB.F41
setenv  BBAA42 $SCR/$JOB.F42
setenv  BBBB43 $SCR/$JOB.F43
setenv  MCQD50 $SCR/$JOB.F50
setenv  MCQD51 $SCR/$JOB.F51
setenv  MCQD52 $SCR/$JOB.F52
setenv  MCQD53 $SCR/$JOB.F53
setenv  MCQD54 $SCR/$JOB.F54
setenv  MCQD55 $SCR/$JOB.F55
setenv  MCQD56 $SCR/$JOB.F56
setenv  MCQD57 $SCR/$JOB.F57
setenv  MCQD58 $SCR/$JOB.F58
setenv  MCQD59 $SCR/$JOB.F59
setenv  MCQD60 $SCR/$JOB.F60
setenv  MCQD61 $SCR/$JOB.F61
setenv  MCQD62 $SCR/$JOB.F62
setenv  MCQD63 $SCR/$JOB.F63
setenv  MCQD64 $SCR/$JOB.F64
setenv NMRINT1 $SCR/$JOB.F61
setenv NMRINT2 $SCR/$JOB.F62
setenv NMRINT3 $SCR/$JOB.F63
setenv NMRINT4 $SCR/$JOB.F64
setenv NMRINT5 $SCR/$JOB.F65
setenv NMRINT6 $SCR/$JOB.F66
setenv DCPHFH2 $SCR/$JOB.F67
setenv DCPHF21 $SCR/$JOB.F68
setenv ELNUINT $SCR/$JOB.F67
setenv NUNUINT $SCR/$JOB.F68
setenv   GVVPT $SCR/$JOB.F69
setenv NUMOIN  $SCR/$JOB.F69
setenv NUMOCAS $SCR/$JOB.F70
setenv NUELMO  $SCR/$JOB.F71
setenv NUELCAS $SCR/$JOB.F72

#    Next files are for GMCQDPT
setenv GMCREF $SCR/$JOB.F70
setenv GMCO2R $SCR/$JOB.F71
setenv GMCROC $SCR/$JOB.F72
setenv GMCOOC $SCR/$JOB.F73
setenv GMCCC0 $SCR/$JOB.F74
setenv GMCHMA $SCR/$JOB.F75
setenv GMCEI1 $SCR/$JOB.F76
setenv GMCEI2 $SCR/$JOB.F77
setenv GMCEOB $SCR/$JOB.F78
setenv GMCEDT $SCR/$JOB.F79
setenv GMCERF $SCR/$JOB.F80
setenv GMCHCR $SCR/$JOB.F81
setenv GMCGJK $SCR/$JOB.F82
setenv GMCGAI $SCR/$JOB.F83
setenv GMCGEO $SCR/$JOB.F84
setenv GMCTE1 $SCR/$JOB.F85
setenv GMCTE2 $SCR/$JOB.F86
setenv GMCHEF $SCR/$JOB.F87
setenv GMCMOL $SCR/$JOB.F88
setenv GMCMOS $SCR/$JOB.F89
setenv GMCWGT $SCR/$JOB.F90
setenv GMCRM2 $SCR/$JOB.F91
setenv GMCRM1 $SCR/$JOB.F92
setenv GMCR00 $SCR/$JOB.F93
setenv GMCRP1 $SCR/$JOB.F94
setenv GMCRP2 $SCR/$JOB.F95
setenv GMCVEF $SCR/$JOB.F96
setenv GMCDIN $SCR/$JOB.F97
setenv GMC2SZ $SCR/$JOB.F98
setenv GMCCCS $SCR/$JOB.F99

#    Next files are used only during closed shell coupled cluster runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set cctyp=`grep -i CCTYP= $SCR/$JOB.F05 | wc -l`
if ($cctyp > 0) set echo
setenv  CCREST $SCR/$JOB.F70
setenv  CCDIIS $SCR/$JOB.F71
setenv  CCINTS $SCR/$JOB.F72
setenv CCT1AMP $SCR/$JOB.F73
setenv CCT2AMP $SCR/$JOB.F74
setenv CCT3AMP $SCR/$JOB.F75
setenv    CCVM $SCR/$JOB.F76
setenv    CCVE $SCR/$JOB.F77
setenv CCQUADS $SCR/$JOB.F78
setenv QUADSVO $SCR/$JOB.F79
setenv EOMSTAR $SCR/$JOB.F80
setenv EOMVEC1 $SCR/$JOB.F81
setenv EOMVEC2 $SCR/$JOB.F82
setenv  EOMHC1 $SCR/$JOB.F83
setenv  EOMHC2 $SCR/$JOB.F84
setenv EOMHHHH $SCR/$JOB.F85
setenv EOMPPPP $SCR/$JOB.F86
setenv EOMRAMP $SCR/$JOB.F87
setenv EOMRTMP $SCR/$JOB.F88
setenv EOMDG12 $SCR/$JOB.F89
setenv    MMPP $SCR/$JOB.F90
setenv   MMHPP $SCR/$JOB.F91
setenv MMCIVEC $SCR/$JOB.F92
setenv MMCIVC1 $SCR/$JOB.F93
setenv MMCIITR $SCR/$JOB.F94
setenv  EOMVL1 $SCR/$JOB.F95
setenv  EOMVL2 $SCR/$JOB.F96
setenv EOMLVEC $SCR/$JOB.F97
setenv  EOMHL1 $SCR/$JOB.F98
setenv  EOMHL2 $SCR/$JOB.F99
setenv  CCVVVV $SCR/$JOB.F80
#
#    Next files are used only during open shell coupled cluster runs.
#
setenv AMPROCC $SCR/$JOB.F70
setenv ITOPNCC $SCR/$JOB.F71
setenv FOCKMTX $SCR/$JOB.F72
setenv  LAMB23 $SCR/$JOB.F73
setenv   VHHAA $SCR/$JOB.F74
setenv   VHHBB $SCR/$JOB.F75
setenv   VHHAB $SCR/$JOB.F76
setenv    VMAA $SCR/$JOB.F77
setenv    VMBB $SCR/$JOB.F78
setenv    VMAB $SCR/$JOB.F79
setenv    VMBA $SCR/$JOB.F80
setenv  VHPRAA $SCR/$JOB.F81
setenv  VHPRBB $SCR/$JOB.F82
setenv  VHPRAB $SCR/$JOB.F83
setenv  VHPLAA $SCR/$JOB.F84
setenv  VHPLBB $SCR/$JOB.F85
setenv  VHPLAB $SCR/$JOB.F86
setenv  VHPLBA $SCR/$JOB.F87
setenv    VEAA $SCR/$JOB.F88
setenv    VEBB $SCR/$JOB.F89
setenv    VEAB $SCR/$JOB.F90
setenv    VEBA $SCR/$JOB.F91
setenv   VPPPP $SCR/$JOB.F92
setenv INTERM1 $SCR/$JOB.F93
setenv INTERM2 $SCR/$JOB.F94
setenv INTERM3 $SCR/$JOB.F95
#
#    Next files are used only during elongation method runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set elgtyp = `grep -i NELONG= $SCR/$JOB.F05 | wc -l`
if ($elgtyp > 0) then
    set ELGNAME=$4
    if (null$4 == null) set ELGNAME=ELGFILE
    set echo
    setenv AOINTS   $SCR/$ELGNAME.F08
    setenv ELGDOS   ~$USER/scr/$JOB.ldos
    setenv ELGDAT   $SCR/$ELGNAME.F71
    setenv ELGPAR   $SCR/$ELGNAME.F72
    setenv ELGCUT   $SCR/$ELGNAME.F74
    setenv ELGVEC   $SCR/$ELGNAME.F75
    setenv EGINTA   $SCR/$ELGNAME.F77
    setenv EGINTB   $SCR/$ELGNAME.F78
    setenv EGTDHF   $SCR/$ELGNAME.F79
    setenv EGTEST   $SCR/$ELGNAME.F80
    unset echo
endif
#
#    Next files are used only during extended TDHF package runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set txtyp=`grep -i RUNTYP=TDHFX $SCR/$JOB.F05 | wc -l`
if ($txtyp > 0) set echo
setenv  OLI201 $SCR/$JOB.F201
setenv  OLI202 $SCR/$JOB.F202
setenv  OLI203 $SCR/$JOB.F203
setenv  OLI204 $SCR/$JOB.F204
setenv  OLI205 $SCR/$JOB.F205
setenv  OLI206 $SCR/$JOB.F206
setenv  OLI207 $SCR/$JOB.F207
setenv  OLI208 $SCR/$JOB.F208
setenv  OLI209 $SCR/$JOB.F209
setenv  OLI210 $SCR/$JOB.F210
setenv  OLI211 $SCR/$JOB.F211
setenv  OLI212 $SCR/$JOB.F212
setenv  OLI213 $SCR/$JOB.F213
setenv  OLI214 $SCR/$JOB.F214
setenv  OLI215 $SCR/$JOB.F215
setenv  OLI216 $SCR/$JOB.F216
setenv  OLI217 $SCR/$JOB.F217
setenv  OLI218 $SCR/$JOB.F218
setenv  OLI219 $SCR/$JOB.F219
setenv  OLI220 $SCR/$JOB.F220
setenv  OLI221 $SCR/$JOB.F221
setenv  OLI222 $SCR/$JOB.F222
setenv  OLI223 $SCR/$JOB.F223
setenv  OLI224 $SCR/$JOB.F224
setenv  OLI225 $SCR/$JOB.F225
setenv  OLI226 $SCR/$JOB.F226
setenv  OLI227 $SCR/$JOB.F227
setenv  OLI228 $SCR/$JOB.F228
setenv  OLI229 $SCR/$JOB.F229
setenv  OLI230 $SCR/$JOB.F230
setenv  OLI231 $SCR/$JOB.F231
setenv  OLI232 $SCR/$JOB.F232
setenv  OLI233 $SCR/$JOB.F233
setenv  OLI234 $SCR/$JOB.F234
setenv  OLI235 $SCR/$JOB.F235
setenv  OLI236 $SCR/$JOB.F236
setenv  OLI237 $SCR/$JOB.F237
setenv  OLI238 $SCR/$JOB.F238
setenv  OLI239 $SCR/$JOB.F239
unset echo

#    Next files are used only during LMO hyperpolarizability analysis
setenv LHYPWRK $SCR/$JOB.F297
setenv LHYPWK2 $SCR/$JOB.F298
setenv BONDDPF $SCR/$JOB.F299

#    Next value is used only within the VB2000 add-on code
setenv GMSJOBNAME $JOB

#    data left over from a previous run might be precious, stop if found.
if ((-e $PUNCH) || (-e $IRCDATA)) then
   echo Please rename/erase $PUNCH and/or $IRCDATA, and resubmit
   exit 4
endif

#  ---- the middle third of the script is to execute GAMESS ----
#
#   Most workstations run DDI over TCP/IP sockets, and therefore execute
#   according to the following clause.  The installer must
#      a) Set the path to point to the DDIKICK and GAMESS executables.
#      b) Build the HOSTLIST variable as a word separated string, i.e. ()'s.
#         There should be one host name for every compute process that is
#         to be run.  DDIKICK will automatically generate a set of data
#         server processes (if required) on the same hosts.
#   An extended explanation of the arguments to ddikick.x can be found
#   in the file gamess/ddi/readme.ddi, if you have any trouble executing.
#
if ($TARGET == sockets) then
#
#        set the path pointing to GAMESS and DDIKICK binaries
#     At Iowa State, we have many operating systems, and store files
#     in different partitions according to which system is being used.
#
#     One such partition is fed to all nodes of the same type and
#     operating system, e.g. the /cu directory below feeds all of
#     our 32 bit PCs running RedHat Fedora Core 1, by means of NFS.
#
#     Note that 'uname' is not a terribly specific way to choose
#     a list of all machines running a specific O/S release, so we
#     use the host names for finer control, for example, we have 
#     three different Solaris releases on two different chip types.
#
   set os=`uname`
   if ($os == AIX)     set GMSPATH=/u1/mike/gamess
   if ($os == Darwin)  set GMSPATH=/Users/mike/desktop/gamess
   if ($os == HP-UX)   set GMSPATH=/zr/mike/gamess
   if ($os == Linux)   set GMSPATH=/cu/mike/gamess
   if ($os == OSF1)    set GMSPATH=/in/mike/gamess
   if ($os == SunOS)   set GMSPATH=/hf/mike/gamess
#       special compilation for Sun E450 uSPARC       (uname also= SunOS)
   if (`hostname` == sc.msg.chem.iastate.edu) set GMSPATH=/sc/mike/gamess
#       special compilation for Sun V40Z Opteron      (uname also= SunOS)
   if (`hostname` == as.msg.chem.iastate.edu) set GMSPATH=/as/mike/gamess
#       special compilation for SGI Altix 450         (uname also= Linux)
   if (`hostname` == br.msg.chem.iastate.edu) set GMSPATH=/br/mike/gamess
#       special compilation for SGI XE210             (uname also= Linux)
   if (`hostname` == se.msg.chem.iastate.edu) set GMSPATH=/se/mike/gamess
   if (`hostname` == sb.msg.chem.iastate.edu) set GMSPATH=/se/mike/gamess
#       special compilation for IBM OpenPower 720     (uname also= Linux)
   if (`hostname` == ga.msg.chem.iastate.edu) set GMSPATH=/ge/mike/gamess
   if (`hostname` == ge.msg.chem.iastate.edu) set GMSPATH=/ge/mike/gamess

#         it is unlikely that you would need to change DDI_VER from 'new'!
#         some antique system lacking pthreads, for example, might have
#         to use the old DDI code, so we keep an execution example below.
   set DDI_VER='new'
   if (`hostname` == antique.fi.ameslab.gov) set DDI_VER='old'

#      -- some special settings for certain operating systems --

#         IBM's AIX needs special setting if node is more than a 4-way SMP
   if ($os == AIX) setenv EXTSHM ON
#         HP's HP-UX uses a nonstandard name for its 'rsh' command
   if ($os == HP-UX) setenv DDI_RSH /usr/bin/remsh
#         Fedora Core 1 can't run DDI processes w/o placing a finite
#         but large limit on the stack size (2**27 bytes seems OK)
   if ($os == Linux) limit stacksize 131072
#         In case this Linux system is using Intel's Math Kernel Library
#         to obtain its BLAS, we insist each process runs single-threaded.
   if ($os == Linux) setenv MKL_SERIAL YES

#
#       Five examples of how to build the HOSTLIST are shown....
#           terminology: CPU= processor core,
#                       NODE= physical enclosure (box/blade)
#
#       1. Sequential execution is sure to be on this very same host
   if ($NCPUS == 1) then
      set NNODES=1
      set HOSTLIST=(`hostname`)
   endif
#
#       2. This is an example of how to run on a 4-way SMP enclosure,
#          where all CPUs (aka COREs) are inside a -single- NODE.
#          The example is repeated for some nodes we use as 2-way's.
   if ($NCPUS > 1) then
      switch (`hostname`)
         case ti.msg.chem.iastate.edu:
         case cd.msg.chem.iastate.edu:
         case zn.msg.chem.iastate.edu:
         case ni.msg.chem.iastate.edu:
         case co.msg.chem.iastate.edu:
         case sc.msg.chem.iastate.edu:
            if ($NCPUS > 4) set NCPUS=4
            set NNODES=1
            set HOSTLIST=(`hostname`:cpus=$NCPUS)
            breaksw
         case se.msg.chem.iastate.edu:
         case sb.msg.chem.iastate.edu:
         case br.msg.chem.iastate.edu:
            if ($NCPUS > 2) set NCPUS=2
            set NNODES=1
            set HOSTLIST=(`hostname`:cpus=$NCPUS)
            breaksw
         default:
            echo I do not know how to run this node in parallel.
            exit 20
      endsw
   endif
#
#       3. A phony example, of six uniprocessors (arbitrary names)
#          Because they are uniprocessors, we just set NNODES = NCPUS.
#          Since their names never change, we just can just specify them.
#          Note that we can use a short name like 'bb' if and only if 
#          system name resolution can map them onto the true host names.
   if (`hostname` == aa.fi.ameslab.gov) then
      set NNODES=$NCPUS
      set HOSTLIST=(aa bb cc dd ee ff)
   endif
#
#       4. An example of 16 uniprocessor boxes in a Beowulf-type cluster.
#          Because they are uniprocessors, we just set NNODES = NCPUS.
#          Their host names fall into the pattern fly1 to fly16,
#          which we can turn into a HOSTLIST with a small loop.
   if (`hostname` == fly1.fi.ameslab.gov) then
      set NNODES=$NCPUS
      set HOSTLIST=()
      set nmax=$NCPUS
      if ($nmax > 16) set nmax=16
      @ CPU=1
      while ($CPU <= $nmax)
         set HOSTLIST=($HOSTLIST fly$CPU)
         @ CPU++
      end
      unset $CPU
   endif
#
#       5. Scalable Computing Lab's clusters running PBS batch queues.
#          Here the task is to manipulate the dynamically assigned host
#          names into the HOSTLIST string for the kickoff program,
#          and to request the host name of the fast network adapters.
#
    if ($?PBS_JOBID) then
#
#          the SCL clusters use SSH2 for remote process generation.
      setenv DDI_RSH ssh
#
#         The IBM cluster has two Gigabit adapters in each 4-way SMP,
#         while the AXP cluster is based on a Myrinet network.
      if (`uname` == AIX)   set NETEXT=".gig,.gig2"
      if (`uname` == Linux) set NETEXT=".myri"
#
#         repeated host names in the PBS host file indicate being assigned
#         CPUs in the same SMP enclosure, which we must count up correctly.
#         Fortunately PBS gives duplicate host names in a row, not scrambled.
#         The number of hosts in the PBS node file (nmax) should equal the
#         requested processor count, NCPUS.  We need to count duplicates
#         in order to learn the number of SMP enclosures, NNODES, and how
#         many CPUs inside each SMP were assigned (NSMPCPU).  For example,
#         if we are assigned the host names "a a a b b c c c" we must build
#         the string "a:cpus=3 b:cpus=2 c:cpus=3" so that ddikick.x will
#         know the SMP structure of the assigned node names.  (C-shell handles
#         variable substitution followed by colon gracefully by ${HOST}:cpus.)
#
      set HOSTLIST=()
      set nmax=`wc -l $PBS_NODEFILE`
      set nmax=$nmax[1]
      if ($nmax != $NCPUS) then
         echo There is processor count confusion
         exit
      endif
#            1st host in the list is sure to be a new SMP enclosure
      set MYNODE=`sed -n -e "1 p" $PBS_NODEFILE`
      set MYNODE=`echo $MYNODE | awk '{split($0,a,"."); print a[1]}'`
#            IPROC counts assigned processors (up to NCPUS),
#            NNODES counts number of SMP enclosures.
#            NSMPCPU counts processors in the current SMP enclosure
      @ IPROC = 2
      @ NNODES = 1
      @ NSMPCPU = 1
      set spacer1=":cpus="
      set spacer2=":netext="
      while($IPROC <= $nmax)
         set MYPROC=`sed -n -e "$IPROC p" $PBS_NODEFILE`
         set MYPROC=`echo $MYPROC | awk '{split($0,a,"."); print a[1]}'`
         if($MYPROC != $MYNODE) then
            set HOSTLIST = ($HOSTLIST $MYNODE$spacer1$NSMPCPU$spacer2$NETEXT)
            set MYNODE=$MYPROC
            @ NSMPCPU = 0
            @ NNODES++
         endif
         @ IPROC++
         @ NSMPCPU++
      end
      set HOSTLIST = ($HOSTLIST $MYNODE$spacer1$NSMPCPU$spacer2$NETEXT)
   endif
#
#        we have now finished setting up a correct HOSTLIST.
#        uncomment the next two if you are doing script debugging.
#--echo "The generated host list is"
#--echo $HOSTLIST
#

#
#    If a $GDDI input group is present the calculation will be using
#    subgroups within DDI.  The master within each group must have a
#    copy of INPUT, and will create separate PUNCH and OUTPUT files.
#    Note that this separates the OUTPUT from the normal log file, and
#    at the end we must take steps to save the files from at least the
#    master of the first group.  Skip if no $GDDI is found in the input.
#
   set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=1 ' | wc -l`
   if ($ngddi > 0) then
      set GDDIjob=true
      echo "This is a GDDI run, keeping output files on local disks"
      echo "until the very end of the run, when they'll be saved from"
      echo "the master process in the first group, only."
      set echo
      setenv IRCDATA $SCR/$JOB.F04
      setenv  OUTPUT $SCR/$JOB.F06
      setenv   PUNCH $SCR/$JOB.F07
      unset echo
      @ n=2   # master in master group already did 'cp' above
      while ($n <= $NNODES)
         set host=$HOSTLIST[$n]
         set host=`echo $host | cut -f 1 -d :` # drop anything behind a colon
         echo rcp $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
         rcp $SCR/$JOB.F05 ${host}:$SCR/$JOB.F05
         @ n++
      end
   else
      set GDDIjob=false
   endif
#
#        Just make sure we have the binaries, before we try to run
#
   if ((-x $GMSPATH/gamess.$VERNO.x) && (-x $GMSPATH/ddikick.x)) then
   else
      echo The GAMESS executable gamess.$VERNO.x or else
      echo the DDIKICK executable ddikick.x
      echo could not be found in directory $GMSPATH,
      echo or else they did not properly link to executable permission.
      exit 8
   endif
#
#        OK, now we are ready to execute!
#    The kickoff program initiates GAMESS process(es) on all CPUs/nodes.
#    If your site has rsh turned off, consider "setenv DDI_RSH /usr/bin/ssh"
#
   if ($DDI_VER == new) then
      set echo
      $GMSPATH/ddikick.x $GMSPATH/gamess.$VERNO.x $JOB \
          -ddi $NNODES $NCPUS $HOSTLIST \
          -scr $SCR < /dev/null
      unset echo
   else
      set path=($GMSPATH $path)
      set echo
      ddikick.x $JOB $GMSPATH gamess.$VERNO.x $SCR $NCPUS $HOSTLIST < /dev/null
      unset echo
   endif
endif

#      ------ end of the TCP/IP socket execution sections -------

#   Compaq Supercluster running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == compaq-sc) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   prun -n $NNODES $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif
 
#   Cray T3E running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-t3e) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   mpprun -n $NCPUS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   Cray X1 running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-x1) then
   set GMSPATH=/u1/mike/gamess
   set OPTS="-m exclusive"
   if ($NCPUS > 16) then
      set PERNODE=16
   else
      set PERNODE=$NCPUS
   endif
   chdir $SCR
   set echo
   aprun -c core=0 -n $NCPUS -N $PERNODE $OPTS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif

#   SGI Origin running SHMEM wants you to
#      a) set the path to point to the GAMESS executable
#   We've heard that setting the environment variable
#       SMA_SYMMETRIC_SIZE to 2147483648 (2 GB)
#   may be helpful if you see DDI_SHPALLOC error messages.
#
if ($TARGET == sgi64) then
   set GMSPATH=/home/hbar4/people/schmidt/gamess
   chdir $SCR
   set echo
   mpirun -np $NCPUS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif

#   CRAY-XT3 running SHMEM/MPI wants you to
#      a) set the path to point to the GAMESS executable
#      b) review the memory settings below.
#   This implementation uses one-sided messaging, i.e. yod does not
#   start up data server processes, just compute processes.
#
#   Set the -shmem parameter to about 400MB smaller than the memory
#   available to each core to allow for MPI buffer space and other OS
#   buffers.  For example, if the XT3 node has 2GB,
#     single-core = 2000MB - 400MB = 1600MB
#     dual-core   = 2GB / 2 cores = 1000MB - 400MB = 600MB
#   You can sometimes drop to subtracting only 300MB per core,
#   depending on the job type.
#
#   The TPN variable below lets you use more memory, by wasting one
#   half of the processors, if that is needed to do your run.
#
#   If you get an error of the type 
#      Fatal error in MPI_Ssend: Invalid rank, error stack:
#   then you have run out of stack space.  Increase the -stack argument
#   from "-stack 24m" to "-stack 32m" or larger.  This usually only occurs
#   for very large jobs.
#
if ($TARGET == cray-xt3) then
   set GMSPATH=/u/boatzj/gamess
   set SMP_SIZE=2  # number of cpus/cores per node (XT3 = dual CPU nodes)
   set TPN=$4      # number of compute processes per node (TPN=tasks/node)
   if (null$TPN == null) set TPN=$SMP_SIZE
#
   if ($TPN == 1) then
      set CORTYP=-SN
      set SHMEM='-shmem 3600M'
   else
      set CORTYP=-VN
      set SHMEM='-shmem 1700M'
   endif
#
   setenv IOBUF_PARAMS '%stdout:ignoreflush'
   env | grep IOBUF
#
   chdir $SCR
   set echo
   yod -stack 24m $CORTYP $SHMEM -size $NCPUS $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif


#   The IBM SP running DDI using mixed LAPI/MPI messaging wants you to
#      a) set the path to point to the GAMESS executable
#      b) define hosts in a host file, which are probably defined by
#         a batch queue system.  An example for LoadLeveler is given.
#   Please note that most IBM systems schedule their batch jobs with
#   the LoadLeveler software product.  Please see gamess/misc/llgms for
#   a "front-end" script that submits this script as a "back-end" job,
#   with all necessary LL accouterments inserted at the top of the job.
#
if ($TARGET == ibm64-sp) then
#
#     point this to where your GAMESS executable is located
   set path=($path /u1/mike/gamess)
#
#     error messages defaulted to American English, try C if lacking en_US
   setenv LOCPATH /usr/lib/nls/loc:/usr/vacpp/bin
   setenv LANG en_US
#
#     this value is picked up inside DDI, then used in a "chdir $SCR"
   setenv DDI_SCRATCH $SCR
#
#     define the name of a host name file.
#
   setenv HOSTFILE $SCR/$JOB.poehosts
   if (-e $HOSTFILE) rm -f $HOSTFILE
#
#        If the job was scheduled by LoadLeveler, let LL control everything.
#
   if ($?LOADLBATCH) then
#        just get POE to tell us what nodes we were dynamically assigned to.
      /usr/bin/poe hostname -stdoutmode ordered > $HOSTFILE
      set SMP_SIZE = $TPN
#
#        Otherwise, if this was not an LoadLeveler job, here's a hack!
#        It is unlikely this will match your SP's characteristics, as
#        we just guess its a 4-way node, 4 processors, run interactively.
#        It is here mainly to illustrate the sort of MP_XXX's you need.
   else
      set SMP_SIZE=4
      set NCPUS=4
      set NNODES=1
      echo `hostname`  > $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE
      echo `hostname` >> $HOSTFILE

      echo "Variables controlling Parallel Environment process kickoff are"
      set echo
      setenv MP_NODES          $NNODES
      setenv MP_PROCS          $NCPUS
      setenv MP_HOSTFILE       $HOSTFILE
      setenv MP_CPU_USE        unique
      setenv MP_ADAPTER_USE    dedicated
#       GAMESS is implemented using both MPI and LAPI active messages.
      setenv MP_MSG_API        MPI,LAPI
      setenv MP_EUILIB         us
#       SP systems with one switch adapter might use css0, not striping csss
      setenv MP_EUIDEVICE      csss
      setenv MP_RESD           no
      unset echo
   endif

#       and now we are ready to execute, using poe to kick off the tasks.
   @ NNODES = ($NCPUS - 1) / $SMP_SIZE + 1
   echo "Running $NCPUS processes on $NNODES nodes ($SMP_SIZE-way SMP)."
   set echo
   /usr/bin/poe gamess.$VERNO.x $JOB -stdinmode none
   unset echo
endif

#                        - two MPI examples -
#                     see ~/gamess/ddi/readme.ddi)
#
if ($TARGET == mpi) then
#---   #
#---   #        EXAMPLE #1
#---   #        two SGI XE210 blades, each 4-cores, named se and sb,
#---   #        the network is a Voltaire/Mellanox Infiniband.
#---   #
#---   @ NPROCS = $NCPUS + $NCPUS
#---   echo Running $NCPUS compute processes and $NCPUS data servers...
#---   #
#---   #     Voltaire/Mellanox software seems to want short hostnames, only.
#---   #     The next line is just a convenient way to hardwire our two
#---   #     names, the goal here is to prepare a disk file with one line
#---   #     per name to feed to the MPI kickoff program.
#---   set HOSTLIST=(se sb)
#---   #
#---   #     so lets begin by creating an empty file for that...
#---   if (-e $SCR/$JOB.hostlist) rm $SCR/$JOB.hostlist
#---   touch $SCR/$JOB.hostlist
#---   #
#---   #     The bit below only works if NCPUS is an even number, by the way.
#---   #     we told you that we weren't trying to be general here!
#---   #     For NCPUS=2, we get se/sb/se/sb
#---   #     For NCPUS=4, we get se/se/sb/sb/se/se/sb/sb
#---   #     Most sites will probably prefer to look at the Myrinet example
#---   #     below, to use dynamically assigned names (and odd CPU counts).
#---   @ NHALF = $NCPUS / 2
#---   #          place the compute processes first...
#---   @ nh=1
#---   @ nhosts=$#HOSTLIST
#---   while ($nh <= $nhosts)
#---      @ np=1
#---      while ($np <= $NHALF)
#---         echo $HOSTLIST[$nh] >> $SCR/$JOB.hostlist
#---         @ np++
#---      end
#---      @ nh++
#---   end
#---   #          ...and then, lay down the data servers.
#---   @ nh=1
#---   @ nhosts=$#HOSTLIST
#---   while ($nh <= $nhosts)
#---      @ np=1
#---      while ($np <= $NHALF)
#---         echo $HOSTLIST[$nh] >> $SCR/$JOB.hostlist
#---         @ np++
#---      end
#---      @ nh++
#---   end
#---   #
#--    #  Next is very clunky way to pass file names to 1st compute process.
#--    #  It requires that you compiled iolib and unport with the "FGE" hack.
#--    #  Since you can have only one file called GAMESS.ENVIRON on your
#--    #  first node, you can run only one GAMESS job at a time on it.
#---   env >& $SCR/GAMESS.ENVIRON
#---   chdir $SCR
#---   #
#---   set echo
#---   /usr/voltaire/mpi.gcc.rsh/bin/mpirun_rsh -noinput \
#---        -np $NPROCS  -hostfile $SCR/$JOB.hostlist \
#---        /se/mike/gamess/gamess.$VERNO.x -scr $SCR
#---   unset echo
#---   #
#---   rm -f $SCR/$JOB.hostlist
#---   rm -f $SCR/GAMESS.ENVIRON
#
#        EXAMPLE #2 (see ~/gamess/ddi/readme.ddi)
#     For a system with Myrinet connecting AXP uniprocessors.
#     The batch scheduler puts a list of host names assigned to
#     this job into PBS_NODEFILE, containing one name per line,
#     such as "a1/a5/a8/a16" for a p=4 job.  Network adapters are
#     called aX.myri and aX.fast in each node.  The mpirun will
#     select the myrinet directly from names like "aX" for MPI
#     traffic, but the small amount of TCP/IP traffic is directed
#     to one or the other adapter by -netext.
#     DDI will later sort out the host names looking for repeats
#     to learn the SMP nature of the cluster, so launch the MPI
#     processes in any order, but be sure to duplicate processes
#     on every CPU so that you get data servers, too.
#
#            note doubling of process count, and double copy of host file
   @ NPROCS = $NCPUS + $NCPUS
   cat $PBS_NODEFILE  > ~/scr/gms-hostlist.$JOB
   cat $PBS_NODEFILE >> ~/scr/gms-hostlist.$JOB
   set echo
   /usr/local/mpich-gm/bin/mpirun.ch_gm -np $NPROCS \
         -machinefile ~/scr/gms-hostlist.$JOB \
         --gm-recv blocking \
         $GMSPATH/gamess.$hw.$VERNO.x -scr $SCR -netext .myri
   unset echo
   rm -f ~/scr/gms-hostlist.$JOB
endif

#   NEC SX Series wants you to
#      a) set the path variable to point to the GAMESS executable
#   this isn't correct, it uses MPI, but need to save the F_ stuff???
#
if ($TARGET == necsx) then
   set GMSPATH=/u1/mike/gamess

   chdir $SCR
   setenv F_RECLUNIT BYTE
   setenv F_ABORT YES
   setenv F_ERROPT1 252,252,2,2,1,1,2,1
   setenv F_PROGINF detail
   setenv F_SETBUF 4096
   echo Running $NCPUS compute processes and $NCPUS data server processes...
   @ NPROCS = $NCPUS + $NCPUS
   set echo
   #--mpirun -np $NPROCS $GMSPATH/gamess.$VERNO.x $JOB < /dev/null
   unset echo
endif

#   in the case of GDDI runs, we save the first group master's files only.
if ($TARGET != sockets) set GDDIjob=false # just didn't set them up for GDDI
if ($GDDIjob == true) then
   cat $OUTPUT
   cp $SCR/$JOB.F07 ~/scr/$JOB.dat
   if(-e $SCR/$JOB.F04) cp $SCR/$JOB.F04 ~/scr/$JOB.irc
   cp $SCR/$JOB.F06.* ~/scr
endif

#
#  ---- the bottom third of the script is to clean up all disk files ----
#
echo ----- accounting info -----
date
#
#   Clean up the master's scratch directory.
#
echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*
#
#   Clean up scratch directory of remote nodes.
#
#   This may not be necessary, e.g. on a T3E where all files are in the
#   same directory, and just got cleaned out by the previous 'rm'.  Many
#   batch queue managers provide cleaning out of scratch directories.
#   It still may be interesting to the user to see the sizes of files.
#
#   The 'lasthost' business prevents multiple cleanup tries on SMP nodes.
#
if ($TARGET == sockets) then
   set nmax=${#HOSTLIST}
   set lasthost=$master
   @ n=2   # master already cleaned above
   while ($n <= $nmax)
      set host=$HOSTLIST[$n]
      set host=`echo $host | cut -f 1 -d :`   # drop anything behind a colon
      if ($host != $lasthost) then
         echo Files from $host are:
         rsh $host -l $USER -n "ls -l $SCR/$JOB.*"
         rsh $host -l $USER -n "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
endif
#
if ($TARGET == mpi) then
   set nmax=`wc -l $PBS_NODEFILE`
   set nmax=$nmax[1]
   set lasthost=$master
   @ n=2   # we already cleaned up the master node just above.
   while ($n <= $nmax)
      set host=`sed -n -e "$n p" $PBS_NODEFILE`
      if ($host != $lasthost) then
         echo Files used on node $host were:
         ssh $host -l $USER "ls -l $SCR/$JOB.*"
         ssh $host -l $USER "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
endif
#
#   IBM SP cleanup code...might need to be something other than 'rsh'.
#
if ($TARGET == ibm64-sp) then
   set lasthost=$master
   @ n=2   # we already cleaned up the master node just above.
   @ nmax=$NCPUS
   while ($n <= $nmax)
      set host=`sed -n -e "$n p" $HOSTFILE`
      if ($host != $lasthost) then
         echo Files used on node $host were:
         rsh $host "ls -l $SCR/$JOB.*"
         rsh $host "rm -f $SCR/$JOB.F*"
         set lasthost=$host
      endif
      @ n++
   end
   rm -f $HOSTFILE
endif
#
#  and this is the end
#
time
exit