############################################################### # # # Shell script for submitting a serial job to Owens' # # PBS queue using the qsub command. # # # ############################################################### # Remarks: A line beginning with # is a comment. # A line beginning with #PBS is a PBS directive. # PBS directives must come first; any directives # after the first executable statement are ignored. # ########################## # # # The PBS directives # # # ########################## # Set the name of the job (up to 15 characters, # no blank spaces, start with alphanumeric character) #PBS -N JobName # By default, the standard output and error streams are sent # to files in the current working directory with names: # job_name.o <- output stream # job_name.e <- error stream # where job_name is the name of the job and # is the job number assigned when the job is submitted. # Use the directives below to change the files to which the # standard output and error streams are sent. # #PBS -o stdout_file # #PBS -e stderr_file # The directive below directs that the standard output and # error streams are to be merged, intermixed, as standard # output. Omit to keep the two as separate files. #PBS -j oe # Specify the maximum wall clock time (walltime). # Format: hhhh:mm:ss hours:minutes:seconds # Be sure to specify a value with some overhead, # to handle deviations in how long the job takes. # # When submitting a type of job for the first time # the required walltime should be overestimated. # In subsequent runs adjust this value to a # reasonable value (required walltime +20% overhead). # # If the job does not finish by the time reached, # the job is terminated. #PBS -l walltime=1:00:00 # Specify the maximum amount of physical memory required. # kb for kilobytes, mb for megabytes, gb for gigabytes. # Take some care in setting this value. # # Oakley's standard nodes have 48GB of RAM. #PBS -l mem=512mb # PBS can send informative email messages to you about the # status of your job. Specify a string which consists of # either the single character "n" (no mail), or one or more # of the characters "a" (send mail when job is aborted), # "b" (send mail when job begins), and "e" (send mail when # job terminates). The default is "a" if not specified. # You should also specify the email address to which the # message should be send via the -M option. # #PBS -m abe # #PBS -m ae #PBS -M user_email_address # Specify the number of nodes requested and the # number of processors per node. # # If GPUs are required, specify it here using the following # format, where x is the number of GPUs per a node. # # There are 62 GPU enabled nodes, with 2 GPUs each. # # #PBS -l nodes=1:ppn=12:gpus=2 # # If more than 48GBs of RAM are required per a node the job # can be run on the bigmem or hugemem nodes. # # There are 8 bigmem nodes with 192GB of RAM each. To request # one add :bigmem to your nodes request. # # #PBS -l nodes=1:ppn=12:bigmem # # There is 1 hugemem node with 1TB of RAM. It has 32 processors # instead of 12. Request it by requesting 32 processors. # # #PBS -l nodes=1:ppn=32 #PBS -l nodes=1:ppn=1 # Specify project to charge usage. # If you only have one project, system will default to that project. But you # will receive a warning message when you submit your job. # #PBS -A PXXXXXXXX ########################################## # # # Output some useful job information. # # # ########################################## echo ------------------------------------------------------ echo -n 'Job is running on node '; cat $PBS_NODEFILE echo ------------------------------------------------------ echo PBS: qsub is running on $PBS_O_HOST echo PBS: executing queue is $PBS_QUEUE echo PBS: working directory is $PBS_O_WORKDIR echo PBS: job identifier is $PBS_JOBID echo PBS: job name is $PBS_JOBNAME echo PBS: node file is $PBS_NODEFILE echo PBS: current home directory is $PBS_O_HOME echo PBS: PATH = $PBS_O_PATH echo ------------------------------------------------------ ########################################################################################### # # # The trap command allows you to specify a command to run in case your job terminates # # abnormally, for example if it runs out of wall time. It is typically used to copy # # output files from a temporary directory to a home or project directory. The following # # example creates a directory in $PBS_O_WORKDIR and copies everything from $TMPDIR into # # it. This executes only if the job terminates abnormally. # # # ########################################################################################### trap "cd $PBS_O_WORKDIR;mkdir $PBS_JOBID;cp -R $TMPDIR/* $PBS_JOBID" TERM ###################################################################### # # # To minimize communications traffic, it is best for your job # # to work with files on the local disk of the compute node. # # Hence, one needs to transfer files from your permanent home # # directory tree to the directory ${WORKDIR} automatically # # created by PBS on the local disk before program execution, # # and to transfer any important output files from the local # # disk back to the permanent home directory tree after program # # execution is completed. # # # # There are essentially two ways to achieve this: (1) to use the # # PBS stagein and stageout utilities, or (2) to manually copy the # # files by commands in this script. The stagein and stageout # # features of OpenPBS are somewhat awkward, especially since # # wildcards and macros in the file lists cannot be used. This # # method also has some timing issues. Hence, we ask you to use # # the second method, and to use secure copy (scp) to do the file # # transfers to avoid NSF bottlenecks. # # # ###################################################################### ##################################################### # # # Specify the permanent directory(ies) on the # # server host. Note that when the job begins # # execution, the current working directory at # # the time the qsub command was issued becomes # # the current working directory of the job. # # # ##################################################### ############################################################### # # # Transfer files from server to local disk. # # # ############################################################### echo Transferring files from server to compute node echo Writing files in node directory cd $TMPDIR cp $PBS_O_WORKDIR/input_file . cp $PBS_O_WORKDIR/program_executable . echo Files in node work directory are as follows: ls -l ############################################################ # # # Execute the run. Do not run in the background. # # # ############################################################ program_executable < input_file > output_file ########################################################### # # # Copy necessary files back to permanent directory. # # # ########################################################### cp *results* $PBS_O_WORKDIR ############################################################### # # # The epilogue script automatically deletes the directory # # created on the local disk (including all files contained # # therein. # # # ############################################################### exit