#!/bin/bash # # Sebastian Mutz, Jingmin Li, Todd Ehlers # Geowissenschaftliche Fakultät der Universität Tübingen # # For questions regarding this specific script, contact: # sebastian.mutz@uni-tuebingen.de, jingmin.li@uni-tuebingen.de # todd.ehlers@uni-tuebingen.de # # ################################################################## # About this template script (based on MPI template): # # # # 1. Scheduler options # # 2. All important options are defined at the top of the run # # script. There is no need for the user to look beyond these. # # 3. Generates a useful directory structure in your work folder # # 5. Restarts itself automatically. Run time only limited by wall # # clock limit and your defined final run year. # # 6. Makes backups of the rerun files. # # 7. Allows manual restarts from a time step of your choice # # 8. Makes backups of your input files, run scripts, configuration # # files and model executable for experiment reproducability. # # 9. No warranty. For your specific purposes, you might want to # # make some changes. # # ################################################################## # # # # # # # # ################### BEGIN USER DECLARATIONS ##################### # ------------------------------------------------------------- # # # # =============================================================== # |||| Torque Portable Batch System (PBS, Scheduler) Options |||| # =============================================================== # # Options: # #PBS -l nodes=4:ppn=20:esd1 #PBS -l walltime=48:00:00 #PBS -M jingmin.li@uni-tuebingen.de #PBS -m abe #PBS -N 2015_e007_2 #PBS -q esd # # Comments: # # request 4 node, 80 cores # set wallclock limit to 48 hours # send email to given address # send email only when programme aborts # set job name to 2014_e007 # set queue to geotest, we have to use this # # # # # # # # =============================================================== # ||||||||| Job script for running ECHAM5 on HPC-BW ||||||||||||| # =============================================================== # # If set and a command has a non-zero exit status, ERR trap is # executed and programme exits set -ex # # ============================ # ||||||||| Basic declarations # # This is the section for user defined options for the model runs. # It includes anything from your user and experiment name to green # house gas concentrations. For more options, look in the namelist # file in your echam/doc directory. Give the option variable a value # in this section and make sure it's written in echam.namelists # under the correct heading (eg. runctl; see ECHAM namelists section), # so it's passed on to the model. # # ---------------------------- # ------- Usr and Dir -------- # # The path names are generated automatically from the options below. # Please stick with directory structure of your echam5 distribution # to avoid any complications. # USR=emili01 # user name MODD=echam5-wiso.v2.3_140305 # model directory relative to home RUND=e007_2 # subdirectoty in ~/model/run/ EXEC=echam5-wiso.2.3_20150511 # name of the executable in ~/model/bin/ INPUT=T159_PI # module load compiler/intel/compiler/intel/13.1 numlib/mkl/11.0.5 mpi/openmpi/1.8-intel-13.1 # LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/bwgrid/lib/netcdf/lib-netcdf-4.2.1.1_c-openmpi-1.4.3-gnu-4.1/lib:/opt/bwgrid/lib/netcdf/lib-netcdf-4.2_fortran-openmpi-1.4.3-gnu-4.1/lib:/opt/bwhpc/tu/geo/echam5/share/lib export LD_LIBRARY_PATH # # ---------------------------- # ------- Job options -------- # EXP=e007_2 # experiment identifier EXPD=hpc-bw_e5w2.3_t159l31.1d_PI # extended experiment name. this will be used to name the output folder RUN_FILE="e007_2_hpc-bw_e5w2.3_t159_PI" # name of the run script you are currently viewing RE_FILE="rerun.txt" # file that stores last date of completed job iteration. used by re-submitting script # LABORT=.true. # stop at the end of the rerun chain? NCPUS=80 #32 # number of CPUs for parallel run NPROCA=8 # number of CPUs to process task A NPROCB=10 #4 # number of CPUs to process task B NTHREADS=1 # number of threads. 1 if no hybrid run #NPROMA=16 # blocking length/vector length # # ---------------------------- # ------- Model options ------ # # Rerun options REI=3 # re-submit time within wallcolock # # Time related options YEARI=1000 # start year of run (do not change for restart runs) YEARF=1017 # final year of run (do not change for restart runs) # Put all years in year variable. not neccessary if model does not rely on time dependent input. #years="1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966" # # Physics and dynamics RES=159 # model resolution LEVELS=31 # number of levels LAMIP=.false. # true if run with amip SST LMLO=.false. # run with slab ocean? l_orbvsop87=.false. # .false. for paleo run ICFC=0 # no cfc ICO2=2 # 0 for no CO2, 2 for constant CO2 level = CO2VMR = 348.E-06 ICH4=2 # 0 for no CH4, 2 for constant CH4 level = CH4VMR = 1.65E-06 IN2O=2 # 0 for no N2O, 2 for constant N2O level = N2OVMR = 306.E-09 co2vmr=280.E-06 # PI co2=280ppm ch4vmr=760.E-09 # PI ch4=760ppb n2ovmr=270.E-09 # PI n2o=270ppb cecc=0.016804 # PI cecc=0.016804 PD cecc= cobld=23.4725 # PI cobld=23.4725 PD cobld= clonp=278.734 # PI clonp=278.734 PD clonp= # # Output options FMT=2 # output format 1=GRIB, 2=netCDF OU=days # output time interval unit OI=1 # output time interval REOU=months # output time interval unit for rerun files REOI=1 # output time interval for rerun files # # Note: needs a text file "rerun.txt" in the same folder as the run file. # For the start of the run, write in it: # ".false. [start year +1] [start month]" # .false. refers to the fact it's not a restart. This and the time step # will be changed automatically before a restart. # # ------------------------------------------------------------- # #################### END USER DECLARATION ####################### # # # # # # # =========================== # ||||||||| Path declarations # # Path to working directory (model folder) WRK_DIR=/home-link/${USR}/ECHAM # MOD_DIR=/home-link/${USR}/models/ECHAM/${MODD} # # Path to job script RUN_DIR=${MOD_DIR}/run/${RUND} # # Path to experiment directory EXP_DIR=${WRK_DIR}/experiments/${EXP}_${EXPD} # # Path to output directory OUT_DIR=${EXP_DIR}/output_raw # # Path to directory storing rerun files RRN_DIR=${OUT_DIR}/rerun # # Path to initial data INI_DIR=${WRK_DIR}/input/${INPUT} # # Path to amip sst BND_DIR=${INI_DIR}/amip2 # # Path to ocean delta values D18_DIR=${MOD_DIR}/iniwiso # # Path to model executable + executable name MODEL=${MOD_DIR}/bin/${EXEC} # # ------------------------------------------------- # Paths in source directory. In src/ everything run # specific is stored. # # Path to source directory S_SRC_DIR=${EXP_DIR}/src # # Path to input files S_INI_DIR=${S_SRC_DIR}/input # # Path to script directory S_SPT_DIR=${S_SRC_DIR}/scripts # # Path to model directory (run file will also be copied here) S_MOD_DIR=${S_SRC_DIR}/model # # ------------------------------------ # Paths to post processing directories # # Path to processed output directory P_OUT_DIR=${EXP_DIR}/output_processed # # Paths to script directories P_SPT_DIR=${EXP_DIR}/scripts P_NCO_DIR=${P_SPT_DIR}/nco # for nco scripts P_NCL_DIR=${P_SPT_DIR}/ncl # for ncl scripts P_GMT_DIR=${P_SPT_DIR}/gmt # for gmt scripts P_FOR_DIR=${P_SPT_DIR}/for # for fortran scripts # # Path to plot directories P_PLT_DIR=${EXP_DIR}/plots P_DFT_DIR=${P_PLT_DIR}/drafts P_FIN_DIR=${P_PLT_DIR}/final # read RERUN YEARB index< ${RUN_DIR}/rerun.txt echo $RERUN #rerun switch, .true. for rerun, .false. for initial run echo $YEARB #final year of this job, one simulation is consist of many jobs. echo $index # # # # # # # ====================================== # ||||||||| Generate directory structure # # Generate directories for initial run if [[ $RERUN == .false. ]]; then # # Make output directories mkdir -p ${EXP_DIR} mkdir -p ${OUT_DIR} && mkdir -p ${RRN_DIR} # # Make source directories mkdir -p ${S_SRC_DIR} mkdir -p ${S_INI_DIR} && mkdir -p ${S_MOD_DIR} && mkdir -p ${S_SPT_DIR} mkdir -p ${S_MOD_DIR}/${MODD} && mkdir -p ${S_MOD_DIR}/${MODD}/run mkdir -p ${S_MOD_DIR}/${MODD}/bin && mkdir -p ${S_MOD_DIR}/${MODD}/config # # Make post processing directories mkdir -p ${P_OUT_DIR} # processed output directory mkdir -p ${P_NCO_DIR} && mkdir -p ${P_NCL_DIR} # scripts directories mkdir -p ${P_GMT_DIR} && mkdir -p ${P_FOR_DIR} # scripts directories mkdir -p ${P_DFT_DIR} && mkdir -p ${P_FIN_DIR} # plots directories fi # # ------------------------------------------- # Save relevant files in the source directory # if [[ $RERUN == .false. ]]; then # save input files cp -r ${INI_DIR} ${S_INI_DIR} #save model files cp -r ${RUN_DIR} ${S_MOD_DIR}/${MODD}/run # save run scripts cp ${MODEL} ${S_MOD_DIR}/${MODD}/bin # save model executable cp ${MOD_DIR}/config/mh* ${S_MOD_DIR}/${MODD}/config # save configuration file(s) fi # # ================================ # ||||||||| Specification of files # # ----------------------------------------------- # Change directory before creating symbolic links. # All need to be in the output directory. cd $OUT_DIR # rm -f unit.?? sst* ice* hdpara.nc hdstart.nc rrtadata greenhouse_gases_A1B.nc # # Create symbolic links for various input files ln -sf ${INI_DIR}/T${RES}L${LEVELS}_jan_spec.nc unit.23 ln -sf ${INI_DIR}/T${RES}_jan_surf.nc unit.24 ln -sf ${INI_DIR}/T${RES}_O3clim2.nc unit.21 ln -sf ${INI_DIR}/T${RES}_VLTCLIM.nc unit.90 ln -sf ${INI_DIR}/T${RES}_VGRATCLIM.nc unit.91 ln -sf ${INI_DIR}/T${RES}_TSLCLIM2.nc unit.92 # ln -sf ${INI_DIR}/surrta_data rrtadata #ln -sf ${INI_DIR}/hdpara.nc hdpara.nc #ln -sf ${INI_DIR}/hdstart.nc hdstart.nc # # Create symbolic links for delta values ln -sf ${D18_DIR}/T${RES}_wisosw_d.nc unit.25 # Create symbolic links for SSTs #ln -sf ${BND_DIR}/T${RES}_amip2sst_clim.nc unit.20 #ln -sf ${BND_DIR}/T${RES}_amip2sic_clim.nc unit.96 ln -sf ${BND_DIR}/SST.amip2.T159.PI.filled.nc unit.20 ln -sf ${BND_DIR}/SIC.amip2.T159.PI.filled.nc unit.96 # #for year in $years ; do #ln -sf ${BND_DIR}/T${RES}_amip2sst_${year}.nc sst${year} #ln -sf ${BND_DIR}/T${RES}_amip2sic_${year}.nc ice${year} #done # # # # # # # ======================= # ||||||||| Preparing run # # #################################### # ########### Restart loop ########### # count=1 while (($count<=$REI)); do # # ========================= # ||||||||| ECHAM5 namelist # cat > namelist.echam << EOF &runctl out_datapath = "${OUT_DIR}/" out_expname = "${EXP}" out_filetype = ${FMT} lresume = ${RERUN} lamip = ${LAMIP} lmlo = ${LMLO} labort = ${LABORT} l_orbvsop87 = ${l_orbvsop87} dt_start = ${YEARI},01,01,12,0,0 dt_stop = ${YEARF},01,01,12,0,0 putdata = ${OI}, '${OU}', 'first', 0 putrerun = ${REOI}, '${REOU}', 'last', 0 ! nproma = ${NPROMA} nproca = ${NPROCA} nprocb = ${NPROCB} / &radctl icfc = ${ICFC} ico2 = ${ICO2} ich4 = ${ICH4} in2o = ${IN2O} co2vmr = ${co2vmr} ch4vmr = ${ch4vmr} n2ovmr = ${n2ovmr} cecc = ${cecc} cobld = ${cobld} clonp = ${clonp} / &wisoctl lwiso=.true. lwiso_rerun=${RERUN} nwiso=3 / EOF # # added 2013.01.29, by Willi Kappler # load modules for the cluster in Tuebingen # . ~/.bashrc # read module path # module load compiler/gnu/4.5 module load mpi/openmpi/1.4.3-gnu-4.5 module load compiler/intel/12.0 module load compiler/intel/ct_4.0 module load system/moab/7.1.1 # # ============================== # ||||||||| Run model executable # mpirun $MODEL # # # Copy monthly rerun files RFILE1=rerun_${EXP}_echam.${YEARB}.${index} cp rerun_${EXP}_echam $RRN_DIR/$RFILE1 RFILE2=rerun_${EXP}_wiso.${YEARB}.${index} cp rerun_${EXP}_wiso $RRN_DIR/$RFILE2 # # Update state and save in text file RERUN=.true. index=$(($index+1)) echo $index count=$(($count+1)) echo $count # cat > ${RUN_DIR}/rerun.txt << EOF $RERUN ${YEARB} $index EOF # done # # # Copy yearly rerun files and reset month counter RFYear=$(($YEARB-1)) # if (($index>12)); then RFILE1=rerun_${EXP}_echam.${RFYear}1231 cp rerun_${EXP}_echam $RRN_DIR/$RFILE1 RFILE2=rerun_${EXP}_wiso.${RFYear}1231 cp rerun_${EXP}_wiso $RRN_DIR/$RFILE2 YEARB=$(($YEARB+1)) RERUN=.true. index=1 fi # cd $RUN_DIR cat > rerun.txt << EOF $RERUN ${YEARB} ${index} EOF # echo $RUN_DIR echo $RUN_FILE # if (($YEARB <= $YEARF)); then # echo "this run was good. ANOTHER!" /usr/local/bin/qsub -o ${RUN_DIR}/ -e ${RUN_DIR}/ ${RUN_DIR}/${RUN_FILE} # fi # # exit script exit