#!/bin/bash # # Sebastian Mutz, Jingmin Li, Todd Ehlers # Geowissenschaftliche Fakultät der Universität Tübingen # # For questions regarding this specific script, contact: # sebastian.mutz@uni-tuebingen.de, jingmin.li@uni-tuebingen.de # todd.ehlers@uni-tuebingen.de # # ################################################################## # About this script: # # # # 1. Scheduler options at the top (Simple Linux Utility # # for Resource Management (SLURM)) # # 2. All important options are defined at the top of the run # # script. There is no need for the user to look beyond these. # # 3. Generates a directory structure in your work folder for model # # output as well as making copies of the model, input, etc. # # 4. Restarts itself automatically. Run time only limited by wall # # clock limit and your defined final run year. # # 5. Makes backups of the rerun files. # # 6. Allows manual restarts from a time step of your choice # # (rerun.txt in the same folder has to be adjusted for that) # # 7. Makes backups of your input files, run scripts, configuration # # files and model executable for experiment reproducability. # # ################################################################## # # # # # # # # ################### BEGIN USER DECLARATIONS ##################### # ------------------------------------------------------------- # # # # ============================================================== # |||| Simple Linux Utility for Resource Management (SLURM) |||| # ============================================================== # # Options: # #SBATCH --job-name=2015_e050 # Specify job name #SBATCH --partition=compute # Specify partition name #SBATCH --nodes=4 # Specify number of nodes #SBATCH --ntasks-per-node=20 # Specify number of tasks on each node #SBATCH --time=08:00:00 # Set a limit on the total run time #SBATCH --mail-type=FAIL # Notify user by email in case of job failure #SBATCH --mail-user=sebastian.mutz@uni-tuebingen.de # Set your e-mail address #SBATCH --account=bb0938 # Charge resources on this project account #SBATCH --output=run.o%j # File name for standard output #SBATCH --error=run.e%j # File name for standard error output # # ============================== # |||| Environment Settings |||| # ============================== # # Settings for Open MPI library #export OMPI_MCA_pml=cm #export OMPI_MCA_mtl=mxm #export OMPI_MCA_mtl_mxm_np=0 # Disable GHC algorithm for collective communication #export OMPI_MCA_coll=^ghc # # set account variables export SLURM_ACCOUNT=bb0938 export SBATCH_ACCOUNT=$SLURM_ACCOUNT export SALLOC_ACCOUNT=$SLURM_ACCOUNT # # Manually add lib paths for Open MPI and netCDF LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/netcdf/netcdf_c-4.3.2-intel14/lib:/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.2-intel14/lib:/sw/rhel6-x64/mpi/openmpi-1.8.4-intel14/lib export LD_LIBRARY_PATH # # # # # # # =========================================================== # ||||||||| Job script for running ECHAM5 on MISTRAL |||||||| # =========================================================== # # If set and a command has a non-zero exit status, ERR trap is # executed and programme exits set -ex # # ============================ # ||||||||| Basic declarations # # This is the section for user defined options for the model runs. # It includes anything from your user and experiment name to green # house gas concentrations. For more options, look in the namelist # file in your echam/doc directory. Give the option variable a value # in this section and make sure it's written in echam.namelists # under the correct heading (eg. runctl; see ECHAM namelists section), # so it's passed on to the model. # # ---------------------------- # ------- Usr and Dir -------- # # The path names are generated automatically from the options below. # Please stick with directory structure of your echam5 distribution # to avoid any complications. # USR=b380055 # user name MODD=echam5-wiso.v2.3_150511 # model directory relative to home RUND=e050 # subdirectoty in ~/model/run/ EXEC=MISTRAL_echam5-wiso.2.3_20151006 # name of the executable in ~/model/bin/ INPUT=T159_sam75_MioIcefull # # ---------------------------- # ------- Job options -------- # EXP=e050 # experiment identifier EXPD=dkrz-mistral_e5w2.3_PI_t159l31.1d # extended experiment name. this will be used to name the output folder RUN_FILE="e050_dkrz-mistral_e5w2.3_t159" # name of the run script you are currently viewing RE_FILE="rerun.txt" # file that stores last date of completed job iteration. used by re-submitting script # LABORT=.true. # stop at the end of the rerun chain? NCPUS=80 # number of CPUs for parallel run NPROCA=8 # number of CPUs to process task A NPROCB=10 # number of CPUs to process task B NTHREADS=1 # number of threads. 1 if no hybrid run #NPROMA=16 # blocking length/vector length # # ---------------------------- # ------- Model options ------ # # Rerun options REI=3 # re-submit time within wallcolock # # Time related options YEARI=1000 # start year of run (do not change for restart runs) YEARF=1021 # final year of run (do not change for restart runs) # Put all years in year variable. not neccessary if model does not rely on time dependent input. #years="1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966" # # Physics and dynamics RES=159 # model resolution LEVELS=31 # number of levels LAMIP=.false. # true if run with amip SST LMLO=.false. # run with slab ocean? # l_orbvsop87=.false. # .false. for paleo run ICFC=0 # no cfc ICO2=2 # 0 for no CO2, 2 for constant CO2 level = CO2VMR = 348.E-06 ICH4=2 # 0 for no CH4, 2 for constant CH4 level = CH4VMR = 1.65E-06 IN2O=2 # 0 for no N2O, 2 for constant N2O level = N2OVMR = 306.E-09 # co2vmr=278.E-06 # PI co2=280ppm ch4vmr=760.E-09 # PI ch4=760ppb n2ovmr=270.E-09 # PI n2o=270ppb # cecc=0.016804 # PI cecc=0.016804 cobld=23.4725 # PI cobld=23.4725 clonp=278.734 # PI clonp=278.734 # # Output options FMT=2 # output format 1=GRIB, 2=netCDF OU=days # output time interval unit OI=1 # output time interval REOU=months # output time interval unit for rerun files REOI=1 # output time interval for rerun files # # Note: needs a text file "rerun.txt" in the same folder as the run file. # For the start of the run, write in it: # ".false. [start year +1] [start month]" # .false. refers to the fact it's not a restart. This and the time step # will be changed automatically before a restart. # # ------------------------------------------------------------- # #################### END USER DECLARATION ####################### # # # # # # # =========================== # ||||||||| Path declarations # # Path to work directory WRK_DIR=/work/${SLURM_ACCOUNT}/ECHAM5_wiso # # Path to model folder MOD_DIR=/pf/b/${USR}/models/ECHAM/${MODD} # # Path to job script RUN_DIR=${MOD_DIR}/run/${RUND} # # Path to experiment directory EXP_DIR=${WRK_DIR}/experiments/${EXP}_${EXPD} # # Path to output directory OUT_DIR=${EXP_DIR}/output_raw # # Path to directory storing rerun files RRN_DIR=${OUT_DIR}/rerun # # Path to initial data INI_DIR=${WRK_DIR}/input/${INPUT} # # Path to amip sst BND_DIR=${INI_DIR}/amip2 # # Path to ocean delta values D18_DIR=${MOD_DIR}/iniwiso # # Path to model executable + executable name MODEL=${MOD_DIR}/bin/${EXEC} # # ------------------------------------------------- # Paths in source directory. In src/ everything run # specific is stored. # # Path to source directory S_SRC_DIR=${EXP_DIR}/src # # Path to input files S_INI_DIR=${S_SRC_DIR}/input # # Path to script directory S_SPT_DIR=${S_SRC_DIR}/scripts # # Path to model directory (run file will also be copied here) S_MOD_DIR=${S_SRC_DIR}/model # # ------------------------------------ # Paths to post processing directories # # Path to processed output directory P_OUT_DIR=${EXP_DIR}/output_processed # # Paths to script directories P_SPT_DIR=${EXP_DIR}/scripts P_NCO_DIR=${P_SPT_DIR}/nco # for nco scripts P_NCL_DIR=${P_SPT_DIR}/ncl # for ncl scripts P_GMT_DIR=${P_SPT_DIR}/gmt # for gmt scripts P_FOR_DIR=${P_SPT_DIR}/for # for fortran scripts # # Path to plot directories P_PLT_DIR=${EXP_DIR}/plots P_DFT_DIR=${P_PLT_DIR}/drafts P_FIN_DIR=${P_PLT_DIR}/final # read RERUN YEARB index< ${RUN_DIR}/rerun.txt echo $RERUN #rerun switch, .true. for rerun, .false. for initial run echo $YEARB #final year of this job, one simulation is consist of many jobs. echo $index # # # # # # # ====================================== # ||||||||| Generate directory structure # # Generate directories for initial run if [[ $RERUN == .false. ]]; then # # Make output directories mkdir -p ${EXP_DIR} mkdir -p ${OUT_DIR} && mkdir -p ${RRN_DIR} # # Make source directories mkdir -p ${S_SRC_DIR} mkdir -p ${S_INI_DIR} && mkdir -p ${S_MOD_DIR} && mkdir -p ${S_SPT_DIR} mkdir -p ${S_MOD_DIR}/${MODD} && mkdir -p ${S_MOD_DIR}/${MODD}/run mkdir -p ${S_MOD_DIR}/${MODD}/bin && mkdir -p ${S_MOD_DIR}/${MODD}/config # # Make post processing directories mkdir -p ${P_OUT_DIR} # processed output directory mkdir -p ${P_NCO_DIR} && mkdir -p ${P_NCL_DIR} # scripts directories mkdir -p ${P_GMT_DIR} && mkdir -p ${P_FOR_DIR} # scripts directories mkdir -p ${P_DFT_DIR} && mkdir -p ${P_FIN_DIR} # plots directories fi # # ------------------------------------------- # Save relevant files in the source directory # if [[ $RERUN == .false. ]]; then # save input files cp -r ${INI_DIR} ${S_INI_DIR} #save model files cp -r ${RUN_DIR} ${S_MOD_DIR}/${MODD}/run # save run scripts cp ${MODEL} ${S_MOD_DIR}/${MODD}/bin # save model executable cp ${MOD_DIR}/config/mh* ${S_MOD_DIR}/${MODD}/config # save configuration file(s) fi # # ================================ # ||||||||| Specification of files # # ----------------------------------------------- # Change directory before creating symbolic links. # All need to be in the output directory. cd $OUT_DIR # rm -f unit.?? sst* ice* hdpara.nc hdstart.nc rrtadata greenhouse_gases_A1B.nc # # Create symbolic links for various input files ln -sf ${INI_DIR}/T${RES}L${LEVELS}_jan_spec.nc unit.23 ln -sf ${INI_DIR}/T${RES}_jan_surf_miocene_icefull278ppm_move_austria_new_geosp_veg.nc unit.24 ln -sf ${INI_DIR}/T${RES}_O3clim2.nc unit.21 ln -sf ${INI_DIR}/T31GR30_VLTCLIM_to_t159_remapbil.nc unit.90 ln -sf ${INI_DIR}/T31GR30_VGRATCLIM_to_t159_remapbil.nc unit.91 ln -sf ${INI_DIR}/T${RES}_TSLCLIM2.nc unit.92 # ln -sf ${INI_DIR}/surrta_data rrtadata #ln -sf ${INI_DIR}/hdpara.nc hdpara.nc #ln -sf ${INI_DIR}/hdstart.nc hdstart.nc # # Create symbolic links for delta values ln -sf ${D18_DIR}/T${RES}_wisosw_d.nc unit.25 # Create symbolic links for SSTs ln -sf ${INI_DIR}/SST_ICE_FULL_278ppm_t159_miocene_fill_land.nc unit.20 ln -sf ${INI_DIR}/SIC_ICE_FULL_278ppm_t159_miocene_fill_land.nc unit.96 # #for year in $years ; do #ln -sf ${BND_DIR}/T${RES}_amip2sst_${year}.nc sst${year} #ln -sf ${BND_DIR}/T${RES}_amip2sic_${year}.nc ice${year} #done # # # # # # # ======================= # ||||||||| Preparing run # # #################################### # ########### Restart loop ########### # count=1 while (($count<=$REI)); do # # ========================= # ||||||||| ECHAM5 namelist # cat > namelist.echam << EOF &runctl out_datapath = "${OUT_DIR}/" out_expname = "${EXP}" out_filetype = ${FMT} lresume = ${RERUN} lamip = ${LAMIP} lmlo = ${LMLO} labort = ${LABORT} l_orbvsop87 = ${l_orbvsop87} dt_start = ${YEARI},01,01,12,0,0 dt_stop = ${YEARF},01,01,12,0,0 putdata = ${OI}, '${OU}', 'first', 0 putrerun = ${REOI}, '${REOU}', 'last', 0 ! nproma = ${NPROMA} nproca = ${NPROCA} nprocb = ${NPROCB} delta_time =120. / &radctl icfc = ${ICFC} ico2 = ${ICO2} ich4 = ${ICH4} in2o = ${IN2O} co2vmr = ${co2vmr} ! ch4vmr = ${ch4vmr} ! n2ovmr = ${n2ovmr} cecc = ${cecc} cobld = ${cobld} clonp = ${clonp} / &wisoctl lwiso=.true. lwiso_rerun=${RERUN} nwiso=3 / EOF # # ============================== # ||||||||| Run model executable # # Use srun (not mpirun or mpiexec) command to launch programs compiled with any MPI library srun -l --cpu_bind=cores --distribution=block:cyclic $MODEL # # # Copy monthly rerun files RFILE1=rerun_${EXP}_echam.${YEARB}.${index} cp rerun_${EXP}_echam $RRN_DIR/$RFILE1 RFILE2=rerun_${EXP}_wiso.${YEARB}.${index} cp rerun_${EXP}_wiso $RRN_DIR/$RFILE2 # # Update state and save in text file RERUN=.true. index=$(($index+1)) echo $index count=$(($count+1)) echo $count # cat > ${RUN_DIR}/rerun.txt << EOF $RERUN ${YEARB} $index EOF # done # # # Copy yearly rerun files and reset month counter RFYear=$(($YEARB-1)) # if (($index>12)); then RFILE1=rerun_${EXP}_echam.${RFYear}1231 cp rerun_${EXP}_echam $RRN_DIR/$RFILE1 RFILE2=rerun_${EXP}_wiso.${RFYear}1231 cp rerun_${EXP}_wiso $RRN_DIR/$RFILE2 YEARB=$(($YEARB+1)) RERUN=.true. index=1 fi # cd $RUN_DIR cat > rerun.txt << EOF $RERUN ${YEARB} ${index} EOF # echo $RUN_DIR echo $RUN_FILE # if (($YEARB <= $YEARF)); then # echo "this run was good. ANOTHER!" /usr/bin/sbatch ${RUN_DIR}/${RUN_FILE} # fi # # exit script exit