import os # ABOUT THIS SCRIPT # ================= # This python script prepares various regional re-analysis products for comparison to model output. # This script: # > unzips files # > combines files and creates means where neccessary # > interpolates to desired resolution # > saves new files # > rezips originals and removes temporary files # Source datasets: # 1. CRU (TS3.21), 2. HAR(HAR30), 3. NARR, 4. UDEL, 5. PRISM # User declarations: # select resolution and variables (check dataset/doc/ESD folder for info incl. available variables and their abbreviations) and append to list # USER DECLARATIONS # ================= RES = "t159" # resolution (for interpolation to it) LVL = "2d" # variable level (surface "2d" or pressure level "3d") - DO NOT CHANGE TIME = "monthly" # time interval for data - DO NOT CHANGE NDATA = 4 # no of datasets # select variables from datasets VAR = [""]*NDATA VAR[0] = ["pet", "pre", "tmn", "tmx", "tmp", "wet"] VAR[1] = ["potevap", "prcp", "slp", "t2", "u10", "v10", "ws10"] VAR[2] = ["air.2m", "apcp", "lhtfl", "pevap", "prate", "prmsl", "uwnd.10m", "vwnd.10m"] VAR[3] = ["air", "precip"] #VAR[4] = ["ppt", "tmin", "tmean", "tmax"] # data paths DATA_PATH = "/esd/esd/data/atmospheric_data/reanalysis/regional/" IPATH = [""]*NDATA IPATH[0] = DATA_PATH+"CRU/CRU_TS_3.21/data/"+TIME+"/"+LVL+"/" IPATH[1] = DATA_PATH+"HAR/HAR30/data/"+TIME+"/"+LVL+"/" IPATH[2] = DATA_PATH+"NARR/data/"+TIME+"/"+LVL+"/" IPATH[3] = DATA_PATH+"UDEL/V3.01/data/"+TIME+"/"+LVL+"/" #IPATH[4] = DATA_PATH+"PRISM/data/"+TIME+"/"+LVL+"/" OPATH = “YOUR/OUTPUT/PATH“ #"cru_ts3.21.1901.2012."+VAR+".dat.nc.gz" #"har_d30km_m_"+LVL+"_"+VAR+"_"+YEAR+".nc" #VAR+"mon.mean.nc" #"PRISM_"+VAR+"stable_4kmM2_"+YEAR+"_all_bil.zip" # consistent names #VAR_CNAME = "pre", "tsf" #VAR_NAME = [""]*len(VAR_CNAME) #VAR_NAME[0] = ["pre", "prcp", "apcp", "ppt"] #VAR_NAME[1] = ["tmp", "t2", "air.2m", "tmean"] # start and end years of datasets YEARI = [1901,2000,1979,1900,1979] # initial year CRU, HAR, NARR, UDEL, PRISM YEARF = [2012,2012,2013,2010,2012] # final year CRU, HAR, NARR, UDEL, PRISM # var name in file (if deviates from var name in file name, NARR) nVarDev = 3 VAR2 = [""]*nVarDev VAR2[0]=["air.2m","air"] VAR2[1]=["uwnd.10m","uwnd"] VAR2[2]=["vwnd.10m","vwnd"] # INSTRUCTIONS # ============ os.environ["res"] = RES # loop through datasets print print "/////////////////////////" print "PREPARING REG. REANALYSES" print "/////////////////////////" print print "Desired Resolution: ",RES for dataset in range(0,len(IPATH)): # go through all datasets print print " DATASET: ", IPATH[dataset] print " ============================================================" for variable in range (0,len(VAR[dataset])): # go through selected variables for each dataset print print " VARIABLE: ", VAR[dataset][variable] print " ---------------" # # CRU TS3.21 if dataset == 0: OFILE = "cru_"+RES+"_"+VAR[dataset][variable]+"_lterm.nc" os.environ["ipath"] = IPATH[dataset]+VAR[dataset][variable]+"/" # path os.environ["ifile"] = "cru_ts3.21."+str(YEARI[dataset])+"."+str(YEARF[dataset])+"."+VAR[dataset][variable]+".dat.nc" # file name os.environ["opath"] = OPATH # outpath os.environ["ofile"] = OFILE # outfile nT = (YEARF[dataset]-YEARI[dataset]+1)*12 # no. of timesteps in dataset # unzip gzipped file print " > unzipping ", "cru_ts3.21.1901.2012."+VAR[dataset][variable]+".dat.nc.gz" os.system("gzip -d -f ${ipath}${ifile}.gz") # construct long term mean print " > construct long term mean" for month in range (0,12): # month loop os.environ["moni"] = str(month) # start month os.environ["monf"] = str(nT-12+month) # final month os.environ["nmon"] = str(nT) # total number of months os.environ["mon"] = str(month) # month if month < 10: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfile name else: os.environ["tfile"] = str(month)+".nc" # temporary outfile name os.system("ncra -d time,${mon},${monf},12 ${ipath}${ifile} ${ipath}${tfile}") # extract months and construct long term mean for each # combine to one file print " > combine in one file" os.environ["tfile"] = "00.nc" # 1st temporary outfile in series os.environ["pfile"] = "tmp_lterm.nc" # tempoary outfile os.system("ncrcat -n 12,2,1 ${ipath}${tfile} ${opath}${pfile}") # interpolate to specified grid print " > Interpolate to ", RES, "grid" os.system("cdo remapbil,${res}grid ${opath}${pfile} ${opath}${ofile}") # interpolate and write in FINAL outfile # write in file print " > data written in:", OPATH+OFILE # recreate original gzipped file print " > re-zipping ", "cru_ts3.21.1901.2012."+VAR[dataset][variable]+".dat.nc" os.system("gzip ${ipath}${ifile}") # remove temporary files print " > cleaning up temporary files" os.system("'rm' ${opath}${pfile}") for month in range (0,12): # month loop if month < 10: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfiles else: os.environ["tfile"] = str(month)+".nc" # temporary outfiles os.system("'rm' ${ipath}${tfile}") # HAR (HAR30) if dataset == 1: OFILE = "har_"+RES+"_"+VAR[dataset][variable]+"_lterm.nc" os.environ["ipath"] = IPATH[dataset]+VAR[dataset][variable]+"/" # path os.environ["opath"] = OPATH # outpath os.environ["ofile"] = OFILE # outfile nT = (YEARF[dataset]-YEARI[dataset]+1) # no. of years in dataset # prepare input files print " > prepare input" for YEAR in range(YEARI[dataset],YEARF[dataset]+1): os.environ["ifile"] = "har_d30km_m_2d_"+VAR[dataset][variable]+"_"+str(YEAR)+".nc" # infile name if YEAR-YEARI[dataset] < 10: os.environ["tfile"] = "0"+str(YEAR-YEARI[dataset])+".nc" # temporary outfile name else: os.environ["tfile"] = str(YEAR-YEARI[dataset])+".nc" # temporary outfile name os.system("nccopy -k 1 ${ipath}${ifile} ${ipath}${tfile}") # copy to files with classic netCDF format ("-k 1" option) # construct long term means for month in range (0,12): # month loop os.environ["nyr"] = str(nT-1) # total number of years (ignore 1st cause is incomplete) os.environ["mon"] = str(month) # total number of years (ignore 1st cause is incomplete) os.environ["tmp"] = "01.nc" # start from year 2, cause 1st is incomplete if month < 10: os.environ["tfile"] = "00"+str(month)+".nc" # temporary outfile name else: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfile name os.system("ncra -d time,${mon},,12 -n ${nyr},2,1 ${ipath}${tmp} ${ipath}${tfile}") # average over month and write in separate files # combine to one file print " > combine in one file" os.environ["tfile"] = "000.nc" # 1st temporary outfile in series os.environ["pfile"] = "tmp_lterm.nc" # tempoary outfile os.system("ncrcat -n 12,3,1 ${ipath}${tfile} ${opath}${pfile}") # interpolate to specified grid print " > Interpolate to ", RES, "grid" os.system("cdo remapbil,${res}grid ${opath}${pfile} ${opath}${ofile}") # interpolate and write in FINAL outfile # write in file print " > data written in:", OPATH+OFILE # remove temporary files print " > cleaning up temporary files" os.system("'rm' ${opath}${pfile}") for YEAR in range(YEARI[dataset],YEARF[dataset]+1): os.environ["ifile"] = "har_d30km_m_2d_"+VAR[dataset][variable]+"_"+str(YEAR)+".nc" # infile name if YEAR-YEARI[dataset] < 10: os.environ["tfile"] = "0"+str(YEAR-YEARI[dataset])+".nc" # temporary outfile name else: os.environ["tfile"] = str(YEAR-YEARI[dataset])+".nc" # temporary outfile name os.system("'rm' ${ipath}${tfile}") for month in range (0,12): # month loop if month < 10: os.environ["tfile"] = "00"+str(month)+".nc" # temporary outfiles else: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfiles os.system("'rm' ${ipath}${tfile}") # NARR if dataset == 2: OFILE = "narr_"+RES+"_"+VAR[dataset][variable]+"_lterm.nc" os.environ["ipath"] = IPATH[dataset] # path os.environ["ifile"] = VAR[dataset][variable]+".mon.mean.nc" # file name os.environ["opath"] = OPATH # outpath os.environ["ofile"] = OFILE # outfile nT = (YEARF[dataset]-YEARI[dataset]+1)*12 # no. of timesteps in dataset # construct long term mean print " > construct long term mean" for month in range (0,12): # month loop os.environ["moni"] = str(month) # start month os.environ["monf"] = str(nT-12+month) # final month os.environ["nmon"] = str(nT) # total number of months os.environ["mon"] = str(month) # month if month < 10: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfile name else: os.environ["tfile"] = str(month)+".nc" # temporary outfile name os.system("ncra -d time,${mon},${monf},12 ${ipath}${ifile} ${ipath}${tfile}") # extract months and construct long term mean for each # combine to one file print " > combine in one file" os.environ["tfile"] = "00.nc" # 1st temporary outfile in series os.environ["pfile"] = "tmp_lterm.nc" # tempoary outfile os.environ["qfile"] = "tmp_lterm2.nc" # tempoary outfile os.system("ncrcat -n 12,2,1 ${ipath}${tfile} ${opath}${pfile}") # rewrite (to allow CDO to remap) os.environ["ivar"] = VAR[dataset][variable] for varDev in range (0,nVarDev): if VAR[dataset][variable] == VAR2[varDev][0]: os.environ["ivar"] = VAR2[varDev][1] os.system("ncl /data/docs2/smutz/data_small/EXTREME/scripts/data_processing/reg_reanalyses/narr_rw.ncl") # call NCL script # interpolate to specified grid print " > Interpolate to ", RES, "grid" os.system("cdo remapcon,${res}grid ${opath}${qfile} ${opath}${ofile}") # interpolate and write in FINAL outfile # write in file print " > data written in:", OPATH+OFILE # remove temporary files print " > cleaning up temporary files" os.system("'rm' ${opath}${pfile}") os.system("'rm' ${opath}${qfile}") for month in range (0,12): # month loop if month < 10: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfiles else: os.environ["tfile"] = str(month)+".nc" # temporary outfiles os.system("'rm' ${ipath}${tfile}") # UDEL V3.01 if dataset == 3: OFILE = "udel_"+RES+"_"+VAR[dataset][variable]+"_lterm.nc" os.environ["ipath"] = IPATH[dataset] # path if VAR[dataset][variable] == "precip": os.environ["ifile"] = VAR[dataset][variable]+".mon.total.v301.nc" # file name else: os.environ["ifile"] = VAR[dataset][variable]+".mon.mean.v301.nc" # file name os.environ["opath"] = OPATH # outpath os.environ["ofile"] = OFILE # outfile nT = (YEARF[dataset]-YEARI[dataset]+1)*12 # no. of timesteps in dataset # construct long term mean print " > construct long term mean" for month in range (0,12): # month loop os.environ["moni"] = str(month) # start month os.environ["monf"] = str(nT-12+month) # final month os.environ["nmon"] = str(nT) # total number of months os.environ["mon"] = str(month) # month if month < 10: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfile name else: os.environ["tfile"] = str(month)+".nc" # temporary outfile name os.system("ncra -d time,${mon},${monf},12 ${ipath}${ifile} ${ipath}${tfile}") # extract months and construct long term mean for each # combine to one file print " > combine in one file" os.environ["tfile"] = "00.nc" # 1st temporary outfile in series os.environ["pfile"] = "tmp_lterm.nc" # tempoary outfile os.system("ncrcat -n 12,2,1 ${ipath}${tfile} ${opath}${pfile}") # interpolate to specified grid print " > Interpolate to ", RES, "grid" os.system("cdo remapbil,${res}grid ${opath}${pfile} ${opath}${ofile}") # interpolate and write in FINAL outfile # write in file print " > data written in:", OPATH+OFILE # remove temporary files print " > cleaning up temporary files" os.system("'rm' ${opath}${pfile}") for month in range (0,12): # month loop if month < 10: os.environ["tfile"] = "0"+str(month)+".nc" # temporary outfiles else: os.environ["tfile"] = str(month)+".nc" # temporary outfiles os.system("'rm' ${ipath}${tfile}")