Skip to content

Commit d967b75

Browse files
guidocioniguidocioni
committed
Adapted code on new production server
1 parent a2556a6 commit d967b75

22 files changed

+361
-75
lines changed

‎copy_data.run

+34-31
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
1-
#!/usr/local/bin/bash
1+
#!/usr/bin/bash
22

33
# Folder to be used to download and process data
4-
export MODEL_DATA_FOLDER="/Volumes/Data/temp/icon-eu/"
4+
export MODEL_DATA_FOLDER="/tmp/icon-eu/"
55
export HOME_FOLDER=$(pwd)
6-
export N_CONCUR_PROCESSES=4
6+
export N_CONCUR_PROCESSES=4
77
export NCFTP_BOOKMARK="altervista"
8-
DATA_DOWNLOAD=true
8+
DATA_DOWNLOAD=false
99
DATA_PLOTTING=false
10-
DATA_UPLOAD=false
10+
DATA_UPLOAD=true
1111

1212
##### LOAD functions to download model data
1313
. ./functions_download_dwd.sh
1414
export SHELL=$(type -p bash)
1515
# We need to open many files at the same time
1616
ulimit -Sn 4096
1717
#
18-
source ~/.bash_profile
18+
# source ~/.bash_profile
1919
# acivate our environment
2020
#conda activate nwp-py2
2121
###########################################
@@ -31,19 +31,21 @@ export hour=`date +"%H"`
3131
export hour_no_zero=`date -u +"%-H"`
3232
# note that this date is in UTC which is needed for retrieving the correct run!
3333

34-
if [ "$hour_no_zero" -ge 4 ] && [ "$hour_no_zero" -lt 9 ]
35-
then
36-
run="00"
37-
elif [ "$hour_no_zero" -ge 9 ] && [ "$hour_no_zero" -lt 16 ]
38-
then
39-
run="06"
40-
elif [ "$hour_no_zero" -ge 16 ] && [ "$hour_no_zero" -lt 21 ]
41-
then
42-
run="12"
43-
elif [ "$hour_no_zero" -ge 21 ]
44-
then
45-
run="18"
46-
fi
34+
# if [ "$hour_no_zero" -ge 4 ] && [ "$hour_no_zero" -lt 9 ]
35+
# then
36+
# run="00"
37+
# elif [ "$hour_no_zero" -ge 9 ] && [ "$hour_no_zero" -lt 16 ]
38+
# then
39+
# run="06"
40+
# elif [ "$hour_no_zero" -ge 16 ] && [ "$hour_no_zero" -lt 21 ]
41+
# then
42+
# run="12"
43+
# elif [ "$hour_no_zero" -ge 21 ]
44+
# then
45+
# run="18"
46+
# fi
47+
48+
run="00"
4749

4850
export run
4951

@@ -56,21 +58,22 @@ cd ${MODEL_DATA_FOLDER} || { echo 'Cannot change to DATA folder' ; exit 1; }
5658

5759
if [ "$DATA_DOWNLOAD" = true ]; then
5860
# Remove older files
59-
rm ${MODEL_DATA_FOLDER}*.nc
61+
# rm ${MODEL_DATA_FOLDER}*.nc
6062
rm ${MODEL_DATA_FOLDER}*.grib2
6163

6264
# Invariant
6365
download_invariant_icon_eu
6466

6567
#2-D variables
66-
variables=("T_2M" "TD_2M" "U_10M" "V_10M" "PMSL" "CAPE_ML" "VMAX_10M" "TOT_PREC" \
67-
"CLCL" "CLCH" "CLCT" "SNOWLMT" "HZEROCL" "H_SNOW" "SNOW_GSP"\
68-
"SNOW_CON" "RAIN_GSP" "RAIN_CON" "TMAX_2M" "TMIN_2M" "WW")
69-
parallel -j ${N_CONCUR_PROCESSES} --delay 1 download_merge_2d_variable_icon_eu ::: "${variables[@]}"
68+
# variables=("T_2M" "TD_2M" "U_10M" "V_10M" "PMSL" "CAPE_ML" "VMAX_10M" "TOT_PREC"\
69+
# "CLCL" "CLCH" "CLCT" "SNOWLMT" "HZEROCL" "H_SNOW" "SNOW_GSP"\
70+
# "SNOW_CON" "RAIN_GSP" "RAIN_CON" "TMAX_2M" "TMIN_2M" "WW")
71+
# parallel -j ${N_CONCUR_PROCESSES} --delay 1 download_merge_2d_variable_icon_eu ::: "${variables[@]}"
7072

7173
#3-D variables on pressure levels
72-
variables=("T" "FI" "RELHUM" "U" "V" "CLC")
73-
parallel -j ${N_CONCUR_PROCESSES} --delay 2 download_merge_3d_variable_icon_eu ::: "${variables[@]}"
74+
# variables=("T" "FI" "RELHUM" "U" "V" "CLC")
75+
# variables=("RELHUM" "U")
76+
# parallel -j ${N_CONCUR_PROCESSES} --delay 2 download_merge_3d_variable_icon_eu ::: "${variables[@]}"
7477

7578
#soil levelsls
7679
download_merge_soil_variable_icon_eu W_SO
@@ -88,7 +91,7 @@ if [ "$DATA_PLOTTING" = true ]; then
8891

8992
export QT_QPA_PLATFORM=offscreen # Needed to avoid errors when using Python without display
9093

91-
python plot_meteogram.py Hamburg Pisa Rome Milano Naples Brocken Palermo Toulouse Utrecht Sassari Cheremule
94+
#python plot_meteogram.py Hamburg Pisa Rome Milano Naples Brocken Palermo Toulouse Utrecht Sassari Cheremule
9295

9396
scripts=("plot_winter.py" "plot_cape.py" "plot_convergence.py" "plot_gph_t_500.py" "plot_gph_t_850.py" \
9497
"plot_gph_thetae_850.py" "plot_hsnow.py" "plot_jetstream.py" "plot_pres_t2m_winds10m.py" "plot_rain_acc.py"\
@@ -142,10 +145,10 @@ fi
142145
# SECTION 4 - CLEANING ############################################################
143146

144147
#Remove images locally
145-
rm ${MODEL_DATA_FOLDER}*.png
146-
rm ${MODEL_DATA_FOLDER}de/*.png
147-
rm ${MODEL_DATA_FOLDER}it/*.png
148-
rm ${MODEL_DATA_FOLDER}*.py
148+
# rm ${MODEL_DATA_FOLDER}*.png
149+
# rm ${MODEL_DATA_FOLDER}de/*.png
150+
# rm ${MODEL_DATA_FOLDER}it/*.png
151+
# rm ${MODEL_DATA_FOLDER}*.py
149152

150153
############################################################
151154

‎functions_download_dwd.sh

+28-9
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,41 @@
11
#Given a variable name and year-month-day-run as environmental variables download and merges the variable
22
################################################
3+
# listurls() {
4+
# filename="$1"
5+
# url="$2"
6+
# wget -t 2 --spider -r -nH -np -nv -nd --reject "index.html" --cut-dirs=3 \
7+
# -A $filename.bz2 $url 2>&1\
8+
# | grep -Eo '(http|https)://(.*).bz2'
9+
# }
10+
# export -f listurls
311
listurls() {
412
filename="$1"
513
url="$2"
6-
wget --spider -r -nH -np -nv -nd --reject "index.html" --cut-dirs=3 \
7-
-A $filename.bz2 $url 2>&1\
8-
| grep -Eo '(http|https)://(.*).bz2'
14+
wget -qO- $url | grep -Eoi '<a [^>]+>' | \
15+
grep -Eo 'href="[^\"]+"' | \
16+
grep -Eo $filename | \
17+
xargs -I {} echo "$url"{}
918
}
1019
export -f listurls
1120
#
1221
get_and_extract_one() {
1322
url="$1"
1423
file=`basename $url | sed 's/\.bz2//g'`
15-
wget -q -O - "$url" | bzip2 -dc > "$file"
24+
wget -t 2 -q -O - "$url" | bzip2 -dc > "$file"
1625
}
1726
export -f get_and_extract_one
1827
##############################################
1928
download_merge_2d_variable_icon_eu()
2029
{
2130
filename="icon-eu_europe_regular-lat-lon_single-level_${year}${month}${day}${run}_*_${1}.grib2"
31+
filename_grep="icon-eu_europe_regular-lat-lon_single-level_${year}${month}${day}${run}_(.*)_${1}.grib2.bz2"
2232
url="https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/${1,,}/"
23-
listurls $filename $url | parallel get_and_extract_one {}
33+
echo "folder: ${url}"
34+
echo "files: ${filename}"
35+
#
36+
listurls $filename_grep $url | parallel -j 20 get_and_extract_one {}
37+
# find ${filename} -empty -type f -delete # Remove empty files
38+
sleep 1
2439
cdo -f nc copy -mergetime ${filename} ${1}_${year}${month}${day}${run}_eur.nc
2540
rm ${filename}
2641
}
@@ -29,9 +44,12 @@ export -f download_merge_2d_variable_icon_eu
2944
download_merge_3d_variable_icon_eu()
3045
{
3146
filename="icon-eu_europe_regular-lat-lon_pressure-level_${year}${month}${day}${run}_*_${1}.grib2"
47+
filename_grep="icon-eu_europe_regular-lat-lon_pressure-level_${year}${month}${day}${run}_(.*)_${1}.grib2.bz2"
3248
url="https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/${1,,}/"
33-
#
34-
listurls $filename $url | parallel get_and_extract_one {}
49+
# echo "folder: ${url}"
50+
# echo "files: ${filename}"
51+
listurls $filename_grep $url | parallel -j 20 get_and_extract_one {}
52+
# # find ${filename} -empty -type f -delete # Remove empty files
3553
cdo merge ${filename} ${1}_${year}${month}${day}${run}_eur.grib2
3654
rm ${filename}
3755
cdo -f nc copy ${1}_${year}${month}${day}${run}_eur.grib2 ${1}_${year}${month}${day}${run}_eur.nc
@@ -52,9 +70,10 @@ export -f download_invariant_icon_eu
5270
download_merge_soil_variable_icon_eu()
5371
{
5472
filename="icon-eu_europe_regular-lat-lon_soil-level_${year}${month}${day}${run}_*_3_${1}.grib2"
73+
filename_grep="icon-eu_europe_regular-lat-lon_soil-level_${year}${month}${day}${run}_(.*)_3_${1}.grib2.bz2"
5574
url="https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/${1,,}/"
56-
listurls $filename $url | parallel get_and_extract_one {}
75+
listurls $filename_grep $url | parallel get_and_extract_one {}
5776
cdo -f nc copy -mergetime ${filename} ${1}_${year}${month}${day}${run}_eur.nc
5877
rm ${filename}
5978
}
60-
export -f download_merge_soil_variable_icon_eu
79+
export -f download_merge_soil_variable_icon_eu

‎plotting/meteoindiretta_logo.png

81.8 KB
Loading

‎plotting/plot_cape.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ def plot_files(dates, **args):
114114
an_var = annotation(
115115
args['ax'], 'Convective Available Potential Energy and Winds @ 850 hPa', loc='lower left', fontsize=6)
116116
an_run = annotation_run(args['ax'], args['time'])
117+
logo = add_logo_on_map(ax=args['ax'],
118+
zoom=0.1, pos=(0.95, 0.08))
117119

118120
if first:
119121
plt.colorbar(cs, orientation='horizontal',
@@ -124,7 +126,7 @@ def plot_files(dates, **args):
124126
else:
125127
plt.savefig(filename, **options_savefig)
126128

127-
remove_collections([cs, an_fc, an_var, an_run, cv])
129+
remove_collections([cs, an_fc, an_var, an_run, cv, logo])
128130

129131
first = False
130132

‎plotting/plot_convergence.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ def plot_files(dates, **args):
116116
an_var = annotation(args['ax'], 'Convergence [' + str(args['conv'].units) + ']',
117117
loc='lower left', fontsize=6)
118118
an_run = annotation_run(args['ax'], args['time'])
119+
logo = add_logo_on_map(ax=args['ax'],
120+
zoom=0.1, pos=(0.95, 0.08))
119121

120122
if first:
121123
plt.colorbar(cs, orientation='horizontal', label='Convergence [' + str(
@@ -126,7 +128,7 @@ def plot_files(dates, **args):
126128
else:
127129
plt.savefig(filename, **options_savefig)
128130

129-
remove_collections([cs, an_fc, an_var, an_run, cv])
131+
remove_collections([cs, an_fc, an_var, an_run, cv, logo])
130132

131133
first = False
132134

‎plotting/plot_gph_t_500.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ def plot_files(dates, **args):
100100
an_fc = annotation_forecast(args['ax'],args['time'][i])
101101
an_var = annotation(args['ax'], 'Geopotential height @500hPa [m] and temperature @500hPa' ,loc='lower left', fontsize=6)
102102
an_run = annotation_run(args['ax'], args['time'])
103+
logo = add_logo_on_map(ax=args['ax'],
104+
zoom=0.1, pos=(0.95, 0.08))
103105

104106
if first:
105107
plt.colorbar(cs, orientation='horizontal', label='Temperature [C]', pad=0.03, fraction=0.04)
@@ -109,7 +111,7 @@ def plot_files(dates, **args):
109111
else:
110112
plt.savefig(filename, **options_savefig)
111113

112-
remove_collections([c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels])
114+
remove_collections([c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels, logo])
113115

114116
first = False
115117

‎plotting/plot_gph_t_850.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ def plot_files(dates, **args):
106106
an_var = annotation(
107107
args['ax'], 'Geopotential height @500hPa [m] and temperature @850hPa [C]', loc='lower left', fontsize=6)
108108
an_run = annotation_run(args['ax'], args['time'])
109+
logo = add_logo_on_map(ax=args['ax'],
110+
zoom=0.1, pos=(0.95, 0.08))
109111

110112
if first:
111113
plt.colorbar(cs, orientation='horizontal',
@@ -117,7 +119,7 @@ def plot_files(dates, **args):
117119
plt.savefig(filename, **options_savefig)
118120

119121
remove_collections(
120-
[c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels])
122+
[c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels, logo])
121123

122124
first = False
123125

‎plotting/plot_gph_thetae_850.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ def plot_files(dates, **args):
111111
an_var = annotation(
112112
args['ax'], 'MSLP [hPa] and $\theta_e$ @850hPa [C]', loc='lower left', fontsize=6)
113113
an_run = annotation_run(args['ax'], args['time'])
114+
logo = add_logo_on_map(ax=args['ax'],
115+
zoom=0.1, pos=(0.95, 0.08))
114116

115117
if first:
116118
plt.colorbar(cs, orientation='horizontal',
@@ -122,7 +124,7 @@ def plot_files(dates, **args):
122124
plt.savefig(filename, **options_savefig)
123125

124126
remove_collections([c, cs, labels, an_fc, an_var,
125-
an_run, maxlabels, minlabels])
127+
an_run, maxlabels, minlabels, logo])
126128

127129
first = False
128130

‎plotting/plot_hsnow.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ def plot_files(dates, **args):
103103
an_fc = annotation_forecast(args['ax'],args['time'][i])
104104
an_var = annotation(args['ax'], 'Snow depth change [cm] since run beginning and snow limit [m]' ,loc='lower left', fontsize=6)
105105
an_run = annotation_run(args['ax'], args['time'])
106+
logo = add_logo_on_map(ax=args['ax'],
107+
zoom=0.1, pos=(0.95, 0.08))
106108

107109
if first:
108110
plt.colorbar(cs, orientation='horizontal', label='Snow depth change [m]', pad=0.035, fraction=0.03)
@@ -112,7 +114,7 @@ def plot_files(dates, **args):
112114
else:
113115
plt.savefig(filename, **options_savefig)
114116

115-
remove_collections([c, cs, labels, an_fc, an_var, an_run])
117+
remove_collections([c, cs, labels, an_fc, an_var, an_run, logo])
116118

117119
first = False
118120

‎plotting/plot_jetstream.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ def plot_files(dates, **args):
113113
an_fc = annotation_forecast(args['ax'],args['time'][i])
114114
an_var = annotation(args['ax'], 'Winds and geopotential [m] @300hPa' ,loc='lower left', fontsize=6)
115115
an_run = annotation_run(args['ax'], args['time'])
116+
logo = add_logo_on_map(ax=args['ax'],
117+
zoom=0.1, pos=(0.95, 0.08))
116118

117119
if first:
118120
plt.colorbar(cs, orientation='horizontal', label='Wind [km/h]', pad=0.035, fraction=0.04)
@@ -122,7 +124,7 @@ def plot_files(dates, **args):
122124
else:
123125
plt.savefig(filename, **options_savefig)
124126

125-
remove_collections([c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels])
127+
remove_collections([c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels, logo])
126128

127129
first = False
128130

‎plotting/plot_pres_t2m_winds10m.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ def plot_files(dates, **args):
112112
an_fc = annotation_forecast(args['ax'],args['time'][i])
113113
an_var = annotation(args['ax'], 'MSLP [hPa], Winds@10m and Temperature@2m' ,loc='lower left', fontsize=6)
114114
an_run = annotation_run(args['ax'], args['time'])
115+
logo = add_logo_on_map(ax=args['ax'],
116+
zoom=0.1, pos=(0.95, 0.08))
115117

116118
if first:
117119
plt.colorbar(cs, orientation='horizontal', label='Temperature [C]', pad=0.03, fraction=0.04)
@@ -121,7 +123,7 @@ def plot_files(dates, **args):
121123
else:
122124
plt.savefig(filename, **options_savefig)
123125

124-
remove_collections([cs, cs2, c, labels, labels2, an_fc, an_var, an_run, cv, maxlabels, minlabels])
126+
remove_collections([cs, cs2, c, labels, labels2, an_fc, an_var, an_run, cv, maxlabels, minlabels, logo])
125127

126128
first = False
127129

‎plotting/plot_rain_acc.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ def plot_files(dates, **args):
101101
an_fc = annotation_forecast(args['ax'],args['time'][i])
102102
an_var = annotation(args['ax'], 'Accumulated precipitation and MSLP [hPa]' ,loc='lower left', fontsize=6)
103103
an_run = annotation_run(args['ax'], args['time'])
104+
logo = add_logo_on_map(ax=args['ax'], zoom=0.1, pos=(0.95, 0.08))
104105

105106
if first:
106107
plt.colorbar(cs, orientation='horizontal', label='Accumulated precipitation [mm]', pad=0.035, fraction=0.035)
@@ -110,7 +111,7 @@ def plot_files(dates, **args):
110111
else:
111112
plt.savefig(filename, **options_savefig)
112113

113-
remove_collections([c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels])
114+
remove_collections([c, cs, labels, an_fc, an_var, an_run, maxlabels, minlabels, logo])
114115

115116
first = False
116117

‎plotting/plot_rain_clouds.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ def plot_files(dates, **args):
134134
an_fc = annotation_forecast(args['ax'],args['time'][i])
135135
an_var = annotation(args['ax'], 'Clouds (grey-low, orange-high), rain, snow and MSLP' ,loc='lower left', fontsize=6)
136136
an_run = annotation_run(args['ax'], args['time'])
137+
logo = add_logo_on_map(ax=args['ax'],
138+
zoom=0.1, pos=(0.95, 0.08))
137139

138140
if first:
139141
if args['projection'] == "euratl":
@@ -160,7 +162,7 @@ def plot_files(dates, **args):
160162
plt.savefig(filename, **options_savefig)
161163

162164
remove_collections([c, cs_rain, cs_snow, cs_clouds_low, cs_clouds_high,
163-
labels, an_fc, an_var, an_run, maxlabels, minlabels])
165+
labels, an_fc, an_var, an_run, maxlabels, minlabels, logo])
164166

165167
first = False
166168

‎plotting/plot_soil_moisture.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
def main():
3535
"""In the main function we basically read the files and prepare the variables to be plotted.
3636
This is not included in utils.py as it can change from case to case."""
37-
dset, time, cum_hour = read_dataset(variables='W_SO')
37+
dset, time, cum_hour = read_dataset(variables=['W_SO'])
3838

3939
saturation = xr.open_dataset(soil_saturation_file)['soil_saturation']
4040

@@ -100,6 +100,8 @@ def plot_files(dates, **args):
100100
an_var = annotation(
101101
args['ax'], 'Soil Moisture saturation', loc='lower left', fontsize=6)
102102
an_run = annotation_run(args['ax'], args['time'])
103+
logo = add_logo_on_map(ax=args['ax'],
104+
zoom=0.1, pos=(0.95, 0.08))
103105

104106
if first:
105107
plt.colorbar(cs, orientation='horizontal',
@@ -110,7 +112,7 @@ def plot_files(dates, **args):
110112
else:
111113
plt.savefig(filename, **options_savefig)
112114

113-
remove_collections([cs, an_fc, an_var, an_run])
115+
remove_collections([cs, an_fc, an_var, an_run, logo])
114116

115117
first = False
116118

0 commit comments

Comments
 (0)