ADD: Data Processing Update

getterk96 · getterk96 · commit a4bc71f9df96 · 2019-08-24T23:14:22.000+08:00
diff --git a/data_exploration/python_data_handling.ipynb b/data_exploration/python_data_handling.ipynb
@@ -40,10 +40,10 @@
    "source": [
     "from netCDF4 import Dataset\n",
     "# Define directory where interpolated files are stored.\n",
-    "DATA_DIR = '/project/meteo/w2w/C7/ppnn_data/'   # At LMU\n",
+    "DATA_DIR = '/data/gaojinghan/EC_esemble/temperature2m/'   # At LMU\n",
     "# DATA_DIR = '/Users/stephanrasp/repositories/ppnn/data/'  # Mac\n",
     "# Define file name\n",
-    "fn = 'data_interpolated.nc'"
+    "fn = 'MD_ECMF_PD_temperature2m_IT_2019041600_FH_132_VT_2019042112.nc'"
    ]
   },
   {
@@ -55,7 +55,7 @@
    "outputs": [],
    "source": [
     "# Open NetCDF rootgroup \n",
-    "rg = Dataset(DATA_DIR + 'data_interpolated.nc')"
+    "rg = Dataset(DATA_DIR + fn)"
    ]
   },
   {
@@ -1018,8 +1018,17 @@
    "toc_position": {},
    "toc_section_display": "block",
    "toc_window_display": false
+  },
+  "pycharm": {
+   "stem_cell": {
+    "cell_type": "raw",
+    "source": [],
+    "metadata": {
+     "collapsed": false
+    }
+   }
   }
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_geo_data.py b/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_geo_data.py
@@ -25,12 +25,12 @@
 server = ECMWFDataServer()
     
 def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2007,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2007,2017)]   
+    date1 = [str(i) + "-01-01" for i in range(2007,2017)]
+    date2 = [str(i) + "-12-31" for i in range(2007,2017)]
     dates = date1
     for j in range(0,10):
         dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/auxiliary/" 
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/auxiliary/"
     for date in dates:
         target = data_dir + "ecmwf_aux_geo_" + date[:4] + ".grib"
         tigge_request(date, target)
@@ -56,6 +56,6 @@ def tigge_request(date, target):
     })
  
 if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50'])
     retrieve_tigge_data()
 
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_pl500_data.py b/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_pl500_data.py
@@ -25,12 +25,12 @@
 server = ECMWFDataServer()
     
 def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2007,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2007,2017)]   
+    date1 = [str(i) + "-01-01" for i in range(2007,2017)] 
+    date2 = [str(i) + "-12-31" for i in range(2007,2017)]   
     dates = date1
     for j in range(0,10):
         dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/auxiliary/" 
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/auxiliary/"
     for date in dates:
         target = data_dir + "ecmwf_aux_pl500_" + date[:4] + ".grib"
         tigge_request(date, target)
@@ -58,5 +58,5 @@ def tigge_request(date, target):
     })
  
 if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50']) 
     retrieve_tigge_data()
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_pl850_data.py b/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_pl850_data.py
@@ -25,12 +25,12 @@
 server = ECMWFDataServer()
     
 def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2016,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2016,2017)]   
+    date1 = [str(i) + "-01-01" for i in range(2016,2017)]
+    date2 = [str(i) + "-12-31" for i in range(2016,2017)]
     dates = date1
-    for j in range(0,10):
+    for j in range(0,1):
         dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/auxiliary/" 
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/auxiliary/"
     for date in dates:
         target = data_dir + "ecmwf_aux_pl850_" + date[:4] + ".grib"
         tigge_request(date, target)
@@ -58,5 +58,5 @@ def tigge_request(date, target):
     })
  
 if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50']) 
     retrieve_tigge_data()
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_surface_data.py b/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_surface_data.py
@@ -25,12 +25,12 @@
 server = ECMWFDataServer()
     
 def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2007,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2007,2017)]   
+    date1 = [str(i) + "-01-01" for i in range(2007,2017)] 
+    date2 = [str(i) + "-12-31" for i in range(2007,2017)]   
     dates = date1
     for j in range(0,10):
         dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/auxiliary/" 
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/auxiliary/"
     for date in dates:
         target = data_dir + "ecmwf_aux_surface_" + date[:4] + ".grib"
         tigge_request(date, target)
@@ -57,5 +57,5 @@ def tigge_request(date, target):
     })
  
 if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50']) 
     retrieve_tigge_data()
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_surface_data_more.py b/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_surface_data_more.py
@@ -28,12 +28,12 @@
 server = ECMWFDataServer()
     
 def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2014,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2014,2017)]   
+    date1 = [str(i) + "-01-01" for i in range(2014,2017)] 
+    date2 = [str(i) + "-12-31" for i in range(2014,2017)]   
     dates = date1
-    for j in range(0,10):
+    for j in range(0,3):
         dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/auxiliary/" 
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/auxiliary/"
     for date in dates:
         target = data_dir + "ecmwf_aux_surface_more_" + date[:4] + ".grib"
         tigge_request(date, target)
@@ -60,5 +60,5 @@ def tigge_request(date, target):
     })
  
 if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50']) 
     retrieve_tigge_data()
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_surface_data_more_rest.py b/data_retrieval/forecasts/retrieve_ecmwf_auxiliary_surface_data_more_rest.py
@@ -28,12 +28,12 @@
 server = ECMWFDataServer()
     
 def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2016,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2016,2017)]   
+    date1 = [str(i) + "-01-01" for i in range(2016,2017)] 
+    date2 = [str(i) + "-12-31" for i in range(2016,2017)]   
     dates = date1
     for j in range(0,1):
         dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/auxiliary/" 
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/auxiliary/"
     for date in dates:
         target = data_dir + "ecmwf_aux_surface_more_" + date[:4] + ".grib"
         tigge_request(date, target)
@@ -60,5 +60,5 @@ def tigge_request(date, target):
     })
  
 if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50']) 
     retrieve_tigge_data()
diff --git a/data_retrieval/forecasts/retrieve_ecmwf_fc_data.py b/data_retrieval/forecasts/retrieve_ecmwf_fc_data.py
@@ -1,57 +1,57 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Thu Aug 24 10:44:53 2017
-
-@author: sebastian
-"""
-
-## retrieve ECMWF forecast data, based on example from
-## https://software.ecmwf.int/wiki/display/WEBAPI/TIGGE+retrieval+efficiency
-
-# ECMWF forecasts from TIGGE data set: 
-#   T2M fields 
-#   all available full years, 2007-2016
-#   init time 00 UTC
-#   36/48 h ahead forecasts (= valid at 12 UTC and 00 UTC)
-#   0.5° resolution
-#   area: -10E, 30E; 30N, 70N (large part of Europe centered around Germany)
-    
-#!/usr/bin/env python
-from ecmwfapi import ECMWFDataServer
-server = ECMWFDataServer()
-    
-def retrieve_tigge_data():
-    date1 = [str(i) + "-01-01" for i in xrange(2007,2017)] 
-    date2 = [str(i) + "-12-31" for i in xrange(2007,2017)]   
-    dates = date1
-    for j in range(0,10):
-        dates[j] = date1[j] + "/to/" + date2[j]
-    data_dir = "/media/sebastian/Elements/Postproc_NN/data/forecasts/" 
-    for date in dates:
-        target = data_dir + "ecmwf_t2m_" + date[:4] + ".grib"
-        tigge_request(date, target)
-          
-def tigge_request(date, target):
-    '''
-       A TIGGE request for ECMWF perturbed forecasts of T2M.
-    '''
-    server.retrieve({
-        'origin'    : "ecmf",
-        'levtype'   : "sfc",
-        'number'    : mem_numbers,
-        'expver'    : "prod",
-        'dataset'   : "tigge",
-        'step'      : "36/48",
-        'grid'      : "0.5/0.5",
-        'param'     : "167",
-        'area'      : "70/-10/30/30",
-        'time'      : "00",
-        'date'      : date,
-        'type'      : "pf",
-        'class'     : "ti",
-        'target'    : target,
-    })
- 
-if __name__ == '__main__':
-    mem_numbers = ''.join([''.join([str(i) + "/" for i in xrange(1,50)]),'50']) 
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 24 10:44:53 2017
+
+@author: sebastian
+"""
+
+## retrieve ECMWF forecast data, based on example from
+## https://software.ecmwf.int/wiki/display/WEBAPI/TIGGE+retrieval+efficiency
+
+# ECMWF forecasts from TIGGE data set: 
+#   T2M fields 
+#   all available full years, 2007-2016
+#   init time 00 UTC
+#   36/48 h ahead forecasts (= valid at 12 UTC and 00 UTC)
+#   0.5° resolution
+#   area: -10E, 30E; 30N, 70N (large part of Europe centered around Germany)
+    
+#!/usr/bin/env python
+from ecmwfapi import ECMWFDataServer
+server = ECMWFDataServer()
+    
+def retrieve_tigge_data():
+    date1 = [str(i) + "-01-01" for i in range(2007,2017)]
+    date2 = [str(i) + "-12-31" for i in range(2007,2017)]
+    dates = date1
+    for j in range(0,10):
+        dates[j] = date1[j] + "/to/" + date2[j]
+    data_dir = "/home/gaojinghan/ppnn/data/forecasts/"
+    for date in dates:
+        target = data_dir + "ecmwf_t2m_" + date[:4] + ".grib"
+        tigge_request(date, target)
+          
+def tigge_request(date, target):
+    '''
+       A TIGGE request for ECMWF perturbed forecasts of T2M.
+    '''
+    server.retrieve({
+        'origin'    : "ecmf",
+        'levtype'   : "sfc",
+        'number'    : mem_numbers,
+        'expver'    : "prod",
+        'dataset'   : "tigge",
+        'step'      : "36/48",
+        'grid'      : "0.5/0.5",
+        'param'     : "167",
+        'area'      : "70/-10/30/30",
+        'time'      : "00",
+        'date'      : date,
+        'type'      : "pf",
+        'class'     : "ti",
+        'target'    : target,
+    })
+ 
+if __name__ == '__main__':
+    mem_numbers = ''.join([''.join([str(i) + "/" for i in range(1,50)]),'50'])
     retrieve_tigge_data()
diff --git a/data_retrieval/instructions.md b/data_retrieval/instructions.md
@@ -24,4 +24,5 @@ All data is available publicly from the TIGGE archives at ECMWF (requires regist
     - `geo` (to download land-sea mask and orography information)
     - `surface` (to download cloud cover, surface pressure and CAPE on the surface level)
     - `pl500` (to download u,v wind components and geopotential height at pressure level 500 hPa)
-    - `pl850` (to download u,v wind components and specific humidity at pressure level 850 hPa)
+    - `pl850` (to download u,v wind components and specific humidity at pressure level 850 hPa)
+    
diff --git a/data_retrieval/observations/getData_temp.R b/data_retrieval/observations/getData_temp.R
@@ -2,19 +2,15 @@
 
 ## note the terms of use: ftp://ftp-cdc.dwd.de/pub/CDC/Terms_of_use.pdf
 
-library(RCurl)
-library(magrittr)
-library(stringr)
-
 #### paths ####
-myPath <- "/path/to/data/"
+myPath <- "/home/gaojinghan/ppnn/data/observation/"
 
 # download folder
 dlPath <-
-  paste0(myPath, "data_files/download/")
+  paste0(myPath, "download/")
 # data folder
 dataPath <- 
-  paste0(myPath, "data_files/")
+  paste0(myPath, "")
 # download path
 dwdPath <-
   "ftp://ftp-cdc.dwd.de/pub/CDC/observations_germany/climate/hourly/air_temperature/historical/"
diff --git a/data_utils/config.py b/data_utils/config.py
@@ -0,0 +1,10 @@
+resolution = 0.5
+max_north = 55.0
+min_north = 0.5
+max_east = 140.0
+min_east = 70.5
+latitude = [min_north + lat * resolution for lat in range(int((max_north - min_north) / resolution) + 1)]
+longitude = [min_east + lon * resolution for lon in range(int((max_east - min_east) / resolution) + 1)]
+storage_dir = "/data/gaojinghan/EC_ensemble/manipulated/"
+observation_raw_dir = "/data/gaojinghan/EC_ensemble/TMP/2M_ABOVE_GROUND/"
+forecast_raw_dir = "/data/gaojinghan/EC_ensemble/"
diff --git a/data_utils/forecast_manipulated.py b/data_utils/forecast_manipulated.py
diff --git a/data_utils/observation_manipulated.py b/data_utils/observation_manipulated.py
diff --git a/nn_postprocessing/nn_src/utils.py b/nn_postprocessing/nn_src/utils.py
diff --git a/nn_postprocessing/run_all.py b/nn_postprocessing/run_all.py
diff --git a/nn_postprocessing/run_experiment.py b/nn_postprocessing/run_experiment.py
