New mapping template and added CSV output of decoded data

decoded_synop.csv

@@ -0,0 +1,13 @@
+report_type,year,month,day,hour,minute,wind_indicator,block_no,station_no,station_id,region,WMO_station_type,lowest_cloud_base,visibility,cloud_cover,time_significance,wind_time_period,wind_direction,wind_speed,air_temperature,dewpoint_temperature,relative_humidity,station_pressure,isobaric_surface,geopotential_height,sea_level_pressure,3hr_pressure_change,pressure_tendency_characteristic,precipitation_s1,ps1_time_period,present_weather,past_weather_1,past_weather_2,past_weather_time_period,cloud_vs_s1,cloud_amount_s1,low_cloud_type,middle_cloud_type,high_cloud_type,maximum_temperature,minimum_temperature,ground_state,ground_temperature,snow_depth,evapotranspiration,evaporation_instrument,temperature_change,tc_time_period,sunshine_amount_1hr,sunshine_amount_24hr,low_cloud_drift_direction,low_cloud_drift_vs,middle_cloud_drift_direction,middle_cloud_drift_vs,high_cloud_drift_direction,high_cloud_drift_vs,e_cloud_genus,e_cloud_direction,e_cloud_elevation,24hr_pressure_change,net_radiation_1hr,net_radiation_24hr,global_solar_radiation_1hr,global_solar_radiation_24hr,diffuse_solar_radiation_1hr,diffuse_solar_radiation_24hr,long_wave_radiation_1hr,long_wave_radiation_24hr,short_wave_radiation_1hr,short_wave_radiation_24hr,net_short_wave_radiation_1hr,net_short_wave_radiation_24hr,direct_solar_radiation_1hr,direct_solar_radiation_24hr,precipitation_s3,ps3_time_period,precipitation_24h,highest_gust_1,highest_gust_2,hg2_time_period,_wsi_series,_wsi_issuer,_wsi_issue_number,_wsi_local,_latitude,_longitude,_station_height
+AAXX,2022,3,21,12,0,,15,015,15015,,1,,50000,0,2,-10,250,1,283.45,264.15,24.799534703795413,97650.0,92500,952,,-200.0,7,0,-6,,,,-6,62,0,30,20,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15015,47.77706163,23.94046026,503
+AAXX,2022,3,21,12,0,,15,020,15020,,1,2500,10000,25,2,-10,310,4,286.15,265.65,23.314606896338145,101770.0,,,103770.0,-200.0,8,0,-6,,,,-6,8,1,30,24,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15020,47.73565324,26.64555017,161
+AAXX,2022,3,21,12,0,,15,090,15090,,1,2500,10000,63,2,-10,310,2,287.05,265.65,21.98759472867802,102710.0,,,103640.0,-310.0,8,0,-6,,,,-6,8,2,30,24,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15090,47.16333333,27.62722222,74.29
+AAXX,2022,3,21,12,0,,15,015,15015,,1,,50000,0,2,-10,250,1,283.45,264.15,24.799534703795413,97650.0,92500,952,,-200.0,7,0,-6,,,,-6,62,0,30,20,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15015,47.77706163,23.94046026,503
+AAXX,2022,3,21,12,0,,15,020,15020,,1,2500,10000,25,2,-10,310,4,286.15,265.65,23.314606896338145,101770.0,,,103770.0,-200.0,8,0,-6,,,,-6,8,1,30,24,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15020,47.73565324,26.64555017,161
+AAXX,2022,3,21,12,0,,15,090,15090,,1,2500,10000,63,2,-10,310,2,287.05,265.65,21.98759472867802,102710.0,,,103640.0,-310.0,8,0,-6,,,,-6,8,2,30,24,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15090,47.16333333,27.62722222,74.29
+AAXX,2022,3,21,12,0,,15,015,15015,,1,,50000,0,2,-10,250,1,283.45,264.15,24.799534703795413,97650.0,92500,952,,-200.0,7,0,-6,,,,-6,62,0,30,20,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15015,47.77706163,23.94046026,503
+AAXX,2022,3,21,12,0,,15,020,15020,,1,2500,10000,25,2,-10,310,4,286.15,265.65,23.314606896338145,101770.0,,,103770.0,-200.0,8,0,-6,,,,-6,8,1,30,24,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15020,47.73565324,26.64555017,161
+AAXX,2022,3,21,12,0,,15,090,15090,,1,2500,10000,63,2,-10,310,2,287.05,265.65,21.98759472867802,102710.0,,,103640.0,-310.0,8,0,-6,,,,-6,8,2,30,24,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15090,47.16333333,27.62722222,74.29
+AAXX,2022,3,21,12,0,15,015,15015,,1,,50000,0,2,-10,250,1,283.45,264.15,24.799534703795413,97650.0,92500,952,,-200.0,7,0,-6,,,,-6,62,0,30,20,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15015,47.77706163,23.94046026,503
+AAXX,2022,3,21,12,0,15,020,15020,,1,2500,10000,25,2,-10,310,4,286.15,265.65,23.314606896338145,101770.0,,,103770.0,-200.0,8,0,-6,,,,-6,8,1,30,24,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15020,47.73565324,26.64555017,161
+AAXX,2022,3,21,12,0,15,090,15090,,1,2500,10000,63,2,-10,310,2,287.05,265.65,21.98759472867802,102710.0,,,103640.0,-310.0,8,0,-6,,,,-6,8,2,30,24,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15090,47.16333333,27.62722222,74.29

synop.csv

@@ -0,0 +1,4 @@
+report_type,year,month,day,hour,minute,wind_indicator,block_no,station_no,station_id,region,WMO_station_type,lowest_cloud_base,visibility,cloud_cover,time_significance,wind_time_period,wind_direction,wind_speed,air_temperature,dewpoint_temperature,relative_humidity,station_pressure,isobaric_surface,geopotential_height,sea_level_pressure,3hr_pressure_change,pressure_tendency_characteristic,precipitation_s1,ps1_time_period,present_weather,past_weather_1,past_weather_2,past_weather_time_period,cloud_vs_s1,cloud_amount_s1,low_cloud_type,middle_cloud_type,high_cloud_type,maximum_temperature,minimum_temperature,ground_state,ground_temperature,snow_depth,evapotranspiration,evaporation_instrument,temperature_change,tc_time_period,sunshine_amount_1hr,sunshine_amount_24hr,low_cloud_drift_direction,low_cloud_drift_vs,middle_cloud_drift_direction,middle_cloud_drift_vs,high_cloud_drift_direction,high_cloud_drift_vs,e_cloud_genus,e_cloud_direction,e_cloud_elevation,24hr_pressure_change,net_radiation_1hr,net_radiation_24hr,global_solar_radiation_1hr,global_solar_radiation_24hr,diffuse_solar_radiation_1hr,diffuse_solar_radiation_24hr,long_wave_radiation_1hr,long_wave_radiation_24hr,short_wave_radiation_1hr,short_wave_radiation_24hr,net_short_wave_radiation_1hr,net_short_wave_radiation_24hr,direct_solar_radiation_1hr,direct_solar_radiation_24hr,precipitation_s3,ps3_time_period,precipitation_24h,highest_gust_1,highest_gust_2,hg2_time_period,_wsi_series,_wsi_issuer,_wsi_issue_number,_wsi_local,_latitude,_longitude,_station_height
+AAXX,2022,3,21,12,0,,15,015,15015,,1,,50000,0,2,-10,250,1,283.45,264.15,24.799534703795413,97650.0,92500,952,,-200.0,7,0,-6,,,,-6,62,0,30,20,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15015,47.77706163,23.94046026,503
+AAXX,2022,3,21,12,0,,15,020,15020,,1,2500,10000,25,2,-10,310,4,286.15,265.65,23.314606896338145,101770.0,,,103770.0,-200.0,8,0,-6,,,,-6,8,1,30,24,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15020,47.73565324,26.64555017,161
+AAXX,2022,3,21,12,0,,15,090,15090,,1,2500,10000,63,2,-10,310,2,287.05,265.65,21.98759472867802,102710.0,,,103640.0,-310.0,8,0,-6,,,,-6,8,2,30,24,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-360,0,20000,0,15090,47.16333333,27.62722222,74.29

synop2bufr/__init__.py

@@ -41,8 +41,9 @@
 PASSED = 1
 
 # Enumerate the keys
-_keys = ['report_type', 'year', 'month', 'day', 'hour', 'minute',
-         'wind_indicator', 'block_no', 'station_no', 'station_id', 'region',
+_keys = ['report_type', 'year', 'month', 'day',
+         'hour', 'minute',
+         'block_no', 'station_no', 'station_id', 'region',
          'WMO_station_type',
          'lowest_cloud_base', 'visibility', 'cloud_cover',
          'time_significance', 'wind_time_period',
@@ -145,24 +146,16 @@ def parse_synop(message: str, year: int, month: int) -> dict:
             iw = decoded['wind_indicator']['value']
         except Exception:
             iw = None
-        # In this conversion, we convert bit number to a value (see code table
-        # 0 02 002)  # noq
-        # Not bit 3 should never be set for synop, units of km/h not reportable
-        if iw == 0:
-            iw_translated = 0b0000  # Wind in m/s, default, no bits set
-        elif iw == 1:
-            iw_translated = 0b1000  # Wind in m/s with anemometer bit 1 (left most) set  # noqa
-        elif iw == 3:
-            iw_translated = 0b0100  # Wind in knots, bit 2 set
-        elif iw == 4:
-            iw_translated = 0b1100  # Wind in knots with anemometer, bits
-            # 1 and 2 set # noq
+
+        # If iw = 1 or 4, the wind was obtained from an
+        # anemometer and thus the corresponding wind data
+        # should be encoded
+        if iw == 1 or iw == 4:
+            ENCODE_WIND = True
         else:
-            iw_translated = None  # 0b1111  # Missing value
+            ENCODE_WIND = False
     else:
-        iw_translated = None  # 0b1111  # Missing value
-
-    output['wind_indicator'] = iw_translated
+        ENCODE_WIND = False
 
     if 'station_id' in decoded:
         try:
@@ -228,7 +221,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
             output['cloud_cover'] = None
 
     # Wind direction is already in degrees
-    if 'surface_wind' in decoded:
+    if ('surface_wind' in decoded) and ENCODE_WIND:
         # See B/C1.10.5.3
         # NOTE: Every time period in the following code shall be a negative number,  # noqa
         # to indicate measurements have been taken up until the present.
@@ -264,7 +257,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
 
     if 'dewpoint_temperature' in decoded:
         try:
-            output['dewpoint_temperature'] = round(decoded['dewpoint_temperature']['value'] +273.15, 2)  # noqa
+            output['dewpoint_temperature'] = round(decoded['dewpoint_temperature']['value'] + 273.15, 2)  # noqa
         except Exception:
             output['dewpoint_temperature'] = None
 
@@ -304,7 +297,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
     # pressure has precision in tens of Pa
     if 'station_pressure' in decoded:
         try:
-            output['station_pressure'] = round(decoded['station_pressure']['value'] * 100, -1) # noqa
+            output['station_pressure'] = round(decoded['station_pressure']['value'] * 100, -1)  # noqa
         except Exception:
             output['station_pressure'] = None
 
@@ -317,7 +310,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
 
     if 'geopotential' in decoded:
         try:
-            output['isobaric_surface'] = round( decoded['geopotential']['surface']['value'] * 100, 1)  # noqa
+            output['isobaric_surface'] = round(decoded['geopotential']['surface']['value'] * 100, 1)  # noqa
         except Exception:
             output['isobaric_surface'] = None
         try:
@@ -328,7 +321,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
     if 'pressure_tendency' in decoded:
         #  By B/C1.3.3, pressure has precision in tens of Pa
         try:
-            output['3hr_pressure_change'] = round(decoded['pressure_tendency']['change']['value'] * 100, -1) # noqa
+            output['3hr_pressure_change'] = round(decoded['pressure_tendency']['change']['value'] * 100, -1)  # noqa
         except Exception:
             output['3hr_pressure_change'] = None
 
@@ -501,7 +494,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
         try:
             output['minimum_temperature'] = decoded['minimum_temperature']['value']  # noqa
             if output['minimum_temperature'] is not None:
-                output['minimum_temperature'] = round( output['minimum_temperature'] + 273.15, 2)  # noqa
+                output['minimum_temperature'] = round(output['minimum_temperature'] + 273.15, 2)  # noqa
         except Exception:
             output['minimum_temperature'] = None
 
@@ -524,7 +517,7 @@ def parse_synop(message: str, year: int, month: int) -> dict:
         if decoded['ground_state']['temperature'] is not None:
             try:
                 #  Convert to Kelvin
-                output['ground_temperature'] = round( decoded['ground_state']['temperature']['value'] + 273.15, 2)  # noqa
+                output['ground_temperature'] = round(decoded['ground_state']['temperature']['value'] + 273.15, 2)  # noqa
             except Exception:
                 output['ground_temperature'] = None
 
@@ -971,7 +964,7 @@ def rad_convert(rad, time):
     #  wind gust speed for region VI (groups 910fmfm and 911fxfx).
     #  These are given and required to be in m/s.
 
-    if 'highest_gust' in decoded:
+    if ('highest_gust' in decoded) and ENCODE_WIND:
         try:
             output['highest_gust_1'] = decoded['highest_gust']['gust_1']['speed']['value']  # noqa
         except Exception:
@@ -1230,6 +1223,20 @@ def transform(data: str, metadata: str, year: int,
                 LOGGER.warning((f"Invalid WSI ({wsi}) found in station file,"
                                 " unable to parse"))
 
+        # Write message to CSV file
+        try:
+            with open("decoded_synop.csv", "a", newline="") as output_csv:
+                dict_writer = csv.DictWriter(output_csv, msg.keys())
+
+                # Check if CSV file is empty before adding headers
+                if os.stat("decoded_synop.csv").st_size == 0:
+                    dict_writer.writeheader()
+
+                # Write data to rows
+                dict_writer.writerow(msg)
+        except Exception:
+            LOGGER.warning(f"Unable to write report of station {tsi} to CSV")
+
         for idx in range(num_s3_clouds):
             # Build the dictionary of mappings for section 3 group 8NsChshs
 
@@ -1313,6 +1320,7 @@ def transform(data: str, metadata: str, year: int,
         # parse
         if conversion_success[tsi]:
             try:
+                # Parse to BUFRMessage object
                 message.parse(msg, mapping)
             except Exception as e:
                 LOGGER.error(e)

synop2bufr/resources/synop-mappings.json

@@ -22,7 +22,7 @@
         {"eccodes_key": "typicalDay", "value": "data:day"},
         {"eccodes_key": "typicalHour", "value": "data:hour"},
         {"eccodes_key": "typicalMinute", "value": "data:minute"},
-        {"eccodes_key": "unexpandedDescriptors", "value":"array:301150, 307080"}
+        {"eccodes_key": "unexpandedDescriptors", "value":"array:301150, 307096"}
     ],
     "data": [
         {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:_wsi_series"},
@@ -50,7 +50,6 @@
         {"eccodes_key": "#1#airTemperature", "value": "data:air_temperature"},
         {"eccodes_key": "#1#dewpointTemperature", "value": "data:dewpoint_temperature"},
         {"eccodes_key": "#1#relativeHumidity", "value": "data:relative_humidity"},
-        {"eccodes_key": "#1#instrumentationForWindMeasurement", "value": "data:wind_indicator"},
         {"eccodes_key": "#1#timeSignificance", "value": "const:2"},
         {"eccodes_key": "#10#timePeriod", "value": "const:-10"},
         {"eccodes_key": "#1#windDirection", "value": "data:wind_direction"},

tests/test_synop2bufr.py

@@ -71,7 +71,6 @@ def test_conversion(single_message):
     # We now need to check that most the dictionary items are what we expect
     assert d['station_id'] == "15001"
     assert d['day'] == 21
-    assert d['wind_indicator'] == 8
     assert d['WMO_station_type'] == 0
     assert d['lowest_cloud_base'] == 600
     assert d['visibility'] == 1500
@@ -134,9 +133,9 @@ def test_bufr(multiple_messages, metadata_string):
     msgs = {}
     for item in result:
         msgs[item['_meta']['id']] = item
-    assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'c99408ed63070de0919208b23aacbbd2'  # noqa
-    assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '40a5cc35b454da20c14af88c711d071f'  # noqa
-    assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '86bc25026de14f9f1cba2e449740c0cd'  # noqa
+    assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '1673febbb27d4ec5a009a96053e513af'  # noqa
+    assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '4e05f1511e770bc8a721a71c6db35fc9'  # noqa
+    assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == 'd82ee99c9aee02ada7ccfb248ded7351'  # noqa
 
 
 def test_invalid_separation():