Centre and subcentre default to missing + documentation updates

wmo-im · Feb 1, 2024 · cc0875b · cc0875b
1 parent 78043d7
commit cc0875b
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 24 deletions.
diff --git a/.gitignore b/.gitignore
@@ -42,6 +42,8 @@ nosetests.xml
 logs
 .vscode/
 .vscode/settings.json
+# Ignore decoded CSV files
+decoded_*.csv
 
 # pycharm
 .idea

diff --git a/Dockerfile b/Dockerfile
@@ -18,8 +18,8 @@ RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";"
 
 ENV LOG_LEVEL=INFO
 # The following need to changed to the correct values for your centre!
-ENV BUFR_ORIGINATING_CENTRE=123
-ENV BUFR_ORIGINATING_SUBCENTRE=123
+ENV BUFR_ORIGINATING_CENTRE=255
+ENV BUFR_ORIGINATING_SUBCENTRE=255
 
 # copy the app
 COPY . /build

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ Dependencies are listed in [requirements.txt](https://github.com/wmo-im/synop2bu
 
 ### Setting Environment Variables
 
-Before using synop2bufr, you need to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, the conversion to BUFR will fail.**
+Before using synop2bufr, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (255).**
 
 It is recommended that you set these environment variables in the Dockerfile, by editing the following lines with your originating centre and subcentre values:
 

diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
@@ -34,7 +34,7 @@ Command line interface (CLI)
 Setting Environment Variables
 -----------------------------
 
-Before running the `synop2bufr data transform` command, you need to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, the conversion to BUFR will fail.**
+Before running the `synop2bufr data transform` command, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (255).**
 
 It is recommended that you set these environment variables in the Dockerfile, by editing the following lines with your originating centre and subcentre values:
 

diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py
@@ -1270,10 +1270,19 @@ def transform(data: str, metadata: str, year: int,
 
     :returns: iterator
     """
+    # =============================================
     # Make warning and error messages array global
+    # =============================================
     global warning_msgs
     global error_msgs
 
+    # Boolean to ensure environment variable warning is only displayed once
+    # Note: The resetting of the warning_msgs array for
+    # each report necessitates this approach, because
+    # we want to ensure the warning is only appended
+    # to the first conversion
+    can_var_warning_be_displayed = True
+
     # ===================
     # First parse metadata file
     # ===================
@@ -1493,42 +1502,44 @@ def transform(data: str, metadata: str, year: int,
                 else:
                     # If station has not been found in the station
                     # list, don't repeat warning unnecessarily
-                    if f"Station {tsi} not found in station file" not in warning_msgs: # noqa
+                    if f"Station {tsi} not found in station file" not in warning_msgs:  # noqa
                         LOGGER.warning(f"Invalid metadata for station {tsi} found in station file, unable to parse")  # noqa
                         warning_msgs.append(f"Invalid metadata for station {tsi} found in station file, unable to parse")  # noqa
 
             # Add information to the mappings
             if conversion_success[tsi]:
-                # First check that the mandatory BUFR header centre
+                # First check if the BUFR header centre
                 # and subcentre codes are present
                 missing_env_vars = []
 
                 if os.environ.get("BUFR_ORIGINATING_CENTRE") is None:
                     missing_env_vars.append("BUFR_ORIGINATING_CENTRE")
+                else:
+                    # Add the BUFR header centre and subcentre to mappings
+                    mapping["header"].append({
+                        "eccodes_key": "bufrHeaderCentre",
+                        "value": f"const:{os.environ.get('BUFR_ORIGINATING_CENTRE')}"  # noqa
+                    })
 
                 if os.environ.get("BUFR_ORIGINATING_SUBCENTRE") is None:
                     missing_env_vars.append("BUFR_ORIGINATING_SUBCENTRE")
+                else:
+                    mapping["header"].append({
+                        "eccodes_key": "bufrHeaderSubCentre",
+                        "value": f"const:{os.environ.get('BUFR_ORIGINATING_SUBCENTRE')}"  # noqa
+                    })
 
                 # If either of these environment variables are not set,
-                # we stop the conversion completely here
-                if missing_env_vars:
-                    # Display error messages
+                # we will default to missing and warn the user once
+                if missing_env_vars and can_var_warning_be_displayed:
+                    # Display ewarning messages
                     for var in missing_env_vars:
-                        LOGGER.error(f"The {var} environment variable is not set, unable to convert message to BUFR") # noqa
-                        error_msgs.append(f"The {var} environment variable is not set, unable to convert message to BUFR") # noqa
-                    conversion_success[tsi] = False
-                    # Exit conversion loop
-                    break
-
-                # Add the BUFR header centre and subcentre to mappings
-                mapping["header"].append({
-                    "eccodes_key": "bufrHeaderCentre",
-                    "value": f"const:{os.environ.get('BUFR_ORIGINATING_CENTRE')}"  # noqa
-                })
-                mapping["header"].append({
-                    "eccodes_key": "bufrHeaderSubCentre",
-                    "value": f"const:{os.environ.get('BUFR_ORIGINATING_SUBCENTRE')}"  # noqa
-                })
+                        var_warning = f"The {var} environment variable is not set, will default to missing!"  # noqa
+                        LOGGER.warning(var_warning)
+                        warning_msgs.append(var_warning)
+                        can_var_warning_be_displayed = False
+                    # Stop duplicated warnings
+                    can_var_warning_be_displayed = False
 
                 # Now we need to add the mappings for the cloud groups
                 # of section 3 and 4

diff --git a/synop2bufr/resources/synop-mappings-307080.json b/synop2bufr/resources/synop-mappings-307080.json
@@ -8,6 +8,8 @@
     "header":[
         {"eccodes_key": "edition", "value": "const:4"},
         {"eccodes_key": "masterTableNumber", "value": "const:0"},
+        {"eccodes_key": "bufrHeaderCentre", "value": "const:255"},
+        {"eccodes_key": "bufrHeaderSubCentre", "value": "const:255"},
         {"eccodes_key": "updateSequenceNumber", "value": "const:0"},
         {"eccodes_key": "dataCategory", "value": "const:0"},
         {"eccodes_key": "internationalDataSubCategory", "value": "const:2"},

diff --git a/synop2bufr/resources/synop-mappings-307096.json b/synop2bufr/resources/synop-mappings-307096.json
@@ -8,6 +8,8 @@
     "header":[
         {"eccodes_key": "edition", "value": "const:4"},
         {"eccodes_key": "masterTableNumber", "value": "const:0"},
+        {"eccodes_key": "bufrHeaderCentre", "value": "const:255"},
+        {"eccodes_key": "bufrHeaderSubCentre", "value": "const:255"},
         {"eccodes_key": "updateSequenceNumber", "value": "const:0"},
         {"eccodes_key": "dataCategory", "value": "const:0"},
         {"eccodes_key": "internationalDataSubCategory", "value": "const:2"},