From 8c13db4bf10c21f1563f6487e32645e3625cc8b9 Mon Sep 17 00:00:00 2001
From: david-i-berry <dberry@wmo.int>
Date: Tue, 5 Sep 2023 15:07:58 +0200
Subject: [PATCH] Catch logging messages in a queue so that they can be added
 to the return object.

---
 synop2bufr/__init__.py | 37 ++++++++++++++++++++++++++++++++++---
 synop2bufr/cli.py      | 13 ++-----------
 2 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py
index 16a7b0d..338b578 100644
--- a/synop2bufr/__init__.py
+++ b/synop2bufr/__init__.py
@@ -24,9 +24,12 @@
 from io import StringIO
 import json
 import logging
+import logging.handlers
 import math
 import os
+from queue import Queue
 import re
+import sys
 from typing import Iterator
 
 from csv2bufr import BUFRMessage
@@ -34,7 +37,28 @@
 
 __version__ = '0.5.1'
 
-LOGGER = logging.getLogger(__name__)
+logging_events = Queue()
+
+# Set up logging
+LOGGER = logging.getLogger(__name__)  # logger for this module
+root_logger = logging.getLogger()  # pymetdecoder writes to the root logger
+
+# Format for the output
+formatter = logging.Formatter(fmt="%(asctime)s [%(levelname)s] %(message)s",
+                              datefmt="%Y-%m-%d %H:%M:%S")
+
+# Handlers, one for an error / warning event queue
+queue_handler = logging.handlers.QueueHandler(logging_events)
+queue_handler.setFormatter(formatter)
+
+# And one for stdout
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+stdout_handler.setFormatter(formatter)
+
+
+# Add these to the root logger
+root_logger.addHandler(queue_handler)
+root_logger.addHandler(stdout_handler)
 
 # status codes
 FAILED = 0
@@ -1515,8 +1539,15 @@ def transform(data: str, metadata: str, year: int,
                     "csv": csv_string
                 }
 
-            result["warnings"] = warning_msgs
-            result["errors"] = error_msgs
+            result["warnings"] = []
+            result["errors"] = []
+
+            while (not logging_events.empty()):
+                event = logging_events.get()
+                if event.levelno == logging.WARNING:
+                    result["warnings"].append(event.message)
+                elif event.levelno in (logging.ERROR, logging.CRITICAL):  # noqs
+                    result["errors"].append(event.message)
 
             # now yield result back to caller
             yield result
diff --git a/synop2bufr/cli.py b/synop2bufr/cli.py
index 71cc37d..2543c04 100644
--- a/synop2bufr/cli.py
+++ b/synop2bufr/cli.py
@@ -35,17 +35,7 @@
     datefmt="%Y-%m-%d %H:%M:%S"
 )
 
-# if (LOGGER.hasHandlers()):
-#     LOGGER.handlers.clear()
-
-# # Configure error handler
-# handler_err = logging.StreamHandler(sys.stderr)
-# handler_err.setLevel(logging.ERROR)
-# handler_err.setFormatter(logging.Formatter(
-#     fmt="%(asctime)s [%(levelname)s] %(message)s",
-#     datefmt="%Y-%m-%d %H:%M:%S"
-# ))
-# LOGGER.addHandler(handler_err)
+LOGGER.setLevel(log_level)
 
 
 def cli_option_verbosity(f):
@@ -120,6 +110,7 @@ def transform(ctx, synop_file, metadata, output_dir, year, month, verbosity):
                 '%Y%m%dT%H%M%S'
             )
 
+
             filename = f"decoded_{timestamp}.csv"
 
             if header_written: