use queue and listener for logging (#2528)

* use queue and listener for logging

* flake8

Author: LKuemmel

packages/helpermodules/logger.py

@@ -1,7 +1,9 @@
 import functools
 import logging
+import logging.handlers
 from logging.handlers import RotatingFileHandler
 from pathlib import Path
+import queue
 import sys
 import threading
 import typing_extensions
@@ -106,72 +108,119 @@ def filter_pos(name: str, record) -> bool:
 def setup_logging() -> None:
     def mb_to_bytes(megabytes: int) -> int:
         return megabytes * 1000000
-    # Mehrere kleine Dateien verwenden, damit nicht zu viel verworfen wird, wenn die Datei voll ist.
+
+    # Main logger
+    log_queue = queue.Queue()
+    queue_handler = logging.handlers.QueueHandler(log_queue)
     main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5.5), backupCount=4)
     main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
     main_file_handler.addFilter(RedactingFilter())
-    logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler])
+    logging.basicConfig(level=logging.DEBUG, handlers=[queue_handler])
     logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "soc"))
     logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "Internal Chargepoint"))
     logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "smarthome"))
+    main_listener = logging.handlers.QueueListener(log_queue, main_file_handler)
+    main_listener.start()
 
+    # Chargelog logger
+    chargelog_queue = queue.Queue()
+    chargelog_queue_handler = logging.handlers.QueueHandler(chargelog_queue)
     chargelog_log = logging.getLogger("chargelog")
     chargelog_log.propagate = False
     chargelog_file_handler = RotatingFileHandler(
         RAMDISK_PATH + 'chargelog.log', maxBytes=mb_to_bytes(2), backupCount=1)
     chargelog_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
     chargelog_file_handler.addFilter(RedactingFilter())
-    chargelog_log.addHandler(chargelog_file_handler)
+    chargelog_log.addHandler(chargelog_queue_handler)
+    chargelog_listener = logging.handlers.QueueListener(chargelog_queue, chargelog_file_handler)
+    chargelog_listener.start()
 
+    # Data migration logger
+    data_migration_queue = queue.Queue()
+    data_migration_queue_handler = logging.handlers.QueueHandler(data_migration_queue)
     data_migration_log = logging.getLogger("data_migration")
     data_migration_log.propagate = False
     data_migration_file_handler = RotatingFileHandler(
         PERSISTENT_LOG_PATH + 'data_migration.log', maxBytes=mb_to_bytes(1), backupCount=1)
     data_migration_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
     data_migration_file_handler.addFilter(RedactingFilter())
-    data_migration_log.addHandler(data_migration_file_handler)
+    data_migration_log.addHandler(data_migration_queue_handler)
+    data_migration_listener = logging.handlers.QueueListener(data_migration_queue, data_migration_file_handler)
+    data_migration_listener.start()
 
+    # MQTT logger
+    mqtt_queue = queue.Queue()
+    mqtt_queue_handler = logging.handlers.QueueHandler(mqtt_queue)
     mqtt_log = logging.getLogger("mqtt")
     mqtt_log.propagate = False
     mqtt_file_handler = RotatingFileHandler(RAMDISK_PATH + 'mqtt.log', maxBytes=mb_to_bytes(3), backupCount=1)
     mqtt_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
     mqtt_file_handler.addFilter(RedactingFilter())
-    mqtt_log.addHandler(mqtt_file_handler)
+    mqtt_log.addHandler(mqtt_queue_handler)
+    mqtt_listener = logging.handlers.QueueListener(mqtt_queue, mqtt_file_handler)
+    mqtt_listener.start()
 
+    # Steuve control command logger
+    steuve_control_command_queue = queue.Queue()
+    steuve_control_command_queue_handler = logging.handlers.QueueHandler(steuve_control_command_queue)
     steuve_control_command_log = logging.getLogger("steuve_control_command")
     steuve_control_command_log.propagate = False
     steuve_control_command_file_handler = RotatingFileHandler(
         PERSISTENT_LOG_PATH + 'steuve_control_command.log', maxBytes=mb_to_bytes(80), backupCount=1)
     steuve_control_command_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
-    steuve_control_command_log.addHandler(steuve_control_command_file_handler)
-
+    steuve_control_command_log.addHandler(steuve_control_command_queue_handler)
+    steuve_control_command_listener = logging.handlers.QueueListener(steuve_control_command_queue,
+                                                                     steuve_control_command_file_handler)
+    steuve_control_command_listener.start()
+
+    # Smarthome logger
+    smarthome_queue = queue.Queue()
+    smarthome_queue_handler = logging.handlers.QueueHandler(smarthome_queue)
     smarthome_log_handler = RotatingFileHandler(RAMDISK_PATH + 'smarthome.log', maxBytes=mb_to_bytes(1), backupCount=1)
     smarthome_log_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
     smarthome_log_handler.addFilter(functools.partial(filter_pos, "smarthome"))
     smarthome_log_handler.addFilter(RedactingFilter())
-    logging.getLogger().addHandler(smarthome_log_handler)
+    logging.getLogger().addHandler(smarthome_queue_handler)
+    smarthome_listener = logging.handlers.QueueListener(smarthome_queue, smarthome_log_handler)
+    smarthome_listener.start()
 
+    # SoC logger
+    soc_queue = queue.Queue()
+    soc_queue_handler = logging.handlers.QueueHandler(soc_queue)
     soc_log_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1)
     soc_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
     soc_log_handler.addFilter(functools.partial(filter_pos, "soc"))
     soc_log_handler.addFilter(RedactingFilter())
-    logging.getLogger().addHandler(soc_log_handler)
+    logging.getLogger().addHandler(soc_queue_handler)
+    soc_listener = logging.handlers.QueueListener(soc_queue, soc_log_handler)
+    soc_listener.start()
 
+    # Internal chargepoint logger
+    internal_chargepoint_queue = queue.Queue()
+    internal_chargepoint_queue_handler = logging.handlers.QueueHandler(internal_chargepoint_queue)
     internal_chargepoint_log_handler = RotatingFileHandler(RAMDISK_PATH + 'internal_chargepoint.log',
                                                            maxBytes=mb_to_bytes(1),
                                                            backupCount=1)
     internal_chargepoint_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
     internal_chargepoint_log_handler.addFilter(functools.partial(filter_pos, "Internal Chargepoint"))
     internal_chargepoint_log_handler.addFilter(RedactingFilter())
-    logging.getLogger().addHandler(internal_chargepoint_log_handler)
-
+    logging.getLogger().addHandler(internal_chargepoint_queue_handler)
+    internal_chargepoint_listener = logging.handlers.QueueListener(internal_chargepoint_queue,
+                                                                   internal_chargepoint_log_handler)
+    internal_chargepoint_listener.start()
+
+    # urllib3 logger
+    urllib3_queue = queue.Queue()
+    urllib3_queue_handler = logging.handlers.QueueHandler(urllib3_queue)
     urllib3_log = logging.getLogger("urllib3.connectionpool")
     urllib3_log.propagate = True
     urllib3_file_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1)
     urllib3_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
     urllib3_file_handler.addFilter(RedactingFilter())
     urllib3_file_handler.addFilter(functools.partial(filter_pos, "soc"))
-    urllib3_log.addHandler(urllib3_file_handler)
+    urllib3_log.addHandler(urllib3_queue_handler)
+    urllib3_listener = logging.handlers.QueueListener(urllib3_queue, urllib3_file_handler)
+    urllib3_listener.start()
 
     logging.getLogger("pymodbus").setLevel(logging.WARNING)
     logging.getLogger("uModbus").setLevel(logging.WARNING)