refresh esd, restore esd file if error happens

with unit tests

Author: zhang-shengping

f5_openstack_agent/lbaasv2/drivers/bigip/esd_filehandler.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 
+import copy
 import glob
 import json
 import os
@@ -31,56 +32,50 @@
 LOG = logging.getLogger(__name__)
 
 
-class EsdJSONValidation(object):
-    """Class reads the json file(s)
+class EsdTagProcessor(object):
+    """Class processes json dictionary
 
-    It checks and parses the content of json file(s) to a dictionary
+    It checks compares the tags from esdjson dictionary to list of valid tags
     """
-    def __init__(self, esddir):
-        self.esdJSONFileList = glob.glob(os.path.join(esddir, '*.json'))
-        self.esdJSONDict = {}
-
-    def read_json(self):
-        for fileList in self.esdJSONFileList:
+    _instance = None
+    esd_dict = {}
+
+    def __new__(cls, *args, **kwargs):
+        if not isinstance(cls._instance, cls):
+            cls._instance = object.__new__(cls, *args, **kwargs)
+        return cls._instance
+
+    def read_json(self, esddir):
+        esdJSONFileList = glob.glob(os.path.join(esddir, '*.json'))
+        esdJSONDict = {}
+        for fileList in esdJSONFileList:
             try:
                 with open(fileList) as json_file:
                     # Reading each file to a dictionary
                     fileJSONDict = json.load(json_file)
                     # Combine all dictionaries to one
-                    self.esdJSONDict.update(fileJSONDict)
+                    esdJSONDict.update(fileJSONDict)
 
             except ValueError as err:
                 LOG.error('ESD JSON File is invalid: %s', err)
                 raise f5_ex.esdJSONFileInvalidException()
 
-        return self.esdJSONDict
-
-
-class EsdTagProcessor(EsdJSONValidation):
-    """Class processes json dictionary
-
-    It checks compares the tags from esdjson dictionary to list of valid tags
-    """
-    def __init__(self, esddir):
-        super(EsdTagProcessor, self).__init__(esddir)
-
-    # this function will return intersection of known valid esd tags
-    # and the ones that user provided
-    def valid_tag_key_subset(self):
-        self.validtags = list(set(self.esdJSONDict.keys()) &
-                              set(self.valid_esd_tags.keys()))
-        if not self.validtags:
-            LOG.error("Intersect of valid esd tags and user esd tags is empty")
-
-        if set(self.validtags) != set(self.esdJSONDict.keys()):
-            LOG.error("invalid tags in the user esd tags")
+        return esdJSONDict
 
-    def process_esd(self, bigips):
+    def process_esd(self, bigips, esddir):
+        esd_json_backup = {}
+        esd_json_backup = copy.deepcopy(self.esd_dict)
         try:
-            dict = self.read_json()
+            dict = self.read_json(esddir)
             self.esd_dict = self.verify_esd_dict(bigips, dict)
         except f5_ex.esdJSONFileInvalidException:
-            self.esd_dict = {}
+            # if error happens, we set backup esd.
+            self.esd_dict = copy.deepcopy(esd_json_backup)
+            LOG.warning(
+                "ESD JSON File is invalid, "
+                "ESD JSON Dict is restored to %s",
+                esd_json_backup
+            )
             raise
 
     def get_esd(self, name):

f5_openstack_agent/lbaasv2/drivers/bigip/icontrol_driver.py

@@ -841,20 +841,22 @@ def init_esd(self):
         # init esd object in lbaas_builder
         # init esd object in service_adapter
         esd_dir = os.path.join(self.get_config_dir(), 'esd')
-        self.esd = EsdTagProcessor(esd_dir)
+        # EsdTagProcessor is a singleton, so nothing new
+        self.esd = EsdTagProcessor()
         try:
-            self.esd.process_esd(self.get_all_bigips())
+            self.esd.process_esd(self.get_all_bigips(), esd_dir)
             self.lbaas_builder.init_esd(self.esd)
             self.service_adapter.init_esd(self.esd)
 
-            LOG.debug('esd details here after process_esd(): ')
-            LOG.debug(self.esd)
-            self.esd_names = self.esd.esd_dict.keys() or []
-            LOG.debug('##### self.esd_names obtainded here:')
-            LOG.debug(self.esd_names)
         except f5ex.esdJSONFileInvalidException as err:
             LOG.error("unable to initialize ESD. Error: %s.", err.message)
 
+        LOG.debug('ESD details here after process_esd(): ')
+        LOG.debug(self.esd)
+        self.esd_names = self.esd.esd_dict.keys() or []
+        LOG.debug('self.esd_names obtainded here:')
+        LOG.debug(self.esd_names)
+
     def _validate_ha(self, bigip):
         # if there was only one address supplied and
         # this is not a standalone device, get the

f5_openstack_agent/lbaasv2/drivers/bigip/test/test_esd_filehandler.py

@@ -17,7 +17,7 @@
 import pytest
 
 from f5_openstack_agent.lbaasv2.drivers.bigip.esd_filehandler import \
-    EsdJSONValidation
+    EsdTagProcessor
 from f5_openstack_agent.lbaasv2.drivers.bigip import exceptions as f5_ex
 
 
@@ -38,57 +38,59 @@ def assertIn(obj1, dict_obj, note=''):
 
     def test_invalid_dir_name(self):
         # invliad directory name
-        reader = EsdJSONValidation('/as87awoiujasdf/')
-        assert not reader.esdJSONFileList
+        esd = EsdTagProcessor()
+        reader = esd.read_json('/as87awoiujasdf/')
+        assert not reader
 
     def test_no_files(self, get_relative_path):
         # verify no files in empty directory
-        reader = EsdJSONValidation(
+        esd = EsdTagProcessor()
+        reader = esd.read_json(
             '{}/{}/empty_dir'.format(get_relative_path, self.remaining_path))
-        assert not reader.esdJSONFileList
+        assert not reader
 
     def test_no_json_files(self, get_relative_path):
         # verify no files are read in dir that contains non-JSON files
-        reader = EsdJSONValidation(
+        esd = EsdTagProcessor()
+        reader = esd.read_json(
             '{}/{}/no_json'.format(get_relative_path, self.remaining_path))
-        assert not reader.esdJSONFileList
+        assert not reader
 
     def test_mix_json_files(self, get_relative_path):
         # verify single JSON file
-        reader = EsdJSONValidation(
+        esd = EsdTagProcessor()
+        reader = esd.read_json(
             '{}/{}/mix_json/'.format(get_relative_path, self.remaining_path))
-        self.assertEqual(1, len(reader.esdJSONFileList))
+        self.assertEqual(3, len(reader.keys()))
 
     def test_json_only_files(self, get_relative_path):
         # expect three files
-        reader = EsdJSONValidation(
+        esd = EsdTagProcessor()
+        reader = esd.read_json(
             '{}/{}/valid'.format(get_relative_path, self.remaining_path))
-        self.assertEqual(3, len(reader.esdJSONFileList))
+        self.assertEqual(3, len(reader.keys()))
 
     def test_invalid_json(self, get_relative_path):
-        handler = EsdJSONValidation(
-            '{}/{}/invalid'.format(get_relative_path, self.remaining_path))
-
-        # verify exception raised
+        esd = EsdTagProcessor()
         with self.assertRaises(f5_ex.esdJSONFileInvalidException):
-            handler.read_json()
+            esd.read_json(
+                '{}/{}/invalid'.format(get_relative_path, self.remaining_path))
 
     def test_valid_json(self, get_relative_path):
-        handler = EsdJSONValidation(
+        esd = EsdTagProcessor()
+        result = esd.read_json(
             '{}/{}/valid/'.format(get_relative_path, self.remaining_path))
-        dict = handler.read_json()
 
         # verify keys in the final dictionary
-        self.assertIn('app_type_1', dict)
-        self.assertIn('app_type_2', dict)
-        self.assertIn('app_type_3', dict)
+        self.assertIn('app_type_1', result)
+        self.assertIn('app_type_2', result)
+        self.assertIn('app_type_3', result)
 
     def test_empty_json(self, get_relative_path):
         # verify empty file is read
-        handler = EsdJSONValidation(
+        esd = EsdTagProcessor()
+        result = esd.read_json(
             '{}/{}/empty_file/'.format(get_relative_path, self.remaining_path))
-        self.assertEqual(1, len(handler.esdJSONFileList))
 
         # verify empty dict is returned
-        dict = handler.read_json()
-        assert not dict
+        assert not result