WIP

Author: StevenMaude

tests/test_scraperwiki.py

@@ -20,22 +20,66 @@
 
 @pytest.fixture(scope="function")
 def clean_db():
-    try:
-        os.remove('scraperwiki.sqlite')
-    except OSError:
-        pass
+    """
+    Robust database cleanup fixture.
+    Crucially, this disposes of the SQLAlchemy engine to release file locks.
+    """
+    # 1. Close the active transaction/connection
+    if scraperwiki.sql._State._connection:
+        try:
+            scraperwiki.sql._State._connection.close()
+        except Exception:
+            pass
+
+    # 2. IMPORTANT: Dispose of the engine.
+    # This closes all connections in the pool that might be holding the file open.
+    if scraperwiki.sql._State.engine:
+        scraperwiki.sql._State.engine.dispose()
+
+    # 3. Reset global state variables
+    scraperwiki.sql._State._connection = None
+    scraperwiki.sql._State._transaction = None
+    scraperwiki.sql._State.engine = None
+    scraperwiki.sql._State.metadata = None
+    scraperwiki.sql._State.table = None
+    scraperwiki.sql._State.table_pending = None
+
+    # 4. Delete the database file
+    if os.path.exists('scraperwiki.sqlite'):
+        try:
+            os.remove('scraperwiki.sqlite')
+        except OSError:
+            # If we still can't delete it (e.g. Windows file locking),
+            # it might be okay if we are overwriting, but usually this implies a leak.
+            pass
+
+    yield  # Run the test
+
+    # 5. Cleanup after test
+    # (Repeat the close/dispose logic to ensure no leaking into the next test)
+    if scraperwiki.sql._State._connection:
+        try:
+             scraperwiki.sql._State._connection.close()
+        except Exception: pass
+
+    if scraperwiki.sql._State.engine:
+        scraperwiki.sql._State.engine.dispose()
+
+    if os.path.exists('scraperwiki.sqlite'):
+        try:
+            os.remove('scraperwiki.sqlite')
+        except OSError:
+            pass
 
 # called TestAAAWarning so that it gets run first by nosetests,
 # which we need, otherwise the warning has already happened.
-@pytest.mark.usefixtures("clean_db")
 class TestAAAWarning(TestCase):
     def test_save_no_warn(self):
         with warnings.catch_warnings():
             warnings.simplefilter("error")
             scraperwiki.sql.save(['id'], dict(id=4, tumble='weed'),
               table_name="warning_test")
 
-@pytest.mark.usefixtures("clean_db")
 class TestSaveGetVar(TestCase):
     def savegetvar(self, var):
         scraperwiki.sql.save_var(u"weird\u1234", var)
@@ -76,12 +120,10 @@ def test_save_multiple_values(self):
         self.assertEqual(u'hello', scraperwiki.sql.get_var(u'foo\xc3'))
         self.assertEqual(u'goodbye\u1234', scraperwiki.sql.get_var(u'bar'))
 
-@pytest.mark.usefixtures("clean_db")
 class TestGetNonexistantVar(TestCase):
     def test_get(self):
         self.assertIsNone(scraperwiki.sql.get_var(u'meatball\xff'))
 
-@pytest.mark.usefixtures("clean_db")
 class TestSaveVar(TestCase):
     def setUp(self):
         scraperwiki.sql.save_var(u"birthday\xfe", u"\u1234November 30, 1888")
@@ -99,7 +141,6 @@ def test_insert(self):
         observed = [(colname, value.decode('utf-8'), _type)]
         self.assertEqual(observed, expected)
 
-@pytest.mark.usefixtures("clean_db")
 class SaveAndCheck(TestCase):
     def save_and_check(self, dataIn, tableIn, dataOut, tableOut=None, twice=True):
         if tableOut == None:
@@ -127,14 +168,12 @@ def save_and_check(self, dataIn, tableIn, dataOut, tableOut=None, twice=True):
             self.assertListEqual(observed1, expected1)
             self.assertListEqual(observed2, expected2)
 
-@pytest.mark.usefixtures("clean_db")
 class SaveAndSelect(TestCase):
     def save_and_select(self, d):
         scraperwiki.sql.save([], {u"foo\xdd": d})
         observed = scraperwiki.sql.select(u'* FROM swdata')[0][u'foo\xdd']
         self.assertEqual(d, observed)
 
-@pytest.mark.usefixtures("clean_db")
 class TestUniqueKeys(SaveAndSelect):
     def test_empty(self):
         scraperwiki.sql.save([], {u"foo\xde": 3}, table_name=u"Chico\xcc")
@@ -181,7 +220,6 @@ def test_two(self):
         uniquecol = indices[u"keys"].index(u'unique')
         self.assertEqual(index[uniquecol], 1)
 
-@pytest.mark.usefixtures("clean_db")
 class TestSaveColumn(TestCase):
     def test_add_column(self):
         # Indicative for
@@ -209,7 +247,6 @@ def test_add_column(self):
         self.assertEqual(stdout, "".encode('utf-8'))
         self.assertEqual(stderr, "".encode('utf-8'))
 
-@pytest.mark.usefixtures("clean_db")
 class TestSave(SaveAndCheck):
     def test_save_int(self):
         self.save_and_check(
@@ -282,7 +319,6 @@ def test_save_and_drop(self):
         scraperwiki.sql.execute(u"DROP TABLE dropper\xaa")
         scraperwiki.sql.save([], dict(foo=9), table_name=u"dropper\xaa")
 
-@pytest.mark.usefixtures("clean_db")
 class TestQuestionMark(TestCase):
     def test_one_question_mark_with_nonlist(self):
         scraperwiki.sql.execute(u'CREATE TABLE zhuozi\xaa (\xaa TEXT);')
@@ -306,7 +342,6 @@ def test_multiple_question_marks(self):
         self.assertListEqual(observed, [{'a': 'apple', 'b': 'banana'}])
         scraperwiki.sql.execute('DROP TABLE zhuozi')
 
-@pytest.mark.usefixtures("clean_db")
 class TestDateTime(TestCase):
     def rawdate(self, table="swdata", column="datetime"):
         connection = sqlite3.connect(DB_NAME)
@@ -359,7 +394,6 @@ def test_status(self):
 
             self.assertEqual(scraperwiki.status('ok'), None)
 
-@pytest.mark.usefixtures("clean_db")
 class TestUnicodeColumns(TestCase):
     maxDiff = None
     def test_add_column_once_only(self):