Merge pull request #145 from DCC-Lab/cleanup-import

Cleanup import, small module fixes

Author: dccote

dcclab/__init__.py

@@ -13,5 +13,5 @@
 from .utils import *
 from .DCCExceptions import *
 
-__version__ = "1.1.1"
+__version__ = "1.1.2"
 __author__ = "Daniel Cote <dccote@cervo.ulaval.ca>"

dcclab/analysis/__init__.py

@@ -0,0 +1,2 @@
+from .computeengine import *
+from .labpca import *

dcclab/analysis/computeengine.py

@@ -2,9 +2,8 @@
 from threading import Thread
 from queue import Empty
 import time
-from json import dumps
 import signal
-from database import *
+from dcclab.database import *
 
 """
 A class to run many tasks in parallel when they are mostly independent.  This engine is perfectly

dcclab/database/database.py

@@ -218,7 +218,7 @@ def connect(self):
                         pwd = keyring.get_password(serviceName, self.mysqlUser)
                         if pwd is None:
                             raise Exception(""" Set the password in the system password manager on the command line with:
-                            python -m keyring set {0} {1}""".format(serviceName, self.mysqlUser))
+{2} -m keyring set {0} {1}""".format(serviceName, self.mysqlUser, sys.executable))
                     else:
                         pwd = None
 
@@ -524,7 +524,7 @@ def connect(self):
                     pwd = keyring.get_password(serviceName, self.mysqlUser)
                     if pwd is None:
                         raise Exception(""" Set the password in the system password manager on the command line with:
-                        python -m keyring set {0} {1}""".format(serviceName, self.mysqlUser))
+{2} -m keyring set {0} {1}""".format(serviceName, self.mysqlUser, sys.executable))
                 else:
                     pwd = None
 

dcclab/database/labdatadb.py

@@ -2,7 +2,6 @@
 import numpy as np
 import pandas as pd
 import re
-from collections.abc import Iterable
 
 class LabdataDB(MySQLDatabase):
     """

dcclab/image/cziUtil.py

@@ -6,9 +6,7 @@
 import xml.etree.ElementTree as ET
 import os
 import fnmatch
-import multiprocessing
 import warnings
-from concurrent.futures import ThreadPoolExecutor
 
 """
 Python script containing utility functions to be used for handling .czi image.

dcclab/image/image.py

@@ -5,7 +5,6 @@
 import PIL.Image
 import matplotlib.pyplot as plt
 import os
-import re
 from typing import List, Union
 
 

dcclab/ml/__init__.py

@@ -2,7 +2,6 @@
 from typing import List
 import pandas as pd
 import numpy as np
-import json
 import os
 
 """

dcclab/ml/dataset.py

@@ -0,0 +1,70 @@
+import env
+import unittest
+from dcclab.analysis import *
+
+def calculateFactorial(inputQueue, outputQueue):
+    try:
+        value = inputQueue.get_nowait()
+        product = 1
+        for i in range(value):
+            product *= (i+1)
+        outputQueue.put( (value,  product) )
+    except Empty as err:
+        pass # not an error
+
+def slowCalculation(inputQueue, outputQueue):
+    try:
+        value = inputQueue.get_nowait()
+        time.sleep(3)
+        outputQueue.put( value )
+    except Empty as err:
+        pass # not an error
+
+def processSimple(queue):
+    while not queue.empty():
+        try:
+            (n, nfactorial) = queue.get_nowait()
+            print('Just finished calculating {0}!'.format(n))
+        except Empty as err:
+            break # we are done
+
+
+class MyTestCase(env.DCCLabTestCase):
+    def testThreads1(self):
+        N = 11
+        print("Calculating n! for numbers 0 to {0} (every calculation is independent)".format(N - 1))
+        print("======================================================================")
+
+        print("Using threads: fast startup time appropriate for quick calculations")
+        engine = ComputeEngine(useThreads=True)
+        for i in range(N):
+            engine.inputQueue.put(i)
+        engine.compute(target=calculateFactorial)
+
+    def testProcesses1(self):
+        N = 11
+        print("Using processes: long startup time appropriate for longer calculations")
+        engine = ComputeEngine(useThreads=False)
+        for i in range(N):
+            engine.inputQueue.put(i)
+        engine.compute(target=calculateFactorial)
+
+    def testThreadsProcessTask(self):
+        N = 11
+        print("Using threads and replacing the processTaskResult function")
+        engine = ComputeEngine(useThreads=True)
+        for i in range(N):
+            engine.inputQueue.put(i)
+        engine.compute(target=calculateFactorial, processTaskResults=processSimple)
+    @unittest.skip
+    def testProcessesVeryLong(self):
+        N = 11
+        print("Using processes with very long calculations and timeout")
+        engine = ComputeEngine(useThreads=False)
+        for i in range(N):
+            engine.inputQueue.put(i)
+        engine.compute(target=slowCalculation, timeoutInSeconds=2)
+
+
+if __name__ == '__main__':
+    unittest.main()