Python 3 persistent_hash was broken. Go back to builtin hash instead.

examples/util/estimate_pi.py

@@ -15,4 +15,4 @@ def map(line, params):
     tot = 0
     for k, v in result_iterator(job.wait()):
         tot += v
-    print (4.0 * tot) / COUNT
+    print ((4.0 * tot) / COUNT)

lib/disco/compat.py

@@ -42,10 +42,6 @@ def sort_cmd(filename, sort_buffer_size):
                 .format(sort_buffer_size, filename), True)
     integer_types = (int)
 
-    from hashlib import md5
-    def persistent_hash(input):
-        return int(md5(str_to_bytes(input)).hexdigest(), 16)
-
 else:
 
     basestring = basestring
@@ -86,6 +82,4 @@ def sort_cmd(filename, sort_buffer_size):
                  "-S", sort_buffer_size,
                  "-o", filename, filename], False)
 
-    persistent_hash = hash
-
     integer_types = (int, long)

lib/disco/fileutils.py

@@ -13,6 +13,9 @@
 HUNK_SIZE       = 1 * MB
 CHUNK_SIZE      = 64 * MB
 
+OS_SEPS         = os.sep + (os.altsep or '')
+
+
 class Chunker(object):
     """
     chunks contain hunks
@@ -137,7 +140,7 @@ def __init__(self):
     def writepath(self, pathname, exclude=()):
         for file in files(pathname):
             name, ext = os.path.splitext(file)
-            if ext not in exclude:
+            if ext not in exclude and file.lstrip(OS_SEPS) not in self.NameToInfo:
                 self.write(file, file)
 
     def writemodule(self, module, arcname=None):

lib/disco/worker/classic/func.py

@@ -7,7 +7,7 @@
 some default values, as well as otherwise useful functions.
 """
 import re
-from disco.compat import pickle_loads, pickle_dumps, bytes_to_str, str_to_bytes, sort_cmd, persistent_hash
+from disco.compat import pickle_loads, pickle_dumps, bytes_to_str, str_to_bytes, sort_cmd
 from disco.error import DataError
 from disco.worker.task_io import *
 
@@ -126,7 +126,7 @@ def init(input_iter, params):
     """
 
 def default_partition(key, nr_partitions, params):
-    return persistent_hash(key) % nr_partitions
+    return hash(key) % nr_partitions
 
 def make_range_partition(min_val, max_val):
     """