Merge pull request #134 from GraphStreamingProject/more_options

Compile time option to run L0 Sampling code and better GraphZeppelin configuration

Author: etwest

CMakeLists.txt

@@ -88,9 +88,13 @@ set(BUILD_SHARED_LIBS "${SAVED_BUILD_SHARED_LIBS}" CACHE BOOL "" FORCE)
 
 
 # AVAILABLE COMPILATION DEFINITIONS:
-# VERIFY_SAMPLES_F   Use a deterministic connected-components 
+# VERIFY_SAMPLES_F   Use a deterministic connected-components
 #                    algorithm to verify post-processing.
-# USE_EAGER_DSU      Use the eager DSU query optimization if this flag is present.
+# USE_EAGER_DSU      Use the eager DSU query optimization if
+#                    this flag is present.
+# L0_SAMPLING        Run the CubeSketch l0 sampling algorithm
+#                    to ensure that we sample uniformly.
+#                    Otherwise, run a support finding algorithm.
 
 add_library(GraphZeppelin
   src/graph.cpp

example_streaming.conf

@@ -5,7 +5,6 @@
 // forward declaration
 class Graph;
 
-// TODO: Replace this with an enum defined by GutterTree repo
 enum GutterSystem {
   GUTTERTREE,
   STANDALONE,
@@ -25,15 +24,15 @@ class GraphConfiguration {
   bool _backup_in_mem = true;
 
   // The number of graph workers
-  size_t _num_groups = 1;
-
-  // How many OMP threads each graph worker uses
-  size_t _group_size = 1;
+  size_t _num_graph_workers = 1;
 
   // Option to create more sketches than for standard connected components
   // Ex factor of 1.5, 1.5 times the sketches
   //    factor of 1, normal quantity of sketches
-  double _adtl_skts_factor = 1;
+  double _sketches_factor = 1;
+
+  // Size of update batches as relative to the size of a Supernode
+  double _batch_factor = 1;
 
   // Configuration for the guttering system
   GutteringConfiguration _gutter_conf;
@@ -50,11 +49,11 @@ class GraphConfiguration {
 
   GraphConfiguration& backup_in_mem(bool backup_in_mem);
 
-  GraphConfiguration& num_groups(size_t num_groups);
+  GraphConfiguration& num_graph_workers(size_t num_groups);
 
-  GraphConfiguration& group_size(size_t group_size);
+  GraphConfiguration& sketches_factor(double factor);
 
-  GraphConfiguration& adtl_skts_factor(double factor);
+  GraphConfiguration& batch_factor(double factor);
 
   GutteringConfiguration& gutter_conf();
 

include/graph_configuration.h

@@ -32,8 +32,7 @@ class GraphWorker {
   // manage configuration
   // configuration should be set before calling start_workers
   static int get_num_groups() {return num_groups;} // return the number of GraphWorkers
-  static int get_group_size() {return group_size;} // return the number of threads in each worker
-  static void set_config(int g, int s) { num_groups = g; group_size = s; }
+  static void set_config(int g) { num_groups = g; }
 private:
   /**
    * Create a GraphWorker object by setting metadata and spinning up a thread.
@@ -69,7 +68,6 @@ class GraphWorker {
 
   // configuration
   static int num_groups;
-  static int group_size;
   static long supernode_size;
 
   // list of all GraphWorkers

include/graph_worker.h

@@ -28,7 +28,6 @@ enum SampleSketchRet {
  */
 class Sketch {
  private:
-  static vec_t failure_factor;  // Pr(failure) = 1 / factor. Determines number of columns in sketch.
   static vec_t n;               // Length of the vector this is sketching.
   static size_t num_elems;      // length of our actual arrays in number of elements
   static size_t num_columns;    // Portion of array length, number of columns
@@ -49,7 +48,7 @@ class Sketch {
   FRIEND_TEST(EXPR_Parallelism, N10kU100k);
 
   // Buckets of this sketch.
-  // Length is column_gen(failure_factor) * guess_gen(n).
+  // Length is num_columns * guess_gen(n).
   // For buckets[i * guess_gen(n) + j], the bucket has a 1/2^j probability
   // of containing an index. The first two are pointers into the buckets array.
   alignas(vec_t) char buckets[];
@@ -83,13 +82,12 @@ class Sketch {
 
   /* configure the static variables of sketches
    * @param n               Length of the vector to sketch. (static variable)
-   * @param failure_factor  1/factor = Failure rate for sketch (determines column width)
+   * @param num_columns     Column width, determines the failure probability of the sketch
    * @return nothing
    */
-  inline static void configure(vec_t _n, vec_t _factor) {
+  inline static void configure(vec_t _n, vec_t _num_columns) {
     n = _n;
-    failure_factor = _factor;
-    num_columns = column_gen(failure_factor);
+    num_columns = _num_columns;
     num_guesses = guess_gen(n);
     num_elems = num_columns * num_guesses + 1;  // +1 for zero bucket optimization
   }
@@ -103,8 +101,6 @@ class Sketch {
     return num_elems * (sizeof(vec_t) + sizeof(vec_hash_t));
   }
 
-  inline static vec_t get_failure_factor() { return failure_factor; }
-
   inline void reset_queried() { already_queried = false; }
 
   inline static size_t get_columns() { return num_columns; }
@@ -168,7 +164,7 @@ class Sketch {
 
   // max number of non-zeroes in vector is n/2*n/2=n^2/4
   static size_t guess_gen(size_t x) { return double_to_ull(log2(x) - 2); }
-  static size_t column_gen(size_t d) { return double_to_ull((log2(d) + 1)); }
+  static size_t column_gen(size_t d) { return double_to_ull(ceil(log2(d))); }
 };
 
 class MultipleQueryException : public std::exception {

include/l0_sampling/sketch.h

@@ -88,12 +88,12 @@ class Supernode {
 
   ~Supernode();
 
-  static inline void configure(uint64_t n, vec_t sketch_fail_factor = default_fail_factor,
+  static inline void configure(uint64_t n, vec_t sketch_num_columns = default_num_columns,
                                double skt_factor = 1) {
-    Sketch::configure(n * n, sketch_fail_factor);
-    max_sketches = log2(n) / (log2(3) - 1) * skt_factor;
+    Sketch::configure(n * n, sketch_num_columns);
+    max_sketches = (log2(n) / num_sketches_div) * skt_factor;
     bytes_size = sizeof(Supernode) + max_sketches * Sketch::sketchSizeof();
-    serialized_size = max_sketches * Sketch::serialized_size();
+    serialized_size = max_sketches * Sketch::serialized_size() + sizeof(SerialType);
   }
 
   static inline size_t get_size() {
@@ -210,7 +210,13 @@ class Supernode {
 
   // void write_sparse_binary_range(std::ostream&binary_out, uint32_t beg, uint32_t end);
 
-  static constexpr size_t default_fail_factor = 4;
+#ifdef L0_SAMPLING
+  static constexpr size_t default_num_columns = 7;
+  static constexpr double num_sketches_div = log2(3) - 1;
+#else
+  static constexpr size_t default_num_columns = 2;
+  static constexpr double num_sketches_div = log2(3) - 1;
+#endif
 };