Merge pull request #162 from fanlai0990/master

[Async FL] Support Papaya implementation

Author: fanlai0990

README.md

@@ -60,7 +60,7 @@ Now that you have FedScale installed, you can start exploring FedScale following
 
 ***We are adding more datasets! Please contribute!***
 
-FedScale consists of 20+ large-scale, heterogeneous FL datasets covering computer vision (CV), natural language processing (NLP), and miscellaneous tasks. 
+FedScale consists of 20+ large-scale, heterogeneous FL datasets and 70+ various [models](./fedscale/utils/models/cv_models/README.md), covering computer vision (CV), natural language processing (NLP), and miscellaneous tasks. 
 Each one is associated with its training, validation, and testing datasets. 
 We acknowledge the contributors of these raw datasets. Please go to the `./benchmark/dataset` directory and follow the dataset [README](./benchmark/dataset/README.md) for more details.
 

benchmark/configs/async_fl/async_fl.yml

@@ -8,7 +8,7 @@ ps_ip: localhost
 # Note that if we collocate ps and worker on same GPU, then we need to decrease this number of available processes on that GPU by 1
 # E.g., master node has 4 available processes, then 1 for the ps, and worker should be set to: worker:3
 worker_ips:
-    - localhost:[2]
+    - localhost:[2,2,2,2]
 
 exp_path: $FEDSCALE_HOME/fedscale/core
 
@@ -28,30 +28,34 @@ setup_commands:
 # ========== Additional job configuration ==========
 # Default parameters are specified in config_parser.py, wherein more description of the parameter can be found
 
+# NOTE: We are supporting and improving the following implementation (Async FL) in FedScale:
+    # - "PAPAYA: Practical, Private, and Scalable Federated Learning", MLSys, 2022
+    # - "Federated Learning with Buffered Asynchronous Aggregation", AISTATS, 2022
+
+# We appreciate you to contribute and/or report bugs. Thank you!
+
 job_conf:
-    - job_name: femnist                   # Generate logs under this folder: log_path/job_name/time_stamp
-    - log_path: $FEDSCALE_HOME/benchmark # Path of log files
-    - num_participants: 800                      # Number of participants per round, we use K=100 in our paper, large K will be much slower
-    - data_set: femnist                     # Dataset: openImg, google_speech, stackoverflow
+    - job_name: async_femnist                    # Generate logs under this folder: log_path/job_name/time_stamp
+    - log_path: $FEDSCALE_HOME/benchmark   # Path of log files
+    - data_set: femnist                    # Dataset: openImg, google_speech, stackoverflow
     - data_dir: $FEDSCALE_HOME/benchmark/dataset/data/femnist    # Path of the dataset
     - data_map_file: $FEDSCALE_HOME/benchmark/dataset/data/femnist/client_data_mapping/train.csv              # Allocation of data to each client, turn to iid setting if not provided
     - device_conf_file: $FEDSCALE_HOME/benchmark/dataset/data/device_info/client_device_capacity     # Path of the client trace
     - device_avail_file: $FEDSCALE_HOME/benchmark/dataset/data/device_info/client_behave_trace
-    - model: shufflenet_v2_x2_0                            # Models: e.g., shufflenet_v2_x2_0, mobilenet_v2, resnet34, albert-base-v2
-    - eval_interval: 20                     # How many rounds to run a testing on the testing set
-    - rounds: 500                          # Number of rounds to run this training. We use 1000 in our paper, while it may converge w/ ~400 rounds
+    - model: resnet18            # NOTE: Please refer to our model zoo README and use models for these small image (e.g., 32x32x3) inputs
+#    - model_zoo: fedscale-zoo
+    - eval_interval: 5                     # How many rounds to run a testing on the testing set
+    - rounds: 1000                          # Number of rounds to run this training. We use 1000 in our paper, while it may converge w/ ~400 rounds
     - filter_less: 21                       # Remove clients w/ less than 21 samples
     - num_loaders: 2
-    - local_steps: 20
+    - local_steps: 5
     - learning_rate: 0.05
     - batch_size: 20
     - test_bsz: 20
-    - use_cuda: False
-    - decay_round: 50
+    - ps_port: 12342
+    - use_cuda: True
     - overcommitment: 1.0
-    - async_buffer: 10
-    - arrival_interval: 3
-
-
-
-
+    - arrival_interval: 5
+    - max_staleness: 5
+    - max_concurrency: 100
+    - async_buffer: 50                     # Number of updates need to be aggregated before generating new model version

benchmark/configs/cifar_cpu/cifar_cpu.yml

@@ -34,13 +34,14 @@ job_conf:
     - num_participants: 4                      # Number of participants per round, we use K=100 in our paper, large K will be much slower
     - data_set: cifar10                     # Dataset: openImg, google_speech, stackoverflow
     - data_dir: $FEDSCALE_HOME/benchmark/dataset/data/    # Path of the dataset
-    - model: shufflenet_v2_x2_0                            # Models: e.g., shufflenet_v2_x2_0, mobilenet_v2, resnet34, albert-base-v2# - gradient_policy: yogi                 # {"fed-yogi", "fed-prox", "fed-avg"}, "fed-avg" by default
+    - model: shufflenet_v2_x2_0              # NOTE: Please refer to our model zoo README and use models for these small image (e.g., 32x32x3) inputs
+#    - model_zoo: fedscale-zoo              # Default zoo (torchcv) uses the pytorchvision zoo, which can not support small images well
     - eval_interval: 5                     # How many rounds to run a testing on the testing set
     - rounds: 600                          # Number of rounds to run this training. We use 1000 in our paper, while it may converge w/ ~400 rounds
     - filter_less: 0                       # Remove clients w/ less than 21 samples
     - num_loaders: 2
     - local_steps: 20
-    - learning_rate: 0.001
+    - learning_rate: 0.05
     - batch_size: 32
     - test_bsz: 32
     - use_cuda: False

benchmark/configs/femnist/conf.yml

@@ -2,13 +2,13 @@
 
 # ========== Cluster configuration ========== 
 # ip address of the parameter server (need 1 GPU process)
-ps_ip: localhost
+ps_ip: 10.0.0.1
 
 # ip address of each worker:# of available gpus process on each gpu in this node
 # Note that if we collocate ps and worker on same GPU, then we need to decrease this number of available processes on that GPU by 1
 # E.g., master node has 4 available processes, then 1 for the ps, and worker should be set to: worker:3
-worker_ips: 
-    - localhost:[2]
+worker_ips:
+    - 10.0.0.1:[4]
 
 exp_path: $FEDSCALE_HOME/fedscale/core
 
@@ -31,24 +31,21 @@ setup_commands:
 job_conf: 
     - job_name: femnist                   # Generate logs under this folder: log_path/job_name/time_stamp
     - log_path: $FEDSCALE_HOME/benchmark # Path of log files
-    - num_participants: 20                      # Number of participants per round, we use K=100 in our paper, large K will be much slower
+    - num_participants: 50                  # Number of participants per round, we use K=100 in our paper, large K will be much slower
     - data_set: femnist                     # Dataset: openImg, google_speech, stackoverflow
     - data_dir: $FEDSCALE_HOME/benchmark/dataset/data/femnist    # Path of the dataset
     - data_map_file: $FEDSCALE_HOME/benchmark/dataset/data/femnist/client_data_mapping/train.csv              # Allocation of data to each client, turn to iid setting if not provided
     - device_conf_file: $FEDSCALE_HOME/benchmark/dataset/data/device_info/client_device_capacity     # Path of the client trace
     - device_avail_file: $FEDSCALE_HOME/benchmark/dataset/data/device_info/client_behave_trace
-    - model: shufflenet_v2_x2_0                            # Models: e.g., shufflenet_v2_x2_0, mobilenet_v2, resnet34, albert-base-v2
-    - gradient_policy: yogi                 # {"fed-yogi", "fed-prox", "fed-avg"}, "fed-avg" by default
-    - eval_interval: 30                     # How many rounds to run a testing on the testing set
-    - rounds: 5000                          # Number of rounds to run this training. We use 1000 in our paper, while it may converge w/ ~400 rounds
+    - model: resnet18             # NOTE: Please refer to our model zoo README and use models for these small image (e.g., 32x32x3) inputs
+#    - model_zoo: fedscale-zoo
+    - eval_interval: 10                     # How many rounds to run a testing on the testing set
+    - rounds: 1000                          # Number of rounds to run this training. We use 1000 in our paper, while it may converge w/ ~400 rounds
     - filter_less: 21                       # Remove clients w/ less than 21 samples
     - num_loaders: 2
-    - yogi_eta: 3e-3 
-    - yogi_tau: 1e-8
-    - local_steps: 20
+    - local_steps: 5
     - learning_rate: 0.05
     - batch_size: 20
     - test_bsz: 20
-    - malicious_factor: 4
-    - use_cuda: False
+    - use_cuda: True