From f7013bd25ab522b032c71c012b643ef318589725 Mon Sep 17 00:00:00 2001
From: hugo
-
:man_dancing: https://discolab.ai/ :man_dancing:
-___ +--- + :magic_wand: **DEVELOPERS:** DISCO is written fully in JavaScript/TypeScript. Have a look at our [developer guide](DEV.md). -___ -:question: **WHY DISCO?** +--- + +:question: **WHY DISCO?** + - To build deep learning models across private datasets without compromising data privacy, ownership, sovereignty, or model performance - To create an easy-to-use platform that allows non-specialists to participate in collaborative learning -___ +--- :gear: **HOW DISCO WORKS** -- DISCO has a *public model – private data* approach -- Private and secure model updates – *not data* – are communicated to either: - - a central server : **federated** learning ( :star2: ) - - directly between users : **decentralized** learning ( :sparkles: ) i.e. no central coordination + +- DISCO has a _public model – private data_ approach +- Private and secure model updates – _not data_ – are communicated to either: + - a central server : **federated** learning ( :star2: ) + - directly between users : **decentralized** learning ( :sparkles: ) i.e. no central coordination - Model updates are then securely aggregated into a trained model - See more [HERE](https://discolab.ai/#/information) -___ -:question: **DISCO TECHNOLOGY** +--- + +:question: **DISCO TECHNOLOGY** + - DISCO runs arbitrary deep learning tasks and model architectures in your browser, via [TF.js](https://www.tensorflow.org/js) - Decentralized learning :sparkles: relies on [peer2peer](https://github.com/feross/simple-peer) communication - Have a look at how DISCO ensures privacy and confidentiality [HERE](docs/PRIVACY.md) -___ +--- :test_tube: **RESEARCH-BASED DESIGN** @@ -56,13 +63,13 @@ And more on the roadmap - :mirror: personalizable ([R10](https://arxiv.org/abs/2103.00710)) - :carrot: fairly incentivizing participation -___ - +--- :checkered_flag: **HOW TO USE DISCO** -- Start by exploring our examples tasks in the [`DISCOllaboratives` page](https://discolab.ai/#/list). + +- Start by exploring our examples tasks in the [`DISCOllaboratives` page](https://discolab.ai/#/list). - The example DISCOllaboratives are based on popular ML tasks such as [GPT2](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf), [Titanic](https://www.kaggle.com/c/titanic), [MNIST](https://www.kaggle.com/c/digit-recognizer) or [CIFAR-10](https://www.kaggle.com/pankrzysiu/cifar10-python) - It is also possible to create your own DISCOllaboratives without coding on the [custom training page](https://discolab.ai/#/create): - - Upload the initial model - - Choose between federated and decentralized for your DISCO training scheme ... connect your data and... done! :bar_chart: - - For more details on ML tasks and custom training have a look at [this guide](./docs/TASK.md) + - Upload the initial model + - Choose between federated and decentralized for your DISCO training scheme ... connect your data and... done! :bar_chart: + - For more details on ML tasks and custom training have a look at [this guide](./docs/TASK.md) diff --git a/app.yaml b/app.yaml index 7d12e8c4b..f70c8352d 100644 --- a/app.yaml +++ b/app.yaml @@ -1,8 +1,8 @@ # Custom means that it will use the Dockerfile -runtime: custom +runtime: custom # Flex environment required for WebSocket support, which is required for PeerJS. -env: flex +env: flex # Limit resources to one instance, one CPU, very little memory or disk. instance_class: F1 diff --git a/cli/README.md b/cli/README.md index 1b6589b65..ddf0c358e 100644 --- a/cli/README.md +++ b/cli/README.md @@ -27,23 +27,32 @@ npm -w cli start -- --help # or -h ``` ## Command arguments + Based on the task specification, we can adjust the command arguments. Available arguments are listed below. Non-mandatory fields will automatically use values from the task specification. + ### Test specification arguments + - `testID`: (mandatory) arbitrary test ID defined by the user for the test run - `task`: (mandatory) pre-defined task (adding a new task is described in the next section) - `numberOfUsers`: number of users participating in the learning round - `save`: whether to save the logs of the test run + ### Learning hyperparameters + - `epochs`: total number of training epochs - `roundDuration`: number of epochs per round - `batchSize`: batch size - `validationSplit`: ratio of the validation set used for evaluation + ### Aggregator parameters + - `aggregator`: aggregator specification - `clippingRadius`, `maxIterations`, `beta`: (optional, for byzantine aggregator settings) byzantine aggregator hyperparameters - `maxShareValue`: (optional, for secure aggregator settings) secure aggregator hyperparameter + ### Differential Privacy parameters + - `epsilon`, `delta`, `dpDefaultClippingRadius`: (optional, for testing with differential privacy) differential privacy hyperparameters ## Adding new tasks @@ -80,10 +89,12 @@ The CLI includes a script to evaluate GPT models on the [HellaSwag](https://rowa To run the evaluation: `npm -w cli run hellaswag_gpt` The script benchmarks the following models: + - A TensorFlow.js implementation of GPT (`gpt-tfjs`) - A pre-trained ONNX model (`Xenova/gpt2`) Both models are evaluated using a shared tokenizer (`Xenova/gpt2`), and the script reports: + - Accuracy (proportion of correct multiple-choice predictions) - Total evaluation time (in seconds) @@ -91,7 +102,6 @@ Both models are evaluated using a shared tokenizer (`Xenova/gpt2`), and the scri Results are printed to the console and saved to a log file: `../datasets/logFile_hellaswag.txt` - This allows for a direct comparison between the inference performance and accuracy of the two architectures. The TFJS implementation is generally slower and more memory-intensive than ONNX, but offers compatibility with browser-based environments and custom training workflows. See the [Benchmarking GPT-TF.js](#benchmarking-gpt-tfjs) section for more details on performance tradeoffs. diff --git a/cli/package.json b/cli/package.json index cc0f741e2..fb40ae17c 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,26 +1,26 @@ { - "name": "cli", - "private": true, - "type": "module", - "main": "dist/cli.js", - "scripts": { - "watch": "nodemon --ext ts --ignore dist --watch ../discojs-node/dist --watch ../server/dist --watch . --exec npm run", - "start": "npm run build && node dist/cli.js", - "benchmark_gpt": "npm run build && node dist/benchmark_gpt.js", - "train_gpt": "npm run build && node dist/train_gpt.js", - "hellaswag_gpt": "npm run build && node dist/hellaswag_gpt.js", - "build": "tsc --build", - "test": ": nothing" - }, - "author": "", - "license": "ISC", - "dependencies": { - "@epfml/discojs-node": "*", - "server": "*", - "tslib": "2" - }, - "devDependencies": { - "nodemon": "3", - "ts-command-line-args": "2" - } + "name": "cli", + "private": true, + "type": "module", + "main": "dist/cli.js", + "scripts": { + "watch": "nodemon --ext ts --ignore dist --watch ../discojs-node/dist --watch ../server/dist --watch . --exec npm run", + "start": "npm run build && node dist/cli.js", + "benchmark_gpt": "npm run build && node dist/benchmark_gpt.js", + "train_gpt": "npm run build && node dist/train_gpt.js", + "hellaswag_gpt": "npm run build && node dist/hellaswag_gpt.js", + "build": "tsc --build", + "test": ": nothing" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@epfml/discojs-node": "*", + "server": "*", + "tslib": "2" + }, + "devDependencies": { + "nodemon": "3", + "ts-command-line-args": "2" + } } diff --git a/cli/src/args.ts b/cli/src/args.ts index ced893a72..e74d623bb 100644 --- a/cli/src/args.ts +++ b/cli/src/args.ts @@ -1,61 +1,95 @@ -import { parse } from 'ts-command-line-args' -import { Map, Set } from 'immutable' +import { parse } from "ts-command-line-args"; +import { Map, Set } from "immutable"; import type { DataType, Network, TaskProvider } from "@epfml/discojs"; -import { defaultTasks } from '@epfml/discojs' +import { defaultTasks } from "@epfml/discojs"; type AggregationStrategy = "mean" | "byzantine" | "secure"; -function parseAggregator(raw: string): AggregationStrategy{ - if (raw === "mean" || raw == "byzantine" || raw == "secure") - return raw; - else - throw new Error(`Aggregator ${raw} is not supported.`); +function parseAggregator(raw: string): AggregationStrategy { + if (raw === "mean" || raw == "byzantine" || raw == "secure") return raw; + else throw new Error(`Aggregator ${raw} is not supported.`); } export interface BenchmarkArguments { - provider: TaskProvider