diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml index ebe0047..df3290a 100644 --- a/.github/workflows/test-and-deploy.yml +++ b/.github/workflows/test-and-deploy.yml @@ -7,33 +7,24 @@ on: concurrency: deploy-numerai-cli-${{ github.ref }} jobs: - build-python36: - name: Build Python 3.6 - runs-on: ubuntu-20.04 + build-python: + name: Build Python ${{ matrix.python-version }} + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: "3.6.15" - - run: pip install . - - run: numerai copy-example - - run: test -e tournament-python3/predict.py - - build-python37: - name: Build Python 3.7 - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 - with: - python-version: "3.7.9" + python-version: ${{ matrix.python-version }} - run: pip install . - run: numerai copy-example - run: test -e tournament-python3/predict.py build-docker: name: Build Docker - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - run: | @@ -44,14 +35,14 @@ jobs: pypi-release: name: PyPI Release - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' - needs: [build-python36, build-python37, build-docker] + needs: [build-python, build-docker] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: - python-version: "3.7.9" + python-version: "3.12" - run: pip install twine wheel - run: python setup.py sdist bdist_wheel - run: | diff --git a/.gitignore b/.gitignore index 4b7bb86..2677034 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,7 @@ terraform.tfstate* .idea/ .DS_STORE + +tournament-python*/ +signals-python*/ +crypto-python*/ \ No newline at end of file diff --git a/numerai/cli/destroy_all.py b/numerai/cli/destroy_all.py index 8834dbc..74747d8 100644 --- a/numerai/cli/destroy_all.py +++ b/numerai/cli/destroy_all.py @@ -62,7 +62,7 @@ def destroy_all(ctx, verbose, preserve_node_config): verbose, provider, env_vars=provider_keys, - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) except Exception as e: diff --git a/numerai/cli/node/destroy.py b/numerai/cli/node/destroy.py index a09a480..fb92036 100644 --- a/numerai/cli/node/destroy.py +++ b/numerai/cli/node/destroy.py @@ -62,7 +62,7 @@ def destroy(ctx, preserve_node_config, verbose): verbose, provider, env_vars=provider_keys, - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) except Exception as e: diff --git a/numerai/cli/uninstall.py b/numerai/cli/uninstall.py index 862f9f8..36b6451 100644 --- a/numerai/cli/uninstall.py +++ b/numerai/cli/uninstall.py @@ -55,7 +55,7 @@ def uninstall(): provider=provider, verbose=True, env_vars=provider_keys, - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) click.secho("cleaning up docker images...") diff --git a/numerai/cli/upgrade.py b/numerai/cli/upgrade.py index 9cc2759..e27cb94 100644 --- a/numerai/cli/upgrade.py +++ b/numerai/cli/upgrade.py @@ -210,7 +210,7 @@ def upgrade(verbose): verbose, provider="aws", env_vars=load_or_init_keys("aws"), - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) click.secho("Upgrade complete!", fg="green") diff --git a/numerai/cli/util/debug.py b/numerai/cli/util/debug.py index 91b640a..9b52f40 100644 --- a/numerai/cli/util/debug.py +++ b/numerai/cli/util/debug.py @@ -151,6 +151,18 @@ def root_cause(std_out, err_msg): "mode: https://docs.docker.com/engine/security/rootless/" ) + if ( + "RepositoryAlreadyExistsException" in all_logs + or "EntityAlreadyExists" in all_logs + or "ResourceAlreadyExistsException" in all_logs + ): + raise exception_with_msg( + "AWS resources with the names this node expects already exist in your account, " + "but they are not tracked in the current local Terraform state. " + "This usually happens after a partial previous apply. " + "Delete or import the existing AWS resources shown above, then retry." + ) + # these are non-errors that either shouldn't be handled or are handled elsewhere if b"Can't update submission after deadline" in err_msg: return diff --git a/numerai/cli/util/docker.py b/numerai/cli/util/docker.py index 86aafe6..541b362 100644 --- a/numerai/cli/util/docker.py +++ b/numerai/cli/util/docker.py @@ -69,7 +69,6 @@ def execute(command, verbose, censor_substr=None): shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - bufsize=1, close_fds=on_posix, ) stdout_q = Queue() diff --git a/numerai/cli/util/terraform.py b/numerai/cli/util/terraform.py index 0f23b4d..20989d5 100644 --- a/numerai/cli/util/terraform.py +++ b/numerai/cli/util/terraform.py @@ -19,7 +19,7 @@ def apply_terraform(nodes_config, affected_providers, provider, verbose): verbose, affected_provider, env_vars=load_or_init_keys(affected_provider), - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) else: click.secho(f"provider {affected_provider} not supported", fg="red") @@ -51,7 +51,7 @@ def create_azure_registry(provider, provider_keys, verbose): verbose, "azure", env_vars=provider_keys, - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) res = terraform("output -json acr_repo_details", True, provider).decode("utf-8") return json.loads(res) @@ -64,13 +64,13 @@ def create_gcp_registry(provider, verbose): 'apply -target="google_project_service.cloud_resource_manager" -auto-approve ', verbose, "gcp", - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) terraform( 'apply -target="google_artifact_registry_repository.registry[0]" -auto-approve ', verbose, "gcp", - inputs={"node_config_file": "nodes.json"}, + inputs={"node_config_file": "../nodes.json"}, ) res = terraform("output -json artifact_registry_details", True, provider).decode( "utf-8" diff --git a/numerai/examples/crypto-python3/Dockerfile b/numerai/examples/crypto-python3/Dockerfile index 8ec60d7..23d1a09 100644 --- a/numerai/examples/crypto-python3/Dockerfile +++ b/numerai/examples/crypto-python3/Dockerfile @@ -1,5 +1,5 @@ # Provides us a working Python 3 environment. -FROM python:3.9 +FROM python:3.13 # These are docker arguments that `numerai node deploy/test` will always pass into docker. # They are then set in your environment so that numerapi can access them when uploading submissions. diff --git a/numerai/examples/crypto-python3/predict.py b/numerai/examples/crypto-python3/predict.py index ebd4232..c233ff7 100644 --- a/numerai/examples/crypto-python3/predict.py +++ b/numerai/examples/crypto-python3/predict.py @@ -11,7 +11,8 @@ logging.basicConfig(filename="log.txt", filemode="a") TOURNAMENT = 12 -DATA_VERSION = "crypto/v1.0" +DATA_VERSION = "crypto/v2.0" +TARGET_COL = "target_binned_return_20" TRAINED_MODEL_PREFIX = "./trained_model" DEFAULT_MODEL_ID = None @@ -22,7 +23,7 @@ # these are set by the docker image that you deploy after training, # but you can also set them manually above for local testing MODEL_ID = os.getenv("MODEL_ID", DEFAULT_MODEL_ID) -napi = numerapi.NumerAPI( +napi = numerapi.CryptoAPI( public_id=os.getenv("NUMERAI_PUBLIC_ID", DEFAULT_PUBLIC_ID), secret_key=os.getenv("NUMERAI_SECRET_KEY", DEFAULT_SECRET_KEY), ) @@ -40,63 +41,44 @@ def train(napi, model_id, force_training=False): return model logging.info("reading training data") - napi.download_dataset(f"{DATA_VERSION}/train_targets.parquet") - target = pd.read_parquet(f"{DATA_VERSION}/train_targets.parquet") - - # TODO: implement get_features and train a model - # This will take a few minutes 🍵 - # logging.info("training model") - # model = lgbm.LGBMRegressor( - # n_estimators=2000, - # learning_rate=0.01, - # max_depth=5, - # num_leaves=2**5-1, - # colsample_bytree=0.1 - # ) - # model.fit( - # train_data[feature_cols], - # train_data["target"] - # ) - - # logging.info("saving model") - # joblib.dump(model, model_name) - # return model - - # just return the target for now - return target + napi.download_dataset(f"{DATA_VERSION}/train.parquet") + train_data = pd.read_parquet(f"{DATA_VERSION}/train.parquet") + feature_cols = [col for col in train_data.columns if col.startswith("feature_")] + + logging.info("training model") + model = lgbm.LGBMRegressor( + n_estimators=2000, + learning_rate=0.01, + max_depth=5, + num_leaves=2**5 - 1, + colsample_bytree=0.1, + ) + model.fit(train_data[feature_cols], train_data[TARGET_COL]) + + logging.info("saving model") + joblib.dump(model, model_name) + return model def predict(napi, model): logging.info("reading prediction data") - napi.download_dataset(f"{DATA_VERSION}/live_universe.parquet") - live_universe = pd.read_parquet(f"{DATA_VERSION}/live_universe.parquet") - - # TODO: implement get_features and predict the target - # logging.info("generating predictions") - # predictions = model.predict(get_features(live_universe)) - # predictions = pd.DataFrame( - # predictions, columns=["prediction"], index=predict_data.index - # ) - # return predictions - - # just return the latest target for now - napi.download_dataset(f"{DATA_VERSION}/train_targets.parquet") - target = pd.read_parquet(f"{DATA_VERSION}/train_targets.parquet") - return ( - target[target.date == target.date.max()] - .drop(columns=["date"]) - .rename(columns={"target": "prediction"}) - .set_index("symbol") + napi.download_dataset(f"{DATA_VERSION}/live.parquet") + predict_data = pd.read_parquet(f"{DATA_VERSION}/live.parquet") + feature_cols = [col for col in predict_data.columns if col.startswith("feature_")] + + logging.info("generating predictions") + predictions = model.predict(predict_data[feature_cols]) + predictions = pd.DataFrame( + predictions, columns=["prediction"], index=predict_data.index ) + return predictions def submit(predictions, predict_output_path="predictions.csv", model_id=None): logging.info("writing predictions to file and submitting") include_index = predictions.index.name is not None predictions.to_csv(predict_output_path, index=include_index) - napi.upload_predictions( - predict_output_path, model_id=model_id, tournament=TOURNAMENT - ) + napi.upload_predictions(predict_output_path, model_id=model_id) if __name__ == "__main__": diff --git a/numerai/examples/crypto-python3/requirements.txt b/numerai/examples/crypto-python3/requirements.txt index d862c27..da7ba24 100644 --- a/numerai/examples/crypto-python3/requirements.txt +++ b/numerai/examples/crypto-python3/requirements.txt @@ -1,5 +1,6 @@ -numerapi==2.14.0 -pandas==2.1.3 -pyarrow==7.0.0 -joblib==1.2.0 -lightgbm==3.3.1 \ No newline at end of file +numerapi==2.22.0 +pandas==2.3.3 +pyarrow==18.1.0 +joblib==1.5.3 +lightgbm==4.5.0 +scikit-learn==1.6.1 diff --git a/numerai/examples/signals-python3/Dockerfile b/numerai/examples/signals-python3/Dockerfile index 8ec60d7..23d1a09 100644 --- a/numerai/examples/signals-python3/Dockerfile +++ b/numerai/examples/signals-python3/Dockerfile @@ -1,5 +1,5 @@ # Provides us a working Python 3 environment. -FROM python:3.9 +FROM python:3.13 # These are docker arguments that `numerai node deploy/test` will always pass into docker. # They are then set in your environment so that numerapi can access them when uploading submissions. diff --git a/numerai/examples/signals-python3/predict.py b/numerai/examples/signals-python3/predict.py index bfa6d78..ea82ae1 100644 --- a/numerai/examples/signals-python3/predict.py +++ b/numerai/examples/signals-python3/predict.py @@ -11,7 +11,7 @@ logging.basicConfig(filename="log.txt", filemode="a") TOURNAMENT = 11 -DATA_VERSION = "signals/v1.0" +DATA_VERSION = "signals/v2.0" TARGET_COL = "target" TRAINED_MODEL_PREFIX = "./trained_model" @@ -23,7 +23,7 @@ # these are set by the docker image that you deploy after training, # but you can also set them manually above for local testing MODEL_ID = os.getenv("MODEL_ID", DEFAULT_MODEL_ID) -napi = numerapi.NumerAPI( +napi = numerapi.SignalsAPI( public_id=os.getenv("NUMERAI_PUBLIC_ID", DEFAULT_PUBLIC_ID), secret_key=os.getenv("NUMERAI_SECRET_KEY", DEFAULT_SECRET_KEY), ) @@ -93,9 +93,7 @@ def submit(predictions, predict_output_path="predictions.csv", model_id=None): include_index = predictions.index.name is not None predictions.to_csv(predict_output_path, index=include_index) print(predictions) - napi.upload_predictions( - predict_output_path, model_id=model_id, tournament=TOURNAMENT - ) + napi.upload_predictions(predict_output_path, model_id=model_id) if __name__ == "__main__": diff --git a/numerai/examples/signals-python3/requirements.txt b/numerai/examples/signals-python3/requirements.txt index d862c27..da7ba24 100644 --- a/numerai/examples/signals-python3/requirements.txt +++ b/numerai/examples/signals-python3/requirements.txt @@ -1,5 +1,6 @@ -numerapi==2.14.0 -pandas==2.1.3 -pyarrow==7.0.0 -joblib==1.2.0 -lightgbm==3.3.1 \ No newline at end of file +numerapi==2.22.0 +pandas==2.3.3 +pyarrow==18.1.0 +joblib==1.5.3 +lightgbm==4.5.0 +scikit-learn==1.6.1 diff --git a/numerai/examples/tournament-python3/Dockerfile b/numerai/examples/tournament-python3/Dockerfile index 8ec60d7..23d1a09 100644 --- a/numerai/examples/tournament-python3/Dockerfile +++ b/numerai/examples/tournament-python3/Dockerfile @@ -1,5 +1,5 @@ # Provides us a working Python 3 environment. -FROM python:3.9 +FROM python:3.13 # These are docker arguments that `numerai node deploy/test` will always pass into docker. # They are then set in your environment so that numerapi can access them when uploading submissions. diff --git a/numerai/examples/tournament-python3/predict.py b/numerai/examples/tournament-python3/predict.py index 069ada3..8d6d5b8 100644 --- a/numerai/examples/tournament-python3/predict.py +++ b/numerai/examples/tournament-python3/predict.py @@ -11,10 +11,9 @@ logging.basicConfig(filename="log.txt", filemode="a") TOURNAMENT = 8 -DATA_VERSION = "v4.1" +DATA_VERSION = "v5.2" ERA_COL = "era" -DATA_TYPE_COL = "data_type" -TARGET_COL = "target_nomi_v4_20" +TARGET_COL = "target" TRAINED_MODEL_PREFIX = "./trained_model" DEFAULT_MODEL_ID = None @@ -35,11 +34,7 @@ def get_features(napi): napi.download_dataset(f"{DATA_VERSION}/features.json") with open(f"{DATA_VERSION}/features.json", "r") as f: feature_metadata = json.load(f) - return feature_metadata["feature_sets"]["small"] + [ - ERA_COL, - DATA_TYPE_COL, - TARGET_COL, - ] + return feature_metadata["feature_sets"]["small"] def train(napi, model_id, force_training=False): @@ -56,7 +51,8 @@ def train(napi, model_id, force_training=False): logging.info("reading training data") napi.download_dataset(f"{DATA_VERSION}/train.parquet") train_data = pd.read_parquet( - f"{DATA_VERSION}/train.parquet", columns=get_features(napi) + f"{DATA_VERSION}/train.parquet", + columns=[ERA_COL] + get_features(napi) + [TARGET_COL], ) # This will take a few minutes 🍵 @@ -97,9 +93,7 @@ def submit(predictions, predict_output_path="predictions.csv", model_id=None): logging.info("writing predictions to file and submitting") include_index = predictions.index.name is not None predictions.to_csv(predict_output_path, index=include_index) - napi.upload_predictions( - predict_output_path, model_id=model_id, tournament=TOURNAMENT - ) + napi.upload_predictions(predict_output_path, model_id=model_id) if __name__ == "__main__": diff --git a/numerai/examples/tournament-python3/requirements.txt b/numerai/examples/tournament-python3/requirements.txt index d862c27..da7ba24 100644 --- a/numerai/examples/tournament-python3/requirements.txt +++ b/numerai/examples/tournament-python3/requirements.txt @@ -1,5 +1,6 @@ -numerapi==2.14.0 -pandas==2.1.3 -pyarrow==7.0.0 -joblib==1.2.0 -lightgbm==3.3.1 \ No newline at end of file +numerapi==2.22.0 +pandas==2.3.3 +pyarrow==18.1.0 +joblib==1.5.3 +lightgbm==4.5.0 +scikit-learn==1.6.1 diff --git a/numerai/terraform/aws/-inputs.tf b/numerai/terraform/aws/-inputs.tf index 52c4624..e6892e0 100644 --- a/numerai/terraform/aws/-inputs.tf +++ b/numerai/terraform/aws/-inputs.tf @@ -13,7 +13,7 @@ variable "az_count" { variable "node_config_file" { description = "Path to the json file used to configure nodes" type = string - default = "nodes.json" + default = "../nodes.json" } variable "node_container_port" { diff --git a/numerai/terraform/aws/-main.tf b/numerai/terraform/aws/-main.tf index 5e1652c..aeb9d17 100644 --- a/numerai/terraform/aws/-main.tf +++ b/numerai/terraform/aws/-main.tf @@ -1,5 +1,12 @@ terraform { required_version = "1.5.6" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">=5.0, <6.0" + } + } } # Specify the provider and access details diff --git a/numerai/terraform/aws/aws/cluster.tf b/numerai/terraform/aws/aws/cluster.tf index 088513b..23ac126 100644 --- a/numerai/terraform/aws/aws/cluster.tf +++ b/numerai/terraform/aws/aws/cluster.tf @@ -86,6 +86,8 @@ resource "aws_iam_role_policy_attachment" "aws_batch_service_role" { data "aws_ami" "ecs_al2" { most_recent = true + owners = ["amazon"] + filter { name = "name" values = ["amzn2-ami-ecs-hvm-*-x86_64-ebs"] @@ -110,7 +112,7 @@ resource "aws_launch_template" "node" { } resource "aws_batch_compute_environment" "node" { - compute_environment_name_prefix = "${local.node_prefix}-" + compute_environment_name = "${local.node_prefix}-compute" compute_resources { instance_role = aws_iam_instance_profile.batch_ecs_instance_role.arn @@ -179,21 +181,20 @@ resource "aws_batch_job_definition" "node" { retry_strategy { attempts = 2 evaluate_on_exit { - on_reason = "CannotInspectContainerError:*" - action = "RETRY" - } - evaluate_on_exit { - on_reason = "CannotPullContainerError:*" - action = "RETRY" - } - evaluate_on_exit { - action = "RETRY" - on_reason = "CannotStartContainerError:*" + # Should catch load and throttling related issues like: + # - CannotPullContainerError + # - CannotStartContainerError + # - CannotInspectContainerError + # - CannotCreateContainerError + # - ThrottlingException + on_status_reason = "Task failed to start" + action = "RETRY" } evaluate_on_exit { - action = "RETRY" - on_reason = "Task failed to start" + on_status_reason = "DockerTimeoutError*" + action = "RETRY" } + evaluate_on_exit { action = "EXIT" on_reason = "*" diff --git a/numerai/terraform/azure/input.tf b/numerai/terraform/azure/input.tf index 0a78367..27df80d 100644 --- a/numerai/terraform/azure/input.tf +++ b/numerai/terraform/azure/input.tf @@ -9,7 +9,7 @@ variable "az_resource_group_location" { variable "node_config_file" { description = "Path to the json file used to configure nodes" type = string - default = "nodes.json" + default = "../nodes.json" } variable "node_container_port" { diff --git a/numerai/terraform/gcp/-inputs.tf b/numerai/terraform/gcp/-inputs.tf index e6eb26e..821759e 100644 --- a/numerai/terraform/gcp/-inputs.tf +++ b/numerai/terraform/gcp/-inputs.tf @@ -7,5 +7,5 @@ variable "region" { variable "node_config_file" { description = "Path to the json file used to configure nodes" type = string - default = "nodes.json" + default = "../nodes.json" } diff --git a/setup.py b/setup.py index 9d50fa9..5b8cd6f 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name="numerai-cli", - version="1.1.3", + version="1.1.4", description="A library for deploying Numer.ai Prediction Nodes.", url="https://github.com/numerai/numerai-cli", author="Numer.ai",