Merge branch 'develop'

Author: pythonlessons

.vscode/launch.json

@@ -10,7 +10,7 @@
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
-            "justMyCode": true
+            "justMyCode": false
         }
     ]
 }

CHANGELOG.md

@@ -1,3 +1,20 @@
+## [0.5.0] - 2024-01-30
+### Added:
+- Added `MACD` indicator to `indicators` file.
+- Added `reward.AccountValueChangeReward` object to calculate reward based on the change in the account value.
+- Added `scalers.ZScoreScaler` that doesn't require min and max to transform data, but uses mean and std instead.
+- Added `ActionSpace` object to handle the action space of the agent.
+- Added support for continuous actions. (float values between 0 and 1)
+
+### Changed:
+- Updated all indicators to have `config` parameter, that we can use so we can serialize the indicators. (save/load configurations to/from file)
+- Changed `reward.simpleReward` to `reward.SimpleReward` Object.
+- Updated `state.State` to have `open`, `high`, `low`, `close` and `volume` attributes.
+- Updated `data_feeder.PdDataFeeder` to be serializable by including `save_config` and `load_config` methods.
+- Included trading fees into `trading_env.TradingEnv` object.
+- Updated `trading_env.TradingEnv` to have `reset` method, which resets the environment to the initial state.
+- Included `save_config` and `load_config` methods into `trading_env.TradingEnv` object, so we can save/load the environment configuration.
+
 ## [0.4.0] - 2024-01-02
 ### Added:
 - Created `indicators` file, where I added `BolingerBands`, `RSI`, `PSAR`, `SMA` indicators

experiments/playing_random_sinusoid.py

@@ -4,9 +4,10 @@
 from finrock.data_feeder import PdDataFeeder
 from finrock.trading_env import TradingEnv
 from finrock.render import PygameRender
-from finrock.scalers import MinMaxScaler
-from finrock.reward import simpleReward
-from finrock.indicators import BolingerBands, SMA, RSI, PSAR
+from finrock.scalers import ZScoreScaler
+from finrock.reward import AccountValueChangeReward
+from finrock.indicators import BolingerBands, SMA, RSI, PSAR, MACD
+from finrock.metrics import DifferentActions, AccountValue, MaxDrawdown, SharpeRatio
 
 df = pd.read_csv('Datasets/random_sinusoid.csv')
 
@@ -16,23 +17,30 @@
         BolingerBands(data=df, period=20, std=2),
         RSI(data=df, period=14),
         PSAR(data=df),
+        MACD(data=df),
         SMA(data=df, period=7),
-        SMA(data=df, period=25),
-        SMA(data=df, period=99),
     ]
 )
 
 env = TradingEnv(
     data_feeder = pd_data_feeder,
-    output_transformer = MinMaxScaler(min=pd_data_feeder.min, max=pd_data_feeder.max),
+    output_transformer = ZScoreScaler(),
     initial_balance = 1000.0,
     max_episode_steps = 1000,
     window_size = 50,
-    reward_function = simpleReward
+    reward_function = AccountValueChangeReward(),
+    metrics = [
+        DifferentActions(),
+        AccountValue(),
+        MaxDrawdown(),
+        SharpeRatio(),
+    ]
 )
 action_space = env.action_space
 input_shape = env.observation_space.shape
 
+env.save_config()
+
 pygameRender = PygameRender(frame_rate=60)
 
 state, info = env.reset()

experiments/testing_ppo_sinusoid_continuous.py

@@ -0,0 +1,45 @@
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+tf.get_logger().setLevel('ERROR')
+for gpu in tf.config.experimental.list_physical_devices('GPU'):
+    tf.config.experimental.set_memory_growth(gpu, True)
+
+from finrock.data_feeder import PdDataFeeder
+from finrock.trading_env import TradingEnv
+from finrock.render import PygameRender
+
+
+df = pd.read_csv('Datasets/random_sinusoid.csv')
+df = df[-1000:]
+
+model_path = "runs/1704798174"
+
+pd_data_feeder = PdDataFeeder.load_config(df, model_path)
+env = TradingEnv.load_config(pd_data_feeder, model_path)
+
+action_space = env.action_space
+input_shape = env.observation_space.shape
+pygameRender = PygameRender(frame_rate=120)
+
+agent = tf.keras.models.load_model(f'{model_path}/ppo_sinusoid_actor.h5')
+
+state, info = env.reset()
+pygameRender.render(info)
+rewards = 0.0
+while True:
+    # simulate model prediction, now use random action
+    action = agent.predict(np.expand_dims(state, axis=0), verbose=False)[0][:-1]
+
+    state, reward, terminated, truncated, info = env.step(action)
+    rewards += reward
+    pygameRender.render(info)
+
+    if terminated or truncated:
+        print(rewards)
+        for metric, value in info['metrics'].items():
+            print(metric, value)
+        state, info = env.reset()
+        rewards = 0.0
+        pygameRender.reset()
+        pygameRender.render(info)

experiments/testing_ppo_sinusoid.py …riments/testing_ppo_sinusoid_discrete.pyexperiments/testing_ppo_sinusoid.py renamed to experiments/testing_ppo_sinusoid_discrete.py experiments/testing_ppo_sinusoid.py renamed to experiments/testing_ppo_sinusoid_discrete.py

@@ -8,54 +8,27 @@
 from finrock.data_feeder import PdDataFeeder
 from finrock.trading_env import TradingEnv
 from finrock.render import PygameRender
-from finrock.scalers import MinMaxScaler
-from finrock.reward import simpleReward
-from finrock.metrics import DifferentActions, AccountValue, MaxDrawdown, SharpeRatio
-from finrock.indicators import BolingerBands, RSI, PSAR, SMA
 
 
 df = pd.read_csv('Datasets/random_sinusoid.csv')
 df = df[-1000:]
 
-pd_data_feeder = PdDataFeeder(
-    df,
-    indicators = [
-        BolingerBands(data=df, period=20, std=2),
-        RSI(data=df, period=14),
-        PSAR(data=df),
-        SMA(data=df, period=7),
-        SMA(data=df, period=25),
-        SMA(data=df, period=99),
-    ]
-    )
+model_path = "runs/1704746665"
 
-env = TradingEnv(
-    data_feeder = pd_data_feeder,
-    output_transformer = MinMaxScaler(min=pd_data_feeder.min, max=pd_data_feeder.max),
-    initial_balance = 1000.0,
-    max_episode_steps = 1000,
-    window_size = 50,
-    reward_function = simpleReward,
-    metrics = [
-        DifferentActions(),
-        AccountValue(),
-        MaxDrawdown(),
-        SharpeRatio(),
-    ]
-)
+pd_data_feeder = PdDataFeeder.load_config(df, model_path)
+env = TradingEnv.load_config(pd_data_feeder, model_path)
 
 action_space = env.action_space
 input_shape = env.observation_space.shape
 pygameRender = PygameRender(frame_rate=120)
 
-agent = tf.keras.models.load_model('runs/1702982487/ppo_sinusoid_actor.h5')
+agent = tf.keras.models.load_model(f'{model_path}/ppo_sinusoid_actor.h5')
 
 state, info = env.reset()
 pygameRender.render(info)
 rewards = 0.0
 while True:
     # simulate model prediction, now use random action
-    # action = np.random.randint(0, action_space)
     prob = agent.predict(np.expand_dims(state, axis=0), verbose=False)[0]
     action = np.argmax(prob)