Merge branch 'master' into thlorenz/fix-direct-connection-metric

* master:
  feat: Execute CommitDiff as BufferTask (#616)
  refactor(processor): improve transaction processing codebase (#817)

Author: thlorenz

Cargo.lock

@@ -53,6 +53,7 @@ use magicblock_ledger::{
 use magicblock_metrics::{metrics::TRANSACTION_COUNT, MetricsService};
 use magicblock_processor::{
     build_svm_env,
+    loader::load_upgradeable_programs,
     scheduler::{state::TransactionSchedulerState, TransactionScheduler},
 };
 use magicblock_program::{
@@ -213,6 +214,14 @@ impl MagicValidator {
                 ))
             });
 
+        load_upgradeable_programs(
+            &accountsdb,
+            &programs_to_load(&config.programs),
+        )
+        .map_err(|err| {
+            ApiError::FailedToLoadProgramsIntoBank(format!("{:?}", err))
+        })?;
+
         validator::init_validator_authority(identity_keypair);
         let base_fee = config.validator.basefee;
         let txn_scheduler_state = TransactionSchedulerState {
@@ -230,12 +239,6 @@ impl MagicValidator {
             shutdown: token.clone(),
         };
         TRANSACTION_COUNT.inc_by(ledger.count_transactions()? as u64);
-        txn_scheduler_state
-            .load_upgradeable_programs(&programs_to_load(&config.programs))
-            .map_err(|err| {
-                ApiError::FailedToLoadProgramsIntoBank(format!("{:?}", err))
-            })?;
-
         // Faucet keypair is only used for airdrops, which are not allowed in
         // the Ephemeral mode by setting the faucet to None in node context
         // (used by the RPC implementation), we effectively disable airdrops

magicblock-api/src/magic_validator.rs

@@ -587,14 +587,14 @@ mod tests {
         persister.set_commit_id(1, &pubkey, 100).unwrap();
 
         persister
-            .set_commit_strategy(100, &pubkey, CommitStrategy::Args)
+            .set_commit_strategy(100, &pubkey, CommitStrategy::StateArgs)
             .unwrap();
 
         let updated = persister
             .get_commit_status_by_message(1, &pubkey)
             .unwrap()
             .unwrap();
-        assert_eq!(updated.commit_strategy, CommitStrategy::Args);
+        assert_eq!(updated.commit_strategy, CommitStrategy::StateArgs);
     }
 
     #[test]

magicblock-committor-service/src/persist/commit_persister.rs

@@ -772,7 +772,7 @@ mod tests {
             commit_type: CommitType::DataAccount,
             created_at: 1000,
             commit_status: CommitStatus::Pending,
-            commit_strategy: CommitStrategy::Args,
+            commit_strategy: CommitStrategy::StateArgs,
             last_retried_at: 1000,
             retries_count: 0,
         }
@@ -907,7 +907,7 @@ mod tests {
         db.insert_commit_status_rows(std::slice::from_ref(&row))
             .unwrap();
 
-        let new_strategy = CommitStrategy::FromBuffer;
+        let new_strategy = CommitStrategy::StateBuffer;
         db.set_commit_strategy(100, &row.pubkey, new_strategy)
             .unwrap();
 

magicblock-committor-service/src/persist/db.rs

@@ -4,39 +4,46 @@ use crate::persist::error::CommitPersistError;
 pub enum CommitStrategy {
     /// Args without the use of a lookup table
     #[default]
-    Args,
+    StateArgs,
     /// Args with the use of a lookup table
-    ArgsWithLookupTable,
+    StateArgsWithLookupTable,
     /// Buffer and chunks which has the most overhead
-    FromBuffer,
+    StateBuffer,
     /// Buffer and chunks with the use of a lookup table
-    FromBufferWithLookupTable,
+    StateBufferWithLookupTable,
+
+    /// Args without the use of a lookup table
+    DiffArgs,
+    /// Args with the use of a lookup table
+    DiffArgsWithLookupTable,
+    /// Buffer and chunks which has the most overhead
+    DiffBuffer,
+    /// Buffer and chunks with the use of a lookup table
+    DiffBufferWithLookupTable,
 }
 
 impl CommitStrategy {
-    pub fn args(use_lookup: bool) -> Self {
-        if use_lookup {
-            Self::ArgsWithLookupTable
-        } else {
-            Self::Args
-        }
-    }
-
     pub fn as_str(&self) -> &str {
         use CommitStrategy::*;
         match self {
-            Args => "Args",
-            ArgsWithLookupTable => "ArgsWithLookupTable",
-            FromBuffer => "FromBuffer",
-            FromBufferWithLookupTable => "FromBufferWithLookupTable",
+            StateArgs => "StateArgs",
+            StateArgsWithLookupTable => "StateArgsWithLookupTable",
+            StateBuffer => "StateBuffer",
+            StateBufferWithLookupTable => "StateBufferWithLookupTable",
+            DiffArgs => "DiffArgs",
+            DiffArgsWithLookupTable => "DiffArgsWithLookupTable",
+            DiffBuffer => "DiffBuffer",
+            DiffBufferWithLookupTable => "DiffBufferWithLookupTable",
         }
     }
 
     pub fn uses_lookup(&self) -> bool {
         matches!(
             self,
-            CommitStrategy::ArgsWithLookupTable
-                | CommitStrategy::FromBufferWithLookupTable
+            CommitStrategy::StateArgsWithLookupTable
+                | CommitStrategy::StateBufferWithLookupTable
+                | CommitStrategy::DiffArgsWithLookupTable
+                | CommitStrategy::DiffBufferWithLookupTable
         )
     }
 }
@@ -45,10 +52,18 @@ impl TryFrom<&str> for CommitStrategy {
     type Error = CommitPersistError;
     fn try_from(value: &str) -> Result<Self, CommitPersistError> {
         match value {
-            "Args" => Ok(Self::Args),
-            "ArgsWithLookupTable" => Ok(Self::ArgsWithLookupTable),
-            "FromBuffer" => Ok(Self::FromBuffer),
-            "FromBufferWithLookupTable" => Ok(Self::FromBufferWithLookupTable),
+            "Args" | "StateArgs" => Ok(Self::StateArgs),
+            "ArgsWithLookupTable" | "StateArgsWithLookupTable" => {
+                Ok(Self::StateArgsWithLookupTable)
+            }
+            "FromBuffer" | "StateBuffer" => Ok(Self::StateBuffer),
+            "FromBufferWithLookupTable" | "StateBufferWithLookupTable" => {
+                Ok(Self::StateBufferWithLookupTable)
+            }
+            "DiffArgs" => Ok(Self::DiffArgs),
+            "DiffArgsWithLookupTable" => Ok(Self::DiffArgsWithLookupTable),
+            "DiffBuffer" => Ok(Self::DiffBuffer),
+            "DiffBufferWithLookupTable" => Ok(Self::DiffBufferWithLookupTable),
             _ => Err(CommitPersistError::InvalidCommitStrategy(
                 value.to_string(),
             )),

magicblock-committor-service/src/persist/types/commit_strategy.rs

@@ -127,7 +127,7 @@ impl BaseTask for ArgsTask {
         }
     }
 
-    fn optimize(
+    fn try_optimize_tx_size(
         self: Box<Self>,
     ) -> Result<Box<dyn BaseTask>, Box<dyn BaseTask>> {
         match self.task_type {
@@ -137,16 +137,8 @@ impl BaseTask for ArgsTask {
                 )))
             }
             ArgsTaskType::CommitDiff(value) => {
-                // TODO (snawaz): Currently, we do not support executing CommitDiff
-                // as BufferTask, which is why we're forcing CommitDiffTask to become CommitTask
-                // before converting this task into BufferTask. Once CommitDiff is supported
-                // by BufferTask, we do not have to do this, as it's essentially a downgrade.
                 Ok(Box::new(BufferTask::new_preparation_required(
-                    BufferTaskType::Commit(CommitTask {
-                        commit_id: value.commit_id,
-                        allow_undelegation: value.allow_undelegation,
-                        committed_account: value.committed_account,
-                    }),
+                    BufferTaskType::CommitDiff(value),
                 )))
             }
             ArgsTaskType::BaseAction(_)