From 65a6802d37039241fa5181f39d5c09a788786c1e Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Mon, 26 Aug 2024 21:53:58 +0800 Subject: [PATCH] add validator tracking and gas premium distribution --- Cargo.lock | 2 + fendermint/app/src/app.rs | 24 +++++-- fendermint/app/src/cmd/run.rs | 3 +- fendermint/app/src/ipc.rs | 14 ++-- fendermint/app/src/lib.rs | 1 + fendermint/app/src/validators.rs | 69 +++++++++++++++++++ fendermint/vm/interpreter/src/chain.rs | 4 +- fendermint/vm/interpreter/src/fvm/exec.rs | 4 +- .../vm/interpreter/src/fvm/gas/actor.rs | 63 ++++++++++++++--- fendermint/vm/interpreter/src/fvm/gas/mod.rs | 19 ++++- .../vm/interpreter/src/fvm/state/exec.rs | 21 +++--- 11 files changed, 190 insertions(+), 34 deletions(-) create mode 100644 fendermint/app/src/validators.rs diff --git a/Cargo.lock b/Cargo.lock index c6203003e..bf262d147 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2828,7 +2828,9 @@ dependencies = [ "fvm_ipld_blockstore", "fvm_ipld_encoding", "fvm_shared", + "hex", "hex-literal 0.4.1", + "k256 0.13.3", "log", "multihash 0.18.1", "num-derive 0.3.3", diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index e34f790f4..f9b6259a1 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -42,12 +42,14 @@ use num_traits::Zero; use serde::{Deserialize, Serialize}; use tendermint::abci::request::CheckTxKind; use tendermint::abci::{request, response}; +use tendermint_rpc::Client; use tracing::instrument; use crate::observe::{ BlockCommitted, BlockProposalEvaluated, BlockProposalReceived, BlockProposalSent, Message, MpoolReceived, }; +use crate::validators::ValidatorTracker; use crate::AppExitCode; use crate::BlockHeight; use crate::{tmconv::*, VERSION}; @@ -116,10 +118,11 @@ pub struct AppConfig { /// Handle ABCI requests. #[derive(Clone)] -pub struct App +pub struct App where SS: Blockstore + Clone + 'static, S: KVStore, + C: Client, { /// Database backing all key-value operations. db: Arc, @@ -160,9 +163,11 @@ where /// /// Zero means unlimited. state_hist_size: u64, + /// Tracks the validator + validators: ValidatorTracker, } -impl App +impl App where S: KVStore + Codec @@ -171,6 +176,7 @@ where + Codec, DB: KVWritable + KVReadable + Clone + 'static, SS: Blockstore + Clone + 'static, + C: Client, { pub fn new( config: AppConfig, @@ -179,6 +185,7 @@ where interpreter: I, chain_env: ChainEnv, snapshots: Option, + client: C, ) -> Result { let app = Self { db: Arc::new(db), @@ -193,13 +200,14 @@ where snapshots, exec_state: Arc::new(tokio::sync::Mutex::new(None)), check_state: Arc::new(tokio::sync::Mutex::new(None)), + validators: ValidatorTracker::new(client), }; app.init_committed_state()?; Ok(app) } } -impl App +impl App where S: KVStore + Codec @@ -208,6 +216,7 @@ where + Codec, DB: KVWritable + KVReadable + 'static + Clone, SS: Blockstore + 'static + Clone, + C: Client, { /// Get an owned clone of the state store. fn state_store_clone(&self) -> SS { @@ -393,7 +402,7 @@ where // the `tower-abci` library would throw an exception when it tried to convert a // `Response::Exception` into a `ConsensusResponse` for example. #[async_trait] -impl Application for App +impl Application for App where S: KVStore + Codec @@ -421,6 +430,7 @@ where Query = BytesMessageQuery, Output = BytesMessageQueryRes, >, + C: Client + Sync, { /// Provide information about the ABCI application. async fn info(&self, _request: request::Info) -> AbciResult { @@ -727,10 +737,14 @@ where state_params.timestamp = to_timestamp(request.header.time); + let validator = self + .validators + .get_validator(&request.header.proposer_address, block_height) + .await?; let state = FvmExecState::new(db, self.multi_engine.as_ref(), block_height, state_params) .context("error creating new state")? .with_block_hash(block_hash) - .with_validator_id(request.header.proposer_address); + .with_validator(validator); tracing::debug!("initialized exec state"); diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index dca099b1f..9d8e68516 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -294,7 +294,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> { None }; - let app: App<_, _, AppStore, _> = App::new( + let app: App<_, _, AppStore, _, _> = App::new( AppConfig { app_namespace: ns.app, state_hist_namespace: ns.state_hist, @@ -310,6 +310,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> { parent_finality_votes: parent_finality_votes.clone(), }, snapshots, + tendermint_client.clone(), )?; if let Some((agent_proxy, config)) = ipc_tuple { diff --git a/fendermint/app/src/ipc.rs b/fendermint/app/src/ipc.rs index 6a03bc4b2..5415a0efe 100644 --- a/fendermint/app/src/ipc.rs +++ b/fendermint/app/src/ipc.rs @@ -15,6 +15,7 @@ use fvm_ipld_blockstore::Blockstore; use std::sync::Arc; use serde::{Deserialize, Serialize}; +use tendermint_rpc::Client; /// All the things that can be voted on in a subnet. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -24,17 +25,18 @@ pub enum AppVote { } /// Queries the LATEST COMMITTED parent finality from the storage -pub struct AppParentFinalityQuery +pub struct AppParentFinalityQuery where SS: Blockstore + Clone + 'static, S: KVStore, + C: Client, { /// The app to get state - app: App, + app: App, gateway_caller: GatewayCaller>>, } -impl AppParentFinalityQuery +impl AppParentFinalityQuery where S: KVStore + Codec @@ -43,8 +45,9 @@ where + Codec, DB: KVWritable + KVReadable + 'static + Clone, SS: Blockstore + 'static + Clone, + C: Client, { - pub fn new(app: App) -> Self { + pub fn new(app: App) -> Self { Self { app, gateway_caller: GatewayCaller::default(), @@ -62,7 +65,7 @@ where } } -impl ParentFinalityStateQuery for AppParentFinalityQuery +impl ParentFinalityStateQuery for AppParentFinalityQuery where S: KVStore + Codec @@ -71,6 +74,7 @@ where + Codec, DB: KVWritable + KVReadable + 'static + Clone, SS: Blockstore + 'static + Clone, + C: Client, { fn get_latest_committed_finality(&self) -> anyhow::Result> { self.with_exec_state(|mut exec_state| { diff --git a/fendermint/app/src/lib.rs b/fendermint/app/src/lib.rs index f9a45b9a5..a43f8d93f 100644 --- a/fendermint/app/src/lib.rs +++ b/fendermint/app/src/lib.rs @@ -8,6 +8,7 @@ pub mod metrics; pub mod observe; mod store; mod tmconv; +mod validators; pub use app::{App, AppConfig}; pub use store::{AppStore, BitswapBlockstore}; diff --git a/fendermint/app/src/validators.rs b/fendermint/app/src/validators.rs new file mode 100644 index 000000000..5870b7160 --- /dev/null +++ b/fendermint/app/src/validators.rs @@ -0,0 +1,69 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +//! Tracks the validator id from tendermint to their corresponding public key. + +use anyhow::anyhow; +use fendermint_crypto::PublicKey; +use fvm_shared::clock::ChainEpoch; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use tendermint::block::Height; +use tendermint_rpc::{Client, Paging}; + +#[derive(Clone)] +pub(crate) struct ValidatorTracker { + client: C, + public_keys: Arc>>, +} + +impl ValidatorTracker { + pub fn new(client: C) -> Self { + Self { + client, + public_keys: Arc::new(RwLock::new(HashMap::new())), + } + } +} + +impl ValidatorTracker { + /// Get the public key of the validator by id. Note that the id is expected to be a validator. + pub async fn get_validator( + &self, + id: &tendermint::account::Id, + height: ChainEpoch, + ) -> anyhow::Result { + if let Some(key) = self.get_from_cache(id) { + return Ok(key); + } + + // this means validators have changed, re-pull all validators + let height = Height::try_from(height)?; + let response = self.client.validators(height, Paging::All).await?; + + let mut new_validators = HashMap::new(); + let mut pubkey = None; + for validator in response.validators { + let p = validator.pub_key.secp256k1().unwrap(); + let compressed = p.to_encoded_point(true); + let b = compressed.as_bytes(); + let key = PublicKey::parse_slice(b, None)?; + + if *id == validator.address { + pubkey = Some(key); + } + + new_validators.insert(validator.address, key); + } + + *self.public_keys.write().unwrap() = new_validators; + + // cannot find the validator, this should not have happened usually + pubkey.ok_or_else(|| anyhow!("{} not validator", id)) + } + + fn get_from_cache(&self, id: &tendermint::account::Id) -> Option { + let keys = self.public_keys.read().unwrap(); + keys.get(id).copied() + } +} diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index a54bb57e7..dc9a9b553 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -383,7 +383,9 @@ where tracing::debug!("chain interpreter applied topdown msgs"); let local_block_height = state.block_height() as u64; - let proposer = state.validator_id().map(|id| id.to_string()); + let proposer = state + .validator_pubkey() + .map(|id| hex::encode(id.serialize_compressed())); let proposer_ref = proposer.as_deref(); atomically(|| { diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 9fcd1a15e..619b278aa 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -13,7 +13,7 @@ use ipc_observability::{emit, measure_time, observe::TracingError, Traceable}; use tendermint_rpc::Client; use crate::fvm::cometbft::EndBlockUpdate; -use crate::fvm::gas::GasMarket; +use crate::fvm::gas::{GasMarket, GasUtilization}; use crate::ExecInterpreter; use super::{ @@ -175,7 +175,7 @@ where state .gas_market_mut() - .record_utilization(apply_ret.msg_receipt.gas_used); + .record_utilization(GasUtilization::from(&apply_ret)); (apply_ret, emitters, latency) }; diff --git a/fendermint/vm/interpreter/src/fvm/gas/actor.rs b/fendermint/vm/interpreter/src/fvm/gas/actor.rs index b832f3437..2fdd78d36 100644 --- a/fendermint/vm/interpreter/src/fvm/gas/actor.rs +++ b/fendermint/vm/interpreter/src/fvm/gas/actor.rs @@ -1,19 +1,26 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::fvm::gas::{Available, Gas, GasMarket}; +use crate::fvm::gas::{Available, Gas, GasMarket, GasUtilization}; use crate::fvm::FvmMessage; use anyhow::Context; use crate::fvm::cometbft::ConsensusBlockUpdate; use fendermint_actor_gas_market::{GasMarketReading, SetConstants}; +use fendermint_crypto::PublicKey; +use fendermint_vm_actor_interface::eam::EthAddress; use fendermint_vm_actor_interface::gas::GAS_MARKET_ACTOR_ADDR; -use fendermint_vm_actor_interface::system; +use fendermint_vm_actor_interface::{reward, system}; use fvm::executor::{ApplyKind, ApplyRet, Executor}; +use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; +use fvm_shared::econ::TokenAmount; +use fvm_shared::METHOD_SEND; #[derive(Default)] pub struct ActorGasMarket { + /// The total gas premium for the miner + gas_premium: TokenAmount, /// The block gas limit block_gas_limit: Gas, /// The accumulated gas usage so far @@ -39,8 +46,9 @@ impl GasMarket for ActorGasMarket { } } - fn record_utilization(&mut self, gas: Gas) { - self.block_gas_used += gas; + fn record_utilization(&mut self, utilization: GasUtilization) { + self.gas_premium += utilization.gas_premium; + self.block_gas_used += utilization.gas_used; // sanity check if self.block_gas_used >= self.block_gas_limit { @@ -80,6 +88,7 @@ impl ActorGasMarket { .context("failed to parse gas market readying")?; Ok(Self { + gas_premium: TokenAmount::from_atto(0), block_gas_limit: reading.block_gas_limit, block_gas_used: 0, constant_update: None, @@ -96,9 +105,43 @@ impl ActorGasMarket { &self, executor: &mut E, block_height: ChainEpoch, + validator: Option, ) -> anyhow::Result<()> { self.commit_constants(executor, block_height)?; - self.commit_utilization(executor, block_height) + self.commit_utilization(executor, block_height)?; + self.distribute_reward(executor, block_height, validator) + } + + fn distribute_reward( + &self, + executor: &mut E, + block_height: ChainEpoch, + validator: Option, + ) -> anyhow::Result<()> { + if validator.is_none() || self.gas_premium.is_zero() { + return Ok(()); + } + + let validator = validator.unwrap(); + let validator = Address::from(EthAddress::new_secp256k1(&validator.serialize())?); + + let msg = FvmMessage { + from: reward::REWARD_ACTOR_ADDR, + to: validator, + sequence: block_height as u64, + // exclude this from gas restriction + gas_limit: i64::MAX as u64, + method_num: METHOD_SEND, + params: fvm_ipld_encoding::RawBytes::default(), + value: self.gas_premium.clone(), + + version: Default::default(), + gas_fee_cap: Default::default(), + gas_premium: Default::default(), + }; + self.exec_msg_implicitly(msg, executor)?; + + Ok(()) } fn commit_constants( @@ -123,7 +166,7 @@ impl ActorGasMarket { gas_fee_cap: Default::default(), gas_premium: Default::default(), }; - self.call_fvm(msg, executor)?; + self.exec_msg_implicitly(msg, executor)?; Ok(()) } @@ -152,11 +195,15 @@ impl ActorGasMarket { gas_premium: Default::default(), }; - self.call_fvm(msg, executor)?; + self.exec_msg_implicitly(msg, executor)?; Ok(()) } - fn call_fvm(&self, msg: FvmMessage, executor: &mut E) -> anyhow::Result { + fn exec_msg_implicitly( + &self, + msg: FvmMessage, + executor: &mut E, + ) -> anyhow::Result { let raw_length = fvm_ipld_encoding::to_vec(&msg).map(|bz| bz.len())?; let apply_ret = executor.execute_message(msg, ApplyKind::Implicit, raw_length)?; diff --git a/fendermint/vm/interpreter/src/fvm/gas/mod.rs b/fendermint/vm/interpreter/src/fvm/gas/mod.rs index f6e26ef93..346f57c94 100644 --- a/fendermint/vm/interpreter/src/fvm/gas/mod.rs +++ b/fendermint/vm/interpreter/src/fvm/gas/mod.rs @@ -1,6 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use fvm::executor::ApplyRet; +use fvm_shared::econ::TokenAmount; + pub mod actor; pub type Gas = u64; @@ -9,6 +12,11 @@ pub struct Available { pub block_gas: Gas, } +pub struct GasUtilization { + gas_used: Gas, + gas_premium: TokenAmount, +} + /// The gas market for fendermint. This should be backed by an fvm actor. pub trait GasMarket { /// The constant parameters that determines the readings of gas market, such as block gas limit. @@ -26,5 +34,14 @@ pub trait GasMarket { fn available(&self) -> Available; /// Tracks the amount of gas consumed by a transaction - fn record_utilization(&mut self, gas: Gas); + fn record_utilization(&mut self, gas: GasUtilization); +} + +impl From<&ApplyRet> for GasUtilization { + fn from(ret: &ApplyRet) -> Self { + Self { + gas_used: ret.msg_receipt.gas_used, + gas_premium: ret.miner_tip.clone(), + } + } } diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index 577317dc1..10497ce20 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -5,6 +5,7 @@ use std::collections::{HashMap, HashSet}; use anyhow::Ok; use cid::Cid; +use fendermint_crypto::PublicKey; use fendermint_vm_genesis::PowerScale; use fvm::{ call_manager::DefaultCallManager, @@ -30,9 +31,6 @@ use fendermint_vm_encoding::IsHumanReadable; pub type BlockHash = [u8; 32]; -/// First 20 bytes of SHA256(PublicKey) -pub type ValidatorId = tendermint::account::Id; - pub type ActorAddressMap = HashMap; /// The result of the message application bundled with any delegated addresses of event emitters. @@ -107,8 +105,8 @@ where /// execution interpreter without having to add yet another piece to track at the app level. block_hash: Option, - /// ID of the validator who created this block. For queries and checks this is empty. - validator_id: Option, + /// Public key of the validator who created this block. For queries and checks this is empty. + validator_pubkey: Option, /// State of parameters that are outside the control of the FVM but can change and need to be persisted. params: FvmUpdatableParams, @@ -156,7 +154,7 @@ where Ok(Self { executor, block_hash: None, - validator_id: None, + validator_pubkey: None, params: FvmUpdatableParams { app_version: params.app_version, base_fee: params.base_fee, @@ -175,8 +173,8 @@ where } /// Set the validator during execution. - pub fn with_validator_id(mut self, validator_id: ValidatorId) -> Self { - self.validator_id = Some(validator_id); + pub fn with_validator(mut self, key: PublicKey) -> Self { + self.validator_pubkey = Some(key); self } @@ -233,8 +231,8 @@ where } /// Identity of the block creator, if we are indeed executing any blocks. - pub fn validator_id(&self) -> Option { - self.validator_id + pub fn validator_pubkey(&self) -> Option { + self.validator_pubkey } /// The timestamp of the currently executing block. @@ -302,7 +300,8 @@ where pub fn update_gas_market(&mut self) -> anyhow::Result<()> { let height = self.block_height(); - self.gas_market.commit(&mut self.executor, height) + self.gas_market + .commit(&mut self.executor, height, self.validator_pubkey) } /// Update the circulating supply, effective from the next block.