Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ utoipa = { version = "5", features = ["axum_extras"] }
miden-protocol = "=0.18.0"
miden-standards = "=0.18.0"
miden-tx = "=0.18.0"
miden-tx-batch = "=0.18.0"
miden-node-proto-build = { version = "=0.18.2", default-features = false }
miden-testing = "=0.18.0"
miden-client = "=0.18.0"
Expand Down
2 changes: 2 additions & 0 deletions crates/server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ diesel_migrations = { version = "2.2", optional = true }
# off at run time until a prover is configured.
miden-client = { workspace = true, features = ["tonic"] }
miden-tx = { workspace = true }
# Batched submission (#239): the batch kernel executor and prover.
miden-tx-batch = { workspace = true }
miden-processor = { workspace = true }

# Optional dependencies for e2e tests
Expand Down
16 changes: 14 additions & 2 deletions crates/server/src/builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,16 @@ impl ServerBuilder {
let rpc_settings = crate::network::RpcSettings::resolve_for(self.rpc, network_type)?;
let network_client = rpc_settings.connect().await?;
match self.execution.startup_notice() {
crate::config::execution::ExecutionNotice::Enabled
if self.execution.batch.batches() =>
{
tracing::info!(
max_txs_per_batch = self.execution.batch.max_transactions,
batch_window_ms = self.execution.batch.window.as_millis(),
"Guardian execution is offered; transactions of different accounts proved \
within the batch window are submitted together as one batch"
)
}
crate::config::execution::ExecutionNotice::Enabled => {
tracing::info!("Guardian execution is offered")
}
Expand Down Expand Up @@ -661,8 +671,10 @@ impl ServerBuilder {
whenever that network's protocol version differs from this one's"
);
}
let execution_shutdown = tokio_util::sync::CancellationToken::new();
let execution = crate::services::execute_proposal::ExecutionState {
executor: rpc_settings.proposal_executor(&self.execution)?,
executor: rpc_settings
.proposal_executor(&self.execution, execution_shutdown.clone())?,
leases: coordination
.as_ref()
.map(|handles| handles.execution_leases.clone())
Expand All @@ -678,7 +690,7 @@ impl ServerBuilder {
&self.execution,
),
config: self.execution.clone(),
shutdown: tokio_util::sync::CancellationToken::new(),
shutdown: execution_shutdown,
};

// Block watch reads the same node over sources of its own (a
Expand Down
132 changes: 132 additions & 0 deletions crates/server/src/config/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ pub const ENV_EXECUTION_EXPIRATION_HORIZON_BLOCKS: &str =
"GUARDIAN_EXECUTION_EXPIRATION_HORIZON_BLOCKS";
pub const ENV_EXECUTION_MAX_CONCURRENT: &str = "GUARDIAN_EXECUTION_MAX_CONCURRENT";
pub const ENV_EXECUTION_RECORD_RETENTION_DAYS: &str = "GUARDIAN_EXECUTION_RECORD_RETENTION_DAYS";
pub const ENV_EXECUTION_MAX_TXS_PER_BATCH: &str = "GUARDIAN_EXECUTION_MAX_TXS_PER_BATCH";
pub const ENV_EXECUTION_BATCH_WINDOW_MS: &str = "GUARDIAN_EXECUTION_BATCH_WINDOW_MS";

/// The upstream remote-prover client defaults to 10 s, below observed proving
/// times, so Guardian always sets its own.
Expand All @@ -30,6 +32,20 @@ pub const DEFAULT_EXECUTION_EXPIRATION_HORIZON_BLOCKS: u32 = 512;
/// memory for a proof permit rather than reaching the prover.
pub const DEFAULT_EXECUTION_MAX_CONCURRENT: u32 = 64;
pub const DEFAULT_EXECUTION_RECORD_RETENTION_DAYS: u32 = 30;
/// One transaction per send: batching is off unless an operator turns it on.
pub const DEFAULT_EXECUTION_MAX_TXS_PER_BATCH: u32 = 1;
/// The node's own batch builder collects transactions for one second, so a batch Guardian builds
/// waits no longer than the node would have.
pub const DEFAULT_EXECUTION_BATCH_WINDOW_MS: u32 = 1_000;

/// The most accounts the protocol admits in one batch (`miden_protocol::MAX_ACCOUNTS_PER_BATCH`).
/// Guardian batches one transaction per account, so no batch can hold more transactions.
pub const MAX_EXECUTION_TXS_PER_BATCH: u32 = 1_024;

/// The longest a transaction past the no-retry boundary may wait for others to share its batch.
/// Crossing the boundary needs 15 s of send window left (`SendWindow::MIN_SEND`), and a planned
/// stop gives sends 8 s, so the wait stays well inside both.
pub const MAX_EXECUTION_BATCH_WINDOW_MS: u32 = 5_000;

/// Every built-in Guardian-executable proposal signs this relative
/// transaction expiration, so a horizon below it would refuse all of them.
Expand Down Expand Up @@ -76,6 +92,33 @@ impl ProverConfig {
}
}

/// How Guardian delivers the transactions it proves: one at a time, or transactions of different
/// accounts together as one proven batch (#239).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BatchConfig {
/// The most transactions one batch holds; `1` sends every transaction alone. The node refuses a
/// batch above its own `max_txs_per_batch` (4 by default on node 0.18), after which Guardian
/// sends that batch's transactions one at a time, so set this at or below the node's limit.
pub max_transactions: u32,
/// How long the first transaction of a batch waits for others before the batch is sent.
pub window: Duration,
}

impl BatchConfig {
pub fn batches(&self) -> bool {
self.max_transactions > 1
}
}

impl Default for BatchConfig {
fn default() -> Self {
Self {
max_transactions: DEFAULT_EXECUTION_MAX_TXS_PER_BATCH,
window: Duration::from_millis(u64::from(DEFAULT_EXECUTION_BATCH_WINDOW_MS)),
}
}
}

/// How long finished execution attempts are kept before the retention sweep removes them.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RecordRetention {
Expand Down Expand Up @@ -159,6 +202,7 @@ pub struct ExecutionConfig {
pub expiration_horizon_blocks: u32,
pub max_concurrent_executions: u32,
pub record_retention: RecordRetention,
pub batch: BatchConfig,
}

impl Default for ExecutionConfig {
Expand All @@ -175,6 +219,7 @@ impl Default for ExecutionConfig {
expiration_horizon_blocks: DEFAULT_EXECUTION_EXPIRATION_HORIZON_BLOCKS,
max_concurrent_executions: DEFAULT_EXECUTION_MAX_CONCURRENT,
record_retention: RecordRetention::Days(DEFAULT_EXECUTION_RECORD_RETENTION_DAYS),
batch: BatchConfig::default(),
}
}
}
Expand Down Expand Up @@ -280,6 +325,7 @@ impl ExecutionConfig {
Some(raw) => RecordRetention::parse(&raw)?,
None => RecordRetention::Days(DEFAULT_EXECUTION_RECORD_RETENTION_DAYS),
},
batch: BatchConfig::from_lookup(&lookup)?,
})
}

Expand All @@ -294,6 +340,40 @@ impl ExecutionConfig {
}
}

impl BatchConfig {
fn from_lookup(
lookup: &impl Fn(&str) -> Result<Option<String>, String>,
) -> Result<Self, String> {
let max_transactions = positive_u32(
lookup,
ENV_EXECUTION_MAX_TXS_PER_BATCH,
DEFAULT_EXECUTION_MAX_TXS_PER_BATCH,
)?;
if max_transactions > MAX_EXECUTION_TXS_PER_BATCH {
return Err(format!(
"{ENV_EXECUTION_MAX_TXS_PER_BATCH} must be at most {MAX_EXECUTION_TXS_PER_BATCH}, \
the most accounts one batch can hold, got {max_transactions}"
));
}
let window_ms = positive_u32(
lookup,
ENV_EXECUTION_BATCH_WINDOW_MS,
DEFAULT_EXECUTION_BATCH_WINDOW_MS,
)?;
if window_ms > MAX_EXECUTION_BATCH_WINDOW_MS {
return Err(format!(
"{ENV_EXECUTION_BATCH_WINDOW_MS} must be at most {MAX_EXECUTION_BATCH_WINDOW_MS}, \
so a transaction past the no-retry boundary is not held back from its send, got \
{window_ms}"
));
}
Ok(Self {
max_transactions,
window: Duration::from_millis(u64::from(window_ms)),
})
}
}

/// Refuses a prover URL the prover client could not connect to. The message never repeats the
/// URL, which can carry credentials.
/// The prover client sends neither userinfo nor a query, so a URL carrying them would only look
Expand Down Expand Up @@ -447,6 +527,58 @@ mod tests {
assert_eq!(config.reconcile_interval, Duration::from_secs(30));
assert_eq!(config.max_concurrent_executions, 64);
assert_eq!(config.record_retention, RecordRetention::Days(30));
assert_eq!(config.batch.max_transactions, 1);
assert!(!config.batch.batches(), "batching is off unless asked for");
assert_eq!(config.batch.window, Duration::from_secs(1));
}

#[test]
fn batching_turns_on_above_one_transaction_and_is_bounded() {
let batch = |vars: &[(&str, &str)]| config_from(vars).map(|config| config.batch);
let on = batch(&[
(ENV_EXECUTION_MAX_TXS_PER_BATCH, "4"),
(ENV_EXECUTION_BATCH_WINDOW_MS, "250"),
])
.unwrap();
assert!(on.batches());
assert_eq!(on.max_transactions, 4);
assert_eq!(on.window, Duration::from_millis(250));
assert!(
!batch(&[(ENV_EXECUTION_MAX_TXS_PER_BATCH, "1")])
.unwrap()
.batches()
);

assert!(batch(&[(ENV_EXECUTION_MAX_TXS_PER_BATCH, "1024")]).is_ok());
let error = batch(&[(ENV_EXECUTION_MAX_TXS_PER_BATCH, "1025")]).unwrap_err();
assert!(error.contains(ENV_EXECUTION_MAX_TXS_PER_BATCH), "{error}");
assert!(batch(&[(ENV_EXECUTION_BATCH_WINDOW_MS, "5000")]).is_ok());
let error = batch(&[(ENV_EXECUTION_BATCH_WINDOW_MS, "5001")]).unwrap_err();
assert!(error.contains(ENV_EXECUTION_BATCH_WINDOW_MS), "{error}");
for key in [
ENV_EXECUTION_MAX_TXS_PER_BATCH,
ENV_EXECUTION_BATCH_WINDOW_MS,
] {
assert!(config_from(&[(key, "0")]).unwrap_err().contains(key));
assert!(config_from(&[(key, "four")]).unwrap_err().contains(key));
}
}

#[test]
fn the_batch_cap_is_the_protocol_account_cap() {
assert_eq!(
MAX_EXECUTION_TXS_PER_BATCH as usize,
miden_protocol::MAX_ACCOUNTS_PER_BATCH
);
}

#[test]
fn the_batch_window_ends_well_before_a_send_must_start() {
let window = Duration::from_millis(u64::from(MAX_EXECUTION_BATCH_WINDOW_MS));
assert!(
window * 3 <= crate::services::execute_proposal::SendWindow::MIN_SEND,
"a transaction crosses the boundary with MIN_SEND left and may wait one window"
);
}

#[test]
Expand Down
52 changes: 49 additions & 3 deletions crates/server/src/metrics/execution.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
//! Guardian execution metrics. Internal phases are diagnosed here and in logs, never on the wire.

use crate::metrics::names::{
EXECUTION_CAPACITY_REFUSALS_TOTAL, EXECUTION_CHAIN_VIEW_DURATION_SECONDS,
EXECUTION_OBSERVATION_OUTAGE_SECONDS, EXECUTION_OLDEST_RESERVATION_AGE_SECONDS,
EXECUTION_OUTCOMES_TOTAL, EXECUTION_PHASE_DURATION_SECONDS, EXECUTION_PROVER_RETRIES_TOTAL,
EXECUTION_BATCH_PROVING_DURATION_SECONDS, EXECUTION_BATCHED_TRANSACTIONS_TOTAL,
EXECUTION_BATCHES_TOTAL, EXECUTION_CAPACITY_REFUSALS_TOTAL,
EXECUTION_CHAIN_VIEW_DURATION_SECONDS, EXECUTION_OBSERVATION_OUTAGE_SECONDS,
EXECUTION_OLDEST_RESERVATION_AGE_SECONDS, EXECUTION_OUTCOMES_TOTAL,
EXECUTION_PHASE_DURATION_SECONDS, EXECUTION_PROVER_RETRIES_TOTAL,
EXECUTION_PROVING_DURATION_SECONDS, EXECUTION_RECONCILE_OUTCOMES_TOTAL,
EXECUTION_RECORDS_PRUNED_TOTAL, LABEL_CODE, LABEL_OUTCOME, LABEL_PHASE,
};
Expand Down Expand Up @@ -62,3 +64,47 @@ pub fn record_observation_outage(seconds: f64) {
pub fn record_records_pruned(attempts: usize) {
metrics::counter!(EXECUTION_RECORDS_PRUNED_TOTAL).increment(attempts as u64);
}

/// What became of a group of transactions the batcher tried to send together.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BatchOutcome {
/// The node admitted the batch.
Accepted,
/// The node's answer did not arrive; the chain settles each transaction.
Unknown,
/// The node refused the batch before processing it, and would refuse each transaction alone
/// the same way.
NotProcessed,
/// The node refused the batch; each transaction was then sent alone.
Refused,
/// No batch was sent (no chain view, a batch that did not assemble or prove, or a send window
/// that closed); each transaction was sent alone.
NotBuilt,
/// Another batch was being proved; each transaction was sent alone.
ProverBusy,
}

impl BatchOutcome {
pub fn as_str(self) -> &'static str {
match self {
BatchOutcome::Accepted => "accepted",
BatchOutcome::Unknown => "unknown",
BatchOutcome::NotProcessed => "not_processed",
BatchOutcome::Refused => "refused",
BatchOutcome::NotBuilt => "not_built",
BatchOutcome::ProverBusy => "prover_busy",
}
}
}

/// Counts one group of `transactions` transactions the batcher tried to send together.
pub fn record_batch(outcome: BatchOutcome, transactions: usize) {
metrics::counter!(EXECUTION_BATCHES_TOTAL, LABEL_OUTCOME => outcome.as_str()).increment(1);
metrics::counter!(EXECUTION_BATCHED_TRANSACTIONS_TOTAL, LABEL_OUTCOME => outcome.as_str())
.increment(transactions as u64);
}

/// Records how long assembling, executing and proving one batch took.
pub fn record_batch_proving(took: std::time::Duration) {
metrics::histogram!(EXECUTION_BATCH_PROVING_DURATION_SECONDS).record(took.as_secs_f64());
}
23 changes: 23 additions & 0 deletions crates/server/src/metrics/names.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ pub const EXECUTION_OLDEST_RESERVATION_AGE_SECONDS: &str =
pub const EXECUTION_OBSERVATION_OUTAGE_SECONDS: &str =
"guardian_execution_observation_outage_seconds";
pub const EXECUTION_RECORDS_PRUNED_TOTAL: &str = "guardian_execution_records_pruned_total";
pub const EXECUTION_BATCHES_TOTAL: &str = "guardian_execution_batches_total";
pub const EXECUTION_BATCHED_TRANSACTIONS_TOTAL: &str =
"guardian_execution_batched_transactions_total";
pub const EXECUTION_BATCH_PROVING_DURATION_SECONDS: &str =
"guardian_execution_batch_proving_duration_seconds";
pub const CANONICALIZATION_RECONCILE_RUNS_TOTAL: &str =
"guardian_canonicalization_reconcile_runs_total";
pub const CANONICALIZATION_RECONCILE_RUN_DURATION_SECONDS: &str =
Expand Down Expand Up @@ -476,6 +481,24 @@ pub const REGISTRY: &[MetricDef] = &[
labels: &[],
help: "Finished execution attempts the retention sweep deleted, with their evidence and outcome.",
},
MetricDef {
name: EXECUTION_BATCHES_TOTAL,
kind: MetricKind::Counter,
labels: &[LABEL_OUTCOME],
help: "Groups of two or more Guardian transactions handed to the batcher, by outcome: accepted, unknown or not_processed (the node answered for the batch), refused (the node refused it and each transaction was sent alone), not_built or prover_busy (not sent as a batch; each transaction was sent alone).",
},
MetricDef {
name: EXECUTION_BATCHED_TRANSACTIONS_TOTAL,
kind: MetricKind::Counter,
labels: &[LABEL_OUTCOME],
help: "Transactions the groups counted by guardian_execution_batches_total held for their batch, by the same outcome; a transaction kept out for its expiration is sent alone and not counted.",
},
MetricDef {
name: EXECUTION_BATCH_PROVING_DURATION_SECONDS,
kind: MetricKind::Histogram,
labels: &[],
help: "Time to assemble, execute and prove one batch of Guardian transactions, in seconds.",
},
MetricDef {
name: CANONICALIZATION_RECONCILE_RUNS_TOTAL,
kind: MetricKind::Counter,
Expand Down
6 changes: 6 additions & 0 deletions crates/server/src/metrics/recorder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,12 @@ pub fn build_recorder() -> PrometheusRecorder {
EXECUTION_PHASE_BUCKETS,
)
.expect("static execution phase buckets are non-empty")
// The skeleton batch kernel proves in milliseconds; the real one will take seconds.
.set_buckets_for_metric(
Matcher::Full(names::EXECUTION_BATCH_PROVING_DURATION_SECONDS.to_string()),
EXECUTION_PHASE_BUCKETS,
)
.expect("static batch proving buckets are non-empty")
.build_recorder()
}

Expand Down
Loading
Loading