Use agent-level activities directly

benches/assets.rs

@@ -3,20 +3,23 @@
 //! market during agent investment.
 use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main};
 use muse2::asset::{AssetIterator, AssetPool, AssetRef};
+use muse2::commodity::CommodityID;
 use muse2::example::Example;
 use muse2::input::load_model;
 use muse2::model::Model;
 use muse2::output::DataWriter;
 use muse2::process::{Process, ProcessID};
-use muse2::simulation::candidate_assets_for_next_year;
+use muse2::region::RegionID;
 use muse2::simulation::investment::collect_preset_demands_for_year;
 use muse2::simulation::investment::competitive::{get_candidate_assets, select_best_assets};
 use muse2::simulation::market::{
     collect_agent_limits, get_demand_portion_for_market, get_responsible_agents,
 };
-use muse2::simulation::optimisation::DispatchRun;
-use muse2::simulation::prices::{Prices, calculate_prices_from_solutions};
+use muse2::simulation::prices::{PriceMap, Prices};
+use muse2::time_slice::TimeSliceID;
+use muse2::units::MoneyPerFlow;
 use rayon::ThreadPoolBuilder;
+use serde::Deserialize;
 use std::hint::black_box;
 use std::sync::Arc;
 use std::time::Duration;
@@ -58,63 +61,55 @@ fn load_bench_model() -> (Model, DataWriter, TempDir, TempDir) {
     (model, writer, example_dir, output_dir)
 }
 
-/// Commission base-year assets, age them up to `YEAR`, and build the candidate assets available
-/// for investment in `YEAR`, mirroring the start of `simulation::run`.
-fn build_assets_for_investment_year(
-    model: &Model,
-) -> (Vec<AssetRef>, Vec<AssetRef>, Vec<AssetRef>) {
+/// Commission base-year assets and age them up to `YEAR`, mirroring the start of `simulation::run`.
+fn build_assets_for_investment_year(model: &Model) -> Vec<AssetRef> {
     let mut user_assets = model.user_assets.clone();
     let mut asset_pool = AssetPool::new();
     asset_pool.commission_new(BASE_YEAR, &mut user_assets);
 
-    // Keep a copy of the assets commissioned for the base year, which are used to seed prices for
-    // the investment year (mirroring the first iteration of `simulation::run`, in which the base
-    // year is dispatched before any investment has taken place)
-    let base_year_assets: Vec<AssetRef> = asset_pool.to_vec();
-
     // Age the asset pool up to the investment year, dropping any assets that have been
     // decommissioned by then
     asset_pool.decommission_old(YEAR);
-    let existing_assets = asset_pool.take();
-
-    // Candidate assets for the investment year
-    let candidates = candidate_assets_for_next_year(
-        &model.processes,
-        Some(YEAR),
-        model.parameters.candidate_asset_capacity,
-    );
-
-    (base_year_assets, existing_assets, candidates)
+    asset_pool.take()
 }
 
-/// Dispatch the base year (with and without the investment-year candidates) to obtain realistic
-/// seed prices for investment appraisal, as per `simulation::run_dispatch_for_year` (this is the
-/// dispatch performed before any agent investment has taken place).
-fn calculate_seed_prices(
-    model: &Model,
-    base_year_assets: &[AssetRef],
-    candidates: &[AssetRef],
-    writer: &mut DataWriter,
-) -> Prices {
-    let market_demands = collect_preset_demands_for_year(&model.commodities, BASE_YEAR);
-    let solution_existing = DispatchRun::new(model, base_year_assets, BASE_YEAR, &market_demands)
-        .run("bench setup: without candidates", writer)
-        .expect("Dispatch without candidates failed");
-    let solution_with_candidates =
-        DispatchRun::new(model, base_year_assets, BASE_YEAR, &market_demands)
-            .with_candidates(candidates)
-            .run("bench setup: with candidates", writer)
-            .expect("Dispatch with candidates failed");
+/// Load recorded base-year prices so benchmark setup does not require a simulation.
+///
+/// The example uses the same full-cost-average strategy for market and fallback prices.
+fn load_seed_prices() -> Prices {
+    #[derive(Deserialize)]
+    struct PriceRow {
+        milestone_year: u32,
+        commodity_id: CommodityID,
+        region_id: RegionID,
+        time_slice: TimeSliceID,
+        price: MoneyPerFlow,
+    }
 
-    calculate_prices_from_solutions(
-        model,
-        &solution_existing,
-        &solution_with_candidates,
-        candidates,
-        &Prices::default(),
-        BASE_YEAR,
-    )
-    .expect("Failed to calculate prices")
+    let mut reader = csv::Reader::from_reader(
+        include_bytes!("../tests/data/two_outputs/commodity_prices.csv").as_slice(),
+    );
+    let mut market = PriceMap::default();
+    for row in reader.deserialize::<PriceRow>() {
+        let row = row.expect("Failed to read benchmark seed prices");
+        if row.milestone_year == BASE_YEAR {
+            market.insert(
+                &row.commodity_id,
+                &row.region_id,
+                &row.time_slice,
+                row.price,
+            );
+        }
+    }
+    assert!(
+        market.iter().next().is_some(),
+        "Benchmark seed prices must include the base year"
+    );
+
+    Prices {
+        fallback: market.clone(),
+        market,
+    }
 }
 
 /// Build `n` synthetic candidate processes, based on cycling through `templates`, so that we can
@@ -144,8 +139,8 @@ fn build_synthetic_processes(templates: &[Arc<Process>], n: usize) -> Vec<Arc<Pr
 #[allow(clippy::too_many_lines)]
 fn criterion_benchmark(c: &mut Criterion) {
     let (model, mut writer, _example_dir, _output_dir) = load_bench_model();
-    let (base_year_assets, existing_assets, candidates) = build_assets_for_investment_year(&model);
-    let prices = calculate_seed_prices(&model, &base_year_assets, &candidates, &mut writer);
+    let existing_assets = build_assets_for_investment_year(&model);
+    let prices = load_seed_prices();
 
     // The market of interest: agent, commodity and region for which assets will be selected
     let commodity = &model.commodities[COMMODITY_ID];

docs/model/README.md

@@ -67,9 +67,9 @@ pre-computed **investment order** derived from the topology of the [commodity gr
 markets are processed deepest-first (most downstream to most upstream), grouped into **layers** of
 independent markets at the same depth. See [Investment Appraisal][investment] for full details.
 
-After all markets in each layer are settled, a **partial system dispatch** is run over all assets
-selected so far. This propagates demand upstream: input commodity flows consumed by newly committed
-assets become demand targets for the markets not yet invested in.
+After all markets in each layer are settled, the activities returned by the agent-level
+optimisations propagate demand upstream: input commodity flows consumed by newly committed assets
+become demand targets for the markets not yet invested in.
 
 ### 3. Mothballing and decommissioning
 

docs/model/commodity_graphs.md

@@ -147,8 +147,8 @@ The investment order is worked out from the commodity graph for each year as fol
    markets that supply it (a [topological sort]).
 4. **Group independent markets into layers.** Each market is given a depth, which is the length of
    the longest supply chain leading to it. Markets with the same depth do not depend on one another,
-   so they are grouped into a **layer**. MUSE2 invests in all markets in a layer before running a
-   single [partial system dispatch][partial-dispatch], rather than one per market, which saves time.
+   so they are grouped into a **layer**. MUSE2 invests in all markets in a layer, then updates
+  [intermediate demands][intermediate-demands] from their selected asset activities.
 
 ### Circularities
 
@@ -267,7 +267,7 @@ When reading these graphs, note that:
 [investment]: ./investment.md
 [prices]: ./prices.md
 [topological sort]: https://en.wikipedia.org/wiki/Topological_sorting
-[partial-dispatch]: ./investment.md#partial-system-dispatch
+[intermediate-demands]: ./investment.md#intermediate-demands
 [order-scc]: ../api/muse2/graph/investment/fn.order_scc.html
 [investment-circularities]: ./investment.md#circularities
 [pricing-circularities]: ./prices.md#circularities

docs/model/investment.md

@@ -23,8 +23,8 @@ demand created by already-committed downstream assets is already known.
 Markets are processed in **layers**: a layer is a set of commodity markets that are independent of
 one another (i.e. at the same depth in the commodity graph and with no dependencies between them).
 All markets within a layer are settled before moving to the next layer. After each layer is
-settled, a single [partial system dispatch](#partial-system-dispatch) is run over all assets
-selected so far. This quantifies the input commodity flows consumed by newly committed assets —
+settled, [intermediate demands](#intermediate-demands) are updated using the asset activities from
+the agent-level optimisations. These determine the input commodity flows of newly committed assets —
 for example, a gas generator committed during electricity market investment will consume gas,
 creating demand that the gas market investment must subsequently meet.
 
@@ -46,8 +46,8 @@ market which is invested in *before* the market they supply (see
 [Commodity Graphs][graph-circularities]). If assets are selected for a feedback process, they create
 extra demand for a market that has already been invested in.
 
-To deal with this, MUSE2 invests in the circularity in two passes over its markets, running a
-[partial system dispatch](#partial-system-dispatch) after each market in each pass:
+To deal with this, MUSE2 invests in the circularity in two passes over its markets, updating
+[intermediate demands](#intermediate-demands) from selected asset activities after each market:
 
 1. **First pass (provisional).** The markets are visited in order, with all processes available,
    including feedback processes. Any feedback assets selected will leave some demand for earlier
@@ -475,47 +475,31 @@ The negative SNAS indicates that at current market prices, this asset does not g
 over its annualised costs. It would still be selected if it has the highest SNAS among all
 available options.
 
-## Partial System Dispatch
+## Intermediate Demands
 
-After each layer of commodity markets is settled during the investment loop, a **partial system
-dispatch** is run over all assets selected so far. Within a [circularity](#circularities), a partial
-dispatch is instead run after each market in each pass, over the assets just selected for that
-market. This is a standard [dispatch optimisation][dispatch-optimisation] solve, with two key
-modifications described below.
+After each layer of commodity markets is settled, the activities returned by the agent-level
+optimisations are converted to commodity flows by multiplying by each asset's flow coefficients.
+Within a [circularity](#circularities), this update happens after each market in each pass. The
+second pass starts from preset demands adjusted only for the retained feedback assets' flows.
 
-### Market subset
-
-Only the commodity markets visited so far — i.e. the current market and all those settled in
-earlier investment rounds — are subject to commodity balance constraints. Markets that are upstream
-of the current investment frontier are not yet constrained, because no assets have been committed
-to serve them yet.
-
-### Input prices for upstream commodities
-
-Because upstream markets are unconstrained, their commodities have no balance constraints and
-therefore no shadow prices in this dispatch. Without any signal on the cost of upstream inputs,
-those inputs would appear free to the solver.
-
-To avoid this, the shadow prices \\( \lambda_{c,r,t} \\) from the previous MSY (see
-[Commodity Prices Used in Appraisal](#commodity-prices-used-in-appraisal)) are passed as explicit
-cost penalties on any input flows corresponding to upstream, unconstrained commodities. This ensures
-the dispatch correctly accounts for the cost of consuming upstream resources, even before those
-markets have been invested in.
-
-### Using the partial dispatch result
-
-The flow map from the partial dispatch is used to update the **net demand** seen by each upstream
-market. For each newly committed asset, its input commodity flows are added to the demand for the
+The resulting flows update the **net demand** seen by each upstream market. For each newly
+selected asset, its input commodity flows are added to the demand for the
 corresponding upstream markets, and its primary output flows reduce the remaining demand for the
 current market. Only primary output flows are counted against demand; secondary outputs do not
 affect the demand targets of other commodity markets. These updated demands then drive the next
 investment decisions as the process moves along the investment order, with each market's committed
-assets shaping the demand seen by those upstream.
+assets shaping the demand seen by those upstream. Flows are aggregated at each commodity's
+time-slice level, and each selection contributes once.
+
+Full-system dispatch still determines prices during the ironing-out loop and the final flows
+reported for each milestone year.
+
+In the base year, agents dispatch only their existing assets. Upstream capacity must therefore
+be sufficient for the input demands implied by downstream agents' own activity profiles.
 
 [framework-overview]: index.html#framework-overview
 [prices]: ./prices.md
 [model-toml]: ../file_formats/input_files.md#model-parameters-modeltoml
 [process-investment-constraints-csv]: ../file_formats/input_files.md#process_investment_constraintscsv
-[dispatch-optimisation]: ./dispatch_optimisation.md
 [graph-circularities]: ./commodity_graphs.md#circularities
 [dispatch-utilisation-penalties]: ./dispatch_optimisation.md#seasonalannual-utilisation-penalties

examples/missing_commodity/assets.csv

@@ -2,6 +2,6 @@ process_id,region_id,agent_id,capacity,commission_year
 GASDRV,GBR,A0_GEX,4002.26,2020
 GASPRC,GBR,A0_GPR,3782.13,2020
 WNDFRM,GBR,A0_ELC,3.964844,2020
-GASCGT,GBR,A0_ELC,2.43,2020
+GASCGT,GBR,A0_ELC,3.0,2020
 RGASBR,GBR,A0_RES,2900.0,2020
 RELCHP,GBR,A0_RES,399.98,2020

examples/simple/assets.csv

@@ -2,6 +2,6 @@ process_id,region_id,agent_id,capacity,commission_year
 GASDRV,GBR,A0_GEX,4002.26,2020
 GASPRC,GBR,A0_GPR,3782.13,2020
 WNDFRM,GBR,A0_ELC,3.964844,2020
-GASCGT,GBR,A0_ELC,2.43,2020
+GASCGT,GBR,A0_ELC,3.0,2020
 RGASBR,GBR,A0_RES,2900.0,2020
 RELCHP,GBR,A0_RES,399.98,2020

examples/two_outputs/assets.csv

@@ -1,11 +1,11 @@
 process_id,region_id,agent_id,capacity,commission_year
 GASDRV,GBR,A0_OAG,4010.2,2020
-OAGRSV,GBR,A0_OAG,1738.05,2020
+OAGRSV,GBR,A0_OAG,2500,2020
 GASPRC,GBR,A0_OAG,3789.64,2020
-OILREF,GBR,A0_OAG,716.5,2020
-OILRF2,GBR,A0_OAG,573.9,2020
+OILREF,GBR,A0_REF,716.5,2020
+OILRF2,GBR,A0_REF,1000,2020
 WNDFRM,GBR,A0_ELC,3.964844,2020
-GASCGT,GBR,A0_ELC,2.999,2020
+GASCGT,GBR,A0_ELC,3.3,2020
 TPETCR,GBR,A0_TRA,451.91,2020
 TDIECR,GBR,A0_TRA,176.58,2020
 TELCCR,GBR,A0_TRA,20.99,2020

schemas/input/model.yaml

@@ -20,16 +20,10 @@ properties:
       inflation. Monetary inputs should use a consistent currency and reference year.
   candidate_asset_capacity:
     type: number
-    description: Capacity to give to candidate assets in dispatch optimisation
+    description: Capacity to give to candidate assets when calculating fallback prices
     default: 0.0001
     notes: |
       The default value should work. Do not change this value unless you know what you're doing!
-  commodity_balance_epsilon:
-    type: number
-    description: Epsilon added to commodity balance lower bounds to force some dispatch by candidate assets
-    default: 1e-6
-    notes: |
-      The default value should work. Do not change this value unless you know what you're doing!
   capacity_tranche_fraction:
     type: number
     description: Scales the size of inferred investment tranches for candidate assets
@@ -59,12 +53,6 @@ properties:
 
       Setting this to `unpriced` uses a fallback price of zero, reverting to the pure
       shadow-price formulation.
-  value_of_lost_load:
-    type: number
-    description: The cost applied to unmet demand
-    default: 1e9
-    notes: |
-      Currently this only applies to the LCOX appraisal.
   seasonal_utilisation_penalty:
     type: number
     description: Penalty per unit of capacity used within a season
@@ -123,10 +111,6 @@ properties:
         type: object
         description: HiGHS options applied to all optimisations
         properties: {}
-      dispatch_options:
-        type: object
-        description: HiGHS options applied only to dispatch optimisations
-        properties: {}
       appraisal_options:
         type: object
         description: HiGHS options applied only to appraisal optimisations

src/asset/pool.rs

@@ -142,7 +142,8 @@ impl AssetPool {
 
     /// Extend the active pool with Commissioned or Ready assets.
     ///
-    /// Returns the newly commissioned assets (those that were in `Ready` state on entry).
+    /// Returns the newly commissioned assets (those that were in `Ready` state on entry),
+    /// in the order they appeared in `assets`.
     pub fn extend<I>(&mut self, assets: I) -> &[AssetRef]
     where
         I: IntoIterator<Item = AssetRef>,