Merge pull request #451 from iiasa/enh/report-extr

Add reporting of resource extraction to IAMC structure

Author: khaeru

doc/api/report/index.rst

@@ -139,13 +139,37 @@ API reference
 .. automodule:: message_ix_models.report
    :members:
 
+   General-purpose code:
+
    .. autosummary::
 
       Config
+      defaults
       prepare_reporter
       register
       report
 
+   The following submodules prepare reporting of specific measures or quantities:
+
+   .. autosummary::
+      :toctree: _autosummary
+      :template: autosummary-module.rst
+      :recursive:
+
+      extraction
+
+.. currentmodule:: message_ix_models.report.key
+
+Keys
+----
+
+.. automodule:: message_ix_models.report.key
+   :members:
+
+   Pre-set :class:`genno.Key` instances
+   and :class:`genno.Keys` in this module can be referenced across modules,
+   instead of hand-typing the same strings in multiple places.
+
 .. currentmodule:: message_ix_models.report.plot
 
 Plots
@@ -167,17 +191,25 @@ Operators
 
    .. autosummary::
 
+      broadcast_wildcard
+      call
       codelist_to_groups
       compound_growth
       exogenous_data
       filter_ts
       from_url
+      get_commodity_groups
       get_ts
       gwp_factors
       make_output_path
       model_periods
+      node_glb
+      nodes_world_agg
       remove_ts
+      select_allow_empty
+      select_expand
       share_curtailment
+      zeros_like
 
    The following functions, defined elsewhere,
    are exposed through :mod:`.operator`
@@ -186,6 +218,8 @@ Operators
    .. autosummary::
 
       message_ix_models.util.add_par_data
+      message_ix_models.util.merge_data
+      message_ix_models.util.nodes_ex_world
 
    Other operators or genno-compatible functions are provided by:
 
@@ -219,6 +253,7 @@ Utilities
 
    .. autosummary::
 
+      IAMCConversion
       add_replacements
       collapse
       collapse_gwp_info

doc/pkg-data/codelists.rst

@@ -18,9 +18,18 @@ These codes have the following annotations:
 ``units`` (mandatory)
    Units typically associated with this commodity.
 ``iea-eweb-flow`` (optional)
-   List of ``FLOW`` codes from the IEA :ref:`tools-iea-web` associated with this MESSAGEix-GLOBIOM commodity.
+   List of ``FLOW`` codes from the IEA :ref:`tools-iea-web`
+   associated with this MESSAGEix-GLOBIOM commodity.
 ``iea-eweb-product`` (optional)
-   List of ``PRODUCT`` codes from the IEA :ref:`tools-iea-web` associated with this MESSAGEix-GLOBIOM commodity.
+   List of ``PRODUCT`` codes from the IEA :ref:`tools-iea-web`
+   associated with this MESSAGEix-GLOBIOM commodity.
+``report`` (optional)
+   English name for IAMC-structured reporting output.
+   See :func:`.add_replacements`.
+``report-only`` (optional)
+   :class:`bool`.
+   If :any:`True`,
+   the code is ignored by :func:`.bare.get_spec`.
 
 .. literalinclude:: ../../message_ix_models/data/commodity.yaml
    :language: yaml

doc/whatsnew.rst

@@ -11,6 +11,11 @@ Next release
   - New method :meth:`~.report.Config.iter_callbacks`.
 
 - New function :func:`.tools.iamc.compare` (:pull:`178`).
+- New module :mod:`.report.extraction` for reporting of resource extraction (:pull:`451`).
+- New reporting operators (:pull:`451`):
+  :func:`.get_commodity_groups`,
+  :func:`.node_glb`, and
+  :func:`.zeros_like`.
 - Expand :doc:`api/report/index` documentation (:pull:`178`)
   to cover features implemented/not implemented by :mod:`genno`-based reporting.
   Module globals :data:`.NOT_IMPLEMENTED_MEASURE` and :data:`.NOT_IMPLEMENTED_IAMC`

message_ix_models/data/commodity.yaml

@@ -6,7 +6,6 @@ biomass:
 coal:
   name: Coal
   units: GWa
-  report: Solids|Fossil
   # NB same value as "Coking coal"; this choice is arbitrary
   # NB in message_doc, this appears as "Hard coal", but this
   #    term usually refers to anthracite, which has a higher
@@ -33,6 +32,45 @@ coal:
   - PEAT
   - SUBCOAL
 
+crude_1:
+  description: Crude oil resource.
+
+crude_2:
+  description: Crude oil resource.
+
+crude_3:
+  description: Crude oil resource.
+
+Oil|Conventional:
+  description: Reporting aggregate for oil resources.
+  report-only: true
+  child: [crude_1, crude_2, crude_3]
+
+crude_4:
+  description: Crude oil resource.
+
+crude_5:
+  description: Crude oil resource.
+
+crude_6:
+  description: Crude oil resource.
+
+crude_7:
+  description: Crude oil resource.
+
+crude_8:
+  description: Crude oil resource.
+
+Oil|Unconventional:
+  description: Reporting aggregate for oil resources.
+  report-only: true
+  child: [crude_4, crude_5, crude_6, crude_7, crude_8]
+
+Oil:
+  description: Reporting aggregate for oil resources.
+  report-only: true
+  child: [Oil|Conventional, Oil|Unconventional]
+
 crudeoil:
   name: Crude oil
   description: >-
@@ -76,10 +114,52 @@ fueloil:
 gas:
   name: Natural Gas
   units: GWa
-  report: Gases
   ipcc-1996-name: "Natural Gas (Dry)"
   iea-eweb-product: [NATGAS]
 
+gas_1:
+  description: Natural gas resource.
+
+gas_2:
+  description: Natural gas resource.
+
+gas_3:
+  description: Natural gas resource.
+
+gas_4:
+  description: Natural gas resource.
+
+Gas|Conventional:
+  description: Reporting aggregate for gas resources.
+  report-only: true
+  child: [gas_1, gas_2, gas_3, gas_4]
+
+gas_5:
+  description: Natural gas resource.
+
+gas_6:
+  description: Natural gas resource.
+
+gas_7:
+  description: Natural gas resource.
+
+gas_8:
+  description: Natural gas resource.
+
+Gas|Unconventional:
+  description: Reporting aggregate for gas resources.
+  report-only: true
+  child: [gas_5, gas_6, gas_7, gas_8]
+
+_Gas:
+  description: >-
+    Reporting aggregate for gas resources.
+    The ID is prefixed with an underscore to distinguish from "gas",
+    which is not a component.
+  report-only: true
+  report: Gas
+  child: [Gas|Conventional, Gas|Unconventional]
+
 hydrogen:
   name: Gaseous hydrogen
   units: GWa
@@ -115,6 +195,15 @@ lignite:
   name: Lignite
   ipcc-1996-name: "Lignite"
 
+_Coal:
+  description: >- 
+    Reporting aggregate for coal resources.
+    The ID is prefixed with an underscore to distinguish from "coal",
+    which is only 1 of multiple children.
+  report-only: true
+  report: Coal
+  child: [coal, lignite]
+
 methanol:
   name: Methanol
   units: GWa
@@ -143,6 +232,15 @@ transport:
   units: GWa
   iea-eweb-flow: [DOMESAIR, DOMESNAV, RAIL, ROAD, TRNONSPE]
 
+uranium:
+  description: Uranium resource.
+
+_Uranium:
+  description: Reporting aggregate for uranium resources.
+  report-only: true
+  report: Uranium
+  child: [uranium]
+
 # The following codes also appear in a recent (2020-02-28) SSP2 scenario, but
 # are not currently used by model.bare.create_res.
 #
@@ -205,14 +303,6 @@ transport:
 # cooling__nuc_lc
 # cooling__solar_th_ppl
 # CrpLnd
-# crude_1
-# crude_2
-# crude_3
-# crude_4
-# crude_5
-# crude_6
-# crude_7
-# crude_8
 # Def_CO2_G4M
 # Def_CO2_GLO
 # dumagr
@@ -260,15 +350,6 @@ transport:
 # freshwater_instream
 # FuelWood
 # FuelWood_G4M
-# gas
-# gas_1
-# gas_2
-# gas_3
-# gas_4
-# gas_5
-# gas_6
-# gas_7
-# gas_8
 # gas_afr
 # gas_cpa
 # gas_eeu
@@ -355,7 +436,6 @@ transport:
 # u5t
 # upstream_landuse
 # uq
-# uranium
 # VOC_LandUseChangeEM
 # VOC_SavanBurnEM
 # water_constraint

message_ix_models/model/bare.py

@@ -129,8 +129,12 @@ def get_spec(context) -> Spec:
     # Add levels
     add.set["level"] = get_codes("level")
 
-    # Add commodities
-    add.set["commodity"] = get_codes("commodity")
+    # Add commodities. Omit those with the annotation ``report-only: true``
+    add.set["commodity"].extend(
+        filter(
+            lambda c: not c.eval_annotation(id="report-only"), get_codes("commodity")
+        )
+    )
 
     # Add units, associated with commodities
     units = set(

message_ix_models/model/transport/structure.py

@@ -1,6 +1,5 @@
 from collections.abc import Sequence
 from copy import deepcopy
-from itertools import chain
 from typing import Any
 
 from sdmx.model.common import Code
@@ -10,6 +9,7 @@
 from message_ix_models.model import disutility
 from message_ix_models.model.structure import generate_set_elements, get_region_codes
 from message_ix_models.util import load_package_data, package_data_path
+from message_ix_models.util.sdmx import leaf_ids
 
 from .util import region_path_fallback
 
@@ -67,22 +67,15 @@ def get_technology_groups(
 
     result: dict[str, list[str]] = {"historical-only": [], "LDV usage": []}
 
-    def _leaf_ids(node) -> list[str]:
-        """Recursively collect leaf IDs."""
-        return list(
-            chain(*[_leaf_ids(c) if len(c.child) else (c.id,) for c in node.child])
-        )
-
     for tech in t_list:
         if len(tech.child):
             # Code with child codes → a group of technologies → store all the leaf IDs
-            result[tech.id] = _leaf_ids(tech)
-        else:
+            result[tech.id] = leaf_ids(tech)
+        elif tech.eval_annotation(id="historical-only") is True:
             # Code without children = an individual technology → add to certain groups
-            if tech.eval_annotation(id="historical-only") is True:
-                result["historical-only"].append(tech.id)
-            if tech.eval_annotation(id="is-disutility") is True:
-                result["LDV usage"].append(tech.id)
+            result["historical-only"].append(tech.id)
+        elif tech.eval_annotation(id="is-disutility") is True:
+            result["LDV usage"].append(tech.id)
 
     return result