Merge pull request #19 from CMB-S4/multiple_toml

Multiple toml
CMB-S4 · Dec 2, 2020 · d258ede · d258ede
2 parents 13dcb2d + 88651d7
commit d258ede
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 19 deletions.
diff --git a/02_atmosphere.ipynb b/02_atmosphere.ipynb
@@ -194,7 +194,7 @@
     "        / (365.25 * config[\"experiment\"][\"observing_efficiency\"])\n",
     "    )\n",
     "    atmosphere_map /= np.sqrt(get_telecope_years(config, site, channel))\n",
-    "    \n",
+    "\n",
     "    atmosphere_map[0] *= config[\"experiment\"].get(\"atmosphere_scaling_T\", 1)\n",
     "    atmosphere_map[1:] *= config[\"experiment\"].get(\"atmosphere_scaling_P\", 1)\n",
     "\n",

diff --git a/05_cli.ipynb b/05_cli.ipynb
@@ -77,9 +77,35 @@
     "\n",
     "def md5sum_file(filename):\n",
     "    \"\"\"Compute md5 checksum of the contents of a file\"\"\"\n",
+    "\n",
     "    return md5sum_string(open(filename, \"r\").read())"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# exports\n",
+    "\n",
+    "import collections\n",
+    "\n",
+    "\n",
+    "def merge_dict(d1, d2):\n",
+    "    \"\"\"\n",
+    "    Modifies d1 in-place to contain values from d2.  If any value\n",
+    "    in d1 is a dictionary (or dict-like), *and* the corresponding\n",
+    "    value in d2 is also a dictionary, then merge them in-place.\n",
+    "    \"\"\"\n",
+    "    for k, v2 in d2.items():\n",
+    "        v1 = d1.get(k)  # returns None if v1 has no value for this key\n",
+    "        if isinstance(v1, collections.Mapping) and isinstance(v2, collections.Mapping):\n",
+    "            merge_dict(v1, v2)\n",
+    "        else:\n",
+    "            d1[k] = v2"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -95,7 +121,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert md5sum_file(\"s4_reference_design.toml\") == console_md5sum[0].split()[0]"
+    "console_md5sum"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert md5sum_file(\"s4_reference_design.toml\") == console_md5sum[-1].split()[0]"
    ]
   },
   {
@@ -113,14 +148,25 @@
     "\n",
     "        Parameters\n",
     "        ----------\n",
-    "        config : filename\n",
+    "        config : str or Path or List\n",
     "            CMB-S4 configuration stored in a TOML file\n",
     "            see for example s4_reference_design.toml in the repository\n",
+    "            It also supports multiple TOML files as a List, in this case\n",
+    "            later files override configuration files of the earlier files.\n",
+    "            check the `config` attribute to verify that the parsing behaved\n",
+    "            as expected.\n",
     "        output_folder : str or Path\n",
     "            Output path\n",
     "        \"\"\"\n",
-    "        self.config_filename = config_filename\n",
-    "        self.config = toml.load(self.config_filename)\n",
+    "        self.config_filename = (\n",
+    "            [config_filename]\n",
+    "            if isinstance(config_filename, (str, Path))\n",
+    "            else config_filename\n",
+    "        )\n",
+    "\n",
+    "        self.config = toml.load(self.config_filename[0])\n",
+    "        for conf in self.config_filename[1:]:\n",
+    "            merge_dict(self.config, toml.load(conf))\n",
     "        self.output_filename_template = \"cmbs4_KCMB_{telescope}-{band}_{site}_nside{nside}_{split}_of_{nsplits}.fits\"\n",
     "        self.output_folder = Path(output_folder)\n",
     "        self.output_folder.mkdir(parents=True, exist_ok=True)\n",
@@ -144,6 +190,7 @@
     "        assert (\n",
     "            nsplits < 8\n",
     "        ), \"We currently only have 7 independent realizations of atmosphere and noise\"\n",
+    "        conf_md5 = \",\".join(map(md5sum_file, self.config_filename))\n",
     "        for site in sites:\n",
     "            for channel in parse_channels(channels):\n",
     "\n",
@@ -211,13 +258,15 @@
     "                            (\"NSPLITS\", nsplits),\n",
     "                            (\"CHANNEL\", channel),\n",
     "                            (\"DATE\", str(date.today())),\n",
-    "                            (\"CONFMD5\", md5sum_file(self.config_filename)),\n",
+    "                            (\"CONFMD5\", conf_md5),\n",
     "                        ],\n",
     "                        coord=\"Q\",\n",
     "                        overwrite=True,\n",
     "                    )\n",
     "                    # only run of full mission and the first split\n",
-    "                    if split in [0, 1] and self.config[\"experiment\"].get(\"include_noise\", False):\n",
+    "                    if split in [0, 1] and self.config[\"experiment\"].get(\n",
+    "                        \"include_noise\", False\n",
+    "                    ):\n",
     "\n",
     "                        if split == 0:\n",
     "                            log.info(\n",
@@ -241,7 +290,7 @@
     "                                (\"NSPLITS\", nsplits),\n",
     "                                (\"CHANNEL\", channel),\n",
     "                                (\"DATE\", str(date.today())),\n",
-    "                                (\"CONFMD5\", md5sum_file(self.config_filename)),\n",
+    "                                (\"CONFMD5\", conf_md5),\n",
     "                            ],\n",
     "                            coord=\"Q\",\n",
     "                            overwrite=True,\n",
@@ -291,7 +340,7 @@
     "    import argparse\n",
     "\n",
     "    parser = argparse.ArgumentParser(description=\"Run s4_design_sim_tool\")\n",
-    "    parser.add_argument(\"config\", type=str, help=\"Configuration file\")\n",
+    "    parser.add_argument(\"config\", type=str, nargs=\"*\", help=\"TOML Configuration files\")\n",
     "    parser.add_argument(\n",
     "        \"--channels\",\n",
     "        type=str,\n",
@@ -340,15 +389,6 @@
     "sim = S4RefSimTool(\"s4_reference_design.toml\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%ls /global/cscratch1/sd/keskital/s4sim/reference_tool/out/00000000/*atmo*"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -383,7 +423,7 @@
    "outputs": [],
    "source": [
     "output_map, header = hp.read_map(\n",
-    "    \"output/SAT-LFS1_pole/cmbs4_sky_atmosphere_noise_KCMB_SAT-LFS1_pole_nside512_1_of_1.fits\", (0,1,2),\n",
+    "    \"output/SAT-LFS1_pole/cmbs4_KCMB_SAT-LFS1_pole_nside512_1_of_1.fits\", (0,1,2),\n",
     "h=True)"
    ]
   },

diff --git a/06_ui.ipynb b/06_ui.ipynb
@@ -19,6 +19,15 @@
     "For support requests, [open an issue on the `s4_design_sim_tool` repository](https://github.com/CMB-S4/s4_design_sim_tool/issues)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# default_exp ui"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -841,6 +850,7 @@
    "source": [
     "#export\n",
     "\n",
+    "\n",
     "def create_wget_script(folder, output_location):\n",
     "    with open(folder / \"download_all.sh\", \"w\") as f:\n",
     "        f.write(\"#!/bin/bash\\n\")\n",