1148 Rework Data structure and Rename Mobility and Vaccination file (#1150)

- The name for the vaccination file was outdated in the epidata Package. Updated from all_county_ageinf_vacc to vacc_county_ageinf
- Remove default mobility matrix file.
- Extend function getSimulationData to really provide all files to execute the simulations.
- Remove Twitter data.
- Rework data structure, where the region are now the first instance.
- clean_data cleans all created data via the pycode

Co-authored-by: xsaschako <51127093+xsaschako@users.noreply.github.com>

Author: HenrZu

cpp/examples/CMakeLists.txt

@@ -100,10 +100,6 @@ add_executable(abm_history_example abm_history_object.cpp)
 target_link_libraries(abm_history_example PRIVATE memilio abm)
 target_compile_options(abm_history_example PRIVATE ${MEMILIO_CXX_FLAGS_ENABLE_WARNING_ERRORS})
 
-add_executable(twitter_mobility_example twitter_mobility.cpp)
-target_link_libraries(twitter_mobility_example PRIVATE memilio ode_secir)
-target_compile_options(twitter_mobility_example PRIVATE ${MEMILIO_CXX_FLAGS_ENABLE_WARNING_ERRORS})
-
 add_executable(ide_seir_example ide_seir.cpp)
 target_link_libraries(ide_seir_example PRIVATE memilio ide_seir)
 target_compile_options(ide_seir_example PRIVATE ${MEMILIO_CXX_FLAGS_ENABLE_WARNING_ERRORS})

cpp/examples/ode_secir_read_graph.cpp

@@ -28,8 +28,8 @@
 std::string setup(int argc, char** argv, const std::string data_dir)
 {
     if (argc == 2) {
-        std::cout << "Using file " << argv[1] << " in data/mobility." << std::endl;
-        return mio::path_join(data_dir, "mobility", (std::string)argv[1]);
+        std::cout << "Using file " << argv[1] << " in data/Germany/mobility." << std::endl;
+        return mio::path_join(data_dir, "Germany", "mobility", (std::string)argv[1]);
     }
     else {
         if (argc > 2) {
@@ -38,13 +38,14 @@ std::string setup(int argc, char** argv, const std::string data_dir)
         else {
             mio::log_warning("No arguments given.");
         }
-        std::cout << "Using default file twitter_scaled_1252 in data/mobility." << std::endl;
+        auto mobility_file = "commuter_mobility_2022.txt";
+        std::cout << "Using file " << mobility_file << " in data/Germany/mobility." << std::endl;
         std::cout << "Usage: read_graph MOBILITY_FILE"
                   << "\n\n";
-        std::cout << "This example performs a simulation based on twitter "
-                     "mobility data."
-                  << std::endl;
-        return mio::path_join(data_dir, "mobility", "twitter_scaled_1252.txt");
+        std::cout
+            << "This example performs a simulation based on mobility data from the German Federal Employment Agency."
+            << std::endl;
+        return mio::path_join(data_dir, "Germany", "mobility", mobility_file);
     }
 }
 
@@ -113,21 +114,22 @@ int main(int argc, char** argv)
     auto read_mobility_result = mio::read_mobility_plain(filename);
     if (!read_mobility_result) {
         std::cout << read_mobility_result.error().formatted_message() << '\n';
-        return -1;
+        std::cout << "Create the mobility file with MEmilio Epidata's getCommuterMobility.py file." << '\n';
+        return 0;
     }
-    auto& twitter_mobility_2018 = read_mobility_result.value();
+    auto& commuter_mobility = read_mobility_result.value();
     std::cout << "Done" << std::endl;
 
     std::cout << "Intializing Graph..." << std::flush;
     mio::Graph<mio::osecir::Model<FP>, mio::MobilityParameters<FP>> graph;
-    for (int node = 0; node < twitter_mobility_2018.rows(); node++) {
+    for (int node = 0; node < commuter_mobility.rows(); node++) {
         graph.add_node(node, model);
     }
-    for (int row = 0; row < twitter_mobility_2018.rows(); row++) {
-        for (int col = 0; col < twitter_mobility_2018.cols(); col++) {
+    for (int row = 0; row < commuter_mobility.rows(); row++) {
+        for (int col = 0; col < commuter_mobility.cols(); col++) {
             graph.add_edge(row, col,
                            Eigen::VectorXd::Constant(10 * (size_t)nb_groups,
-                                                     twitter_mobility_2018(row, col) /
+                                                     commuter_mobility(row, col) /
                                                          graph.nodes()[row].property.populations.get_total()));
         }
     }

cpp/examples/twitter_mobility.cpp

@@ -17,8 +17,8 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-#ifndef READ_TWITTER_H
-#define READ_TWITTER_H
+#ifndef MEMILIO_IO_MOBILITY_IO_H
+#define MEMILIO_IO_MOBILITY_IO_H
 
 #include "memilio/io/json_serializer.h"
 #include "memilio/mobility/graph.h"
@@ -214,4 +214,4 @@ IOResult<void> save_edges(const std::vector<std::vector<TimeSeries<double>>>& en
 
 } // namespace mio
 
-#endif // READ_TWITTER_H
+#endif // MEMILIO_IO_MOBILITY_IO_H

cpp/memilio/io/mobility_io.h

@@ -325,7 +325,7 @@ IOResult<void> set_nodes(const Parameters& params, Date start_date, Date end_dat
 /**
  * @brief Sets the graph edges.
  * Reads the commuting matrices from txt files and sets the graph edges with that.
- * @param[in] data_dir Directory that contains the data files.
+ * @param[in] mobility_data_file File that contains the commuting matrix.
  * @param[in, out] params_graph Graph whose nodes are set by the function.
  * @param[in] mobile_compartments Compartments that commute.
  * @param[in] contact_locations_size Number of contact locations.
@@ -335,20 +335,15 @@ IOResult<void> set_nodes(const Parameters& params, Date start_date, Date end_dat
  */
 template <class ContactLocation, class Model, class MobilityParams, class MobilityCoefficientGroup,
           class InfectionState, class ReadFunction>
-IOResult<void> set_edges(const fs::path& data_dir, Graph<Model, MobilityParams>& params_graph,
+IOResult<void> set_edges(const fs::path& mobility_data_file, Graph<Model, MobilityParams>& params_graph,
                          std::initializer_list<InfectionState>& mobile_compartments, size_t contact_locations_size,
                          ReadFunction&& read_func, std::vector<ScalarType> commuting_weights,
                          std::vector<std::vector<size_t>> indices_of_saved_edges = {})
 {
     // mobility between nodes
-    BOOST_OUTCOME_TRY(auto&& mobility_data_commuter,
-                      read_func((data_dir / "mobility" / "commuter_mobility.txt").string()));
-    BOOST_OUTCOME_TRY(auto&& mobility_data_twitter,
-                      read_func((data_dir / "mobility" / "twitter_scaled_1252.txt").string()));
+    BOOST_OUTCOME_TRY(auto&& mobility_data_commuter, read_func(mobility_data_file.string()));
     if (mobility_data_commuter.rows() != Eigen::Index(params_graph.nodes().size()) ||
-        mobility_data_commuter.cols() != Eigen::Index(params_graph.nodes().size()) ||
-        mobility_data_twitter.rows() != Eigen::Index(params_graph.nodes().size()) ||
-        mobility_data_twitter.cols() != Eigen::Index(params_graph.nodes().size())) {
+        mobility_data_commuter.cols() != Eigen::Index(params_graph.nodes().size())) {
         return mio::failure(mio::StatusCode::InvalidValue,
                             "Mobility matrices do not have the correct size. You may need to run "
                             "transformMobilitydata.py from pycode memilio epidata package.");
@@ -377,20 +372,9 @@ IOResult<void> set_edges(const fs::path& data_dir, Graph<Model, MobilityParams>&
                         commuter_coeff_ij * commuting_weights[size_t(age)];
                 }
             }
-            //others
-            auto total_population = populations.get_total();
-            auto twitter_coeff    = mobility_data_twitter(county_idx_i, county_idx_j) /
-                                 total_population; //data is absolute numbers, we need relative
-            for (auto age = AgeGroup(0); age < populations.template size<mio::AgeGroup>(); ++age) {
-                for (auto compartment : mobile_compartments) {
-                    auto coeff_idx = populations.get_flat_index({age, compartment});
-                    mobility_coeffs[size_t(ContactLocation::Other)].get_baseline()[coeff_idx] = twitter_coeff;
-                }
-            }
-
             //only add edges with mobility above thresholds for performance
             //thresholds are chosen empirically so that more than 99% of mobility is covered, approx. 1/3 of the edges
-            if (commuter_coeff_ij > 4e-5 || twitter_coeff > 1e-5) {
+            if (commuter_coeff_ij > 4e-5) {
                 params_graph.add_edge(county_idx_i, county_idx_j, std::move(mobility_coeffs), indices_of_saved_edges);
             }
         }

cpp/memilio/mobility/graph.h

@@ -335,11 +335,10 @@ IOResult<void> export_input_data_county_timeseries(
 }
 #else
 template <class Model>
-IOResult<void>
-export_input_data_county_timeseries(std::vector<Model> models, const std::string& dir, std::vector<int> const& region,
-                                    Date date, const std::vector<double>& scaling_factor_inf, double scaling_factor_icu,
-                                    int num_days, const std::string& divi_data_path,
-                                    const std::string& confirmed_cases_path, const std::string& population_data_path)
+IOResult<void> export_input_data_county_timeseries(
+    std::vector<Model> models, const std::string& results_dir, std::vector<int> const& region, Date date,
+    const std::vector<double>& scaling_factor_inf, double scaling_factor_icu, int num_days,
+    const std::string& divi_data_path, const std::string& confirmed_cases_path, const std::string& population_data_path)
 {
     mio::log_warning("HDF5 not available. Cannot export time series of extrapolated real data.");
     return success();
@@ -352,18 +351,19 @@ export_input_data_county_timeseries(std::vector<Model> models, const std::string
  * @param[in] date Date for which the data should be read.
  * @param[in] scaling_factor_inf Factors by which to scale the confirmed cases of rki data.
  * @param[in] scaling_factor_icu Factor by which to scale the icu cases of divi data.
- * @param[in] dir Directory of files.
+ * @param[in] pydata_dir Directory of files.
  */
 template <class Model>
 IOResult<void> read_input_data_germany(std::vector<Model>& model, Date date,
                                        const std::vector<double>& scaling_factor_inf, double scaling_factor_icu,
-                                       const std::string& dir)
+                                       const std::string& pydata_dir)
 {
     BOOST_OUTCOME_TRY(
-        details::set_divi_data(model, path_join(dir, "germany_divi.json"), {0}, date, scaling_factor_icu));
-    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(dir, "cases_all_age_ma7.json"), {0}, date,
-                                                        scaling_factor_inf));
-    BOOST_OUTCOME_TRY(details::set_population_data(model, path_join(dir, "county_current_population.json"), {0}));
+        details::set_divi_data(model, path_join(pydata_dir, "germany_divi.json"), {0}, date, scaling_factor_icu));
+    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(pydata_dir, "cases_all_age_ma7.json"), {0},
+                                                        date, scaling_factor_inf));
+    BOOST_OUTCOME_TRY(
+        details::set_population_data(model, path_join(pydata_dir, "county_current_population.json"), {0}));
     return success();
 }
 
@@ -374,19 +374,20 @@ IOResult<void> read_input_data_germany(std::vector<Model>& model, Date date,
  * @param[in] state Vector of region keys of states of interest.
  * @param[in] scaling_factor_inf Factors by which to scale the confirmed cases of rki data.
  * @param[in] scaling_factor_icu Factor by which to scale the icu cases of divi data.
- * @param[in] dir Directory of files.
+ * @param[in] pydata_dir Directory of files.
  */
 template <class Model>
 IOResult<void> read_input_data_state(std::vector<Model>& model, Date date, std::vector<int>& state,
                                      const std::vector<double>& scaling_factor_inf, double scaling_factor_icu,
-                                     const std::string& dir)
+                                     const std::string& pydata_dir)
 {
 
     BOOST_OUTCOME_TRY(
-        details::set_divi_data(model, path_join(dir, "state_divi.json"), state, date, scaling_factor_icu));
-    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(dir, "cases_all_state_age_ma7.json"), state,
-                                                        date, scaling_factor_inf));
-    BOOST_OUTCOME_TRY(details::set_population_data(model, path_join(dir, "county_current_population.json"), state));
+        details::set_divi_data(model, path_join(pydata_dir, "state_divi.json"), state, date, scaling_factor_icu));
+    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(pydata_dir, "cases_all_state_age_ma7.json"),
+                                                        state, date, scaling_factor_inf));
+    BOOST_OUTCOME_TRY(
+        details::set_population_data(model, path_join(pydata_dir, "county_current_population.json"), state));
     return success();
 }
 
@@ -397,33 +398,32 @@ IOResult<void> read_input_data_state(std::vector<Model>& model, Date date, std::
  * @param[in] county Vector of region keys of counties of interest.
  * @param[in] scaling_factor_inf Factors by which to scale the confirmed cases of rki data.
  * @param[in] scaling_factor_icu Factor by which to scale the icu cases of divi data.
- * @param[in] dir Directory of files.
+ * @param[in] pydata_dir Directory of files.
  * @param[in] num_days [Default: 0] Number of days to be simulated; required to extrapolate real data.
  * @param[in] export_time_series [Default: false] If true, reads data for each day of simulation and writes it in the same directory as the input files.
  */
 template <class Model>
 IOResult<void> read_input_data_county(std::vector<Model>& model, Date date, const std::vector<int>& county,
                                       const std::vector<double>& scaling_factor_inf, double scaling_factor_icu,
-                                      const std::string& dir, int num_days = 0, bool export_time_series = false)
+                                      const std::string& pydata_dir, int num_days = 0, bool export_time_series = false)
 {
-    BOOST_OUTCOME_TRY(details::set_divi_data(model, path_join(dir, "pydata/Germany", "county_divi_ma7.json"), county,
-                                             date, scaling_factor_icu));
-    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(
-        model, path_join(dir, "pydata/Germany", "cases_all_county_age_ma7.json"), county, date, scaling_factor_inf));
-    BOOST_OUTCOME_TRY(details::set_population_data(
-        model, path_join(dir, "pydata/Germany", "county_current_population.json"), county));
+    BOOST_OUTCOME_TRY(
+        details::set_divi_data(model, path_join(pydata_dir, "county_divi_ma7.json"), county, date, scaling_factor_icu));
+    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(pydata_dir, "cases_all_county_age_ma7.json"),
+                                                        county, date, scaling_factor_inf));
+    BOOST_OUTCOME_TRY(
+        details::set_population_data(model, path_join(pydata_dir, "county_current_population.json"), county));
 
     if (export_time_series) {
         // Use only if extrapolated real data is needed for comparison. EXPENSIVE !
         // Run time equals run time of the previous functions times the num_days !
         // (This only represents the vectorization of the previous function over all simulation days...)
         log_warning("Exporting time series of extrapolated real data. This may take some minutes. "
                     "For simulation runs over the same time period, deactivate it.");
-        BOOST_OUTCOME_TRY(
-            export_input_data_county_timeseries(model, dir, county, date, scaling_factor_inf, scaling_factor_icu,
-                                                num_days, path_join(dir, "pydata/Germany", "county_divi_ma7.json"),
-                                                path_join(dir, "pydata/Germany", "cases_all_county_age_ma7.json"),
-                                                path_join(dir, "pydata/Germany", "county_current_population.json")));
+        BOOST_OUTCOME_TRY(export_input_data_county_timeseries(
+            model, pydata_dir, county, date, scaling_factor_inf, scaling_factor_icu, num_days,
+            path_join(pydata_dir, "county_divi_ma7.json"), path_join(pydata_dir, "cases_all_county_age_ma7.json"),
+            path_join(pydata_dir, "county_current_population.json")));
     }
     return success();
 }
@@ -435,19 +435,19 @@ IOResult<void> read_input_data_county(std::vector<Model>& model, Date date, cons
  * @param[in] county vector of region keys of interest
  * @param[in] scaling_factor_inf factors by which to scale the confirmed cases of rki data
  * @param[in] scaling_factor_icu factor by which to scale the icu cases of divi data
- * @param[in] dir directory of files
+ * @param[in] pydata_dir directory of files
  * @param[in] age_group_names strings specifying age group names
  */
 template <class Model>
 IOResult<void> read_input_data(std::vector<Model>& model, Date date, const std::vector<int>& node_ids,
                                const std::vector<double>& scaling_factor_inf, double scaling_factor_icu,
-                               const std::string& data_dir, int num_days = 0, bool export_time_series = false)
+                               const std::string& pydata_dir, int num_days = 0, bool export_time_series = false)
 {
-    BOOST_OUTCOME_TRY(
-        details::set_divi_data(model, path_join(data_dir, "critical_cases.json"), node_ids, date, scaling_factor_icu));
-    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(data_dir, "confirmed_cases.json"), node_ids,
+    BOOST_OUTCOME_TRY(details::set_divi_data(model, path_join(pydata_dir, "critical_cases.json"), node_ids, date,
+                                             scaling_factor_icu));
+    BOOST_OUTCOME_TRY(details::set_confirmed_cases_data(model, path_join(pydata_dir, "confirmed_cases.json"), node_ids,
                                                         date, scaling_factor_inf));
-    BOOST_OUTCOME_TRY(details::set_population_data(model, path_join(data_dir, "population_data.json"), node_ids));
+    BOOST_OUTCOME_TRY(details::set_population_data(model, path_join(pydata_dir, "population_data.json"), node_ids));
 
     if (export_time_series) {
         // Use only if extrapolated real data is needed for comparison. EXPENSIVE !
@@ -456,9 +456,9 @@ IOResult<void> read_input_data(std::vector<Model>& model, Date date, const std::
         log_warning("Exporting time series of extrapolated real data. This may take some minutes. "
                     "For simulation runs over the same time period, deactivate it.");
         BOOST_OUTCOME_TRY(export_input_data_county_timeseries(
-            model, data_dir, node_ids, date, scaling_factor_inf, scaling_factor_icu, num_days,
-            path_join(data_dir, "critical_cases.json"), path_join(data_dir, "confirmed_cases.json"),
-            path_join(data_dir, "population_data.json")));
+            model, pydata_dir, node_ids, date, scaling_factor_inf, scaling_factor_icu, num_days,
+            path_join(pydata_dir, "critical_cases.json"), path_join(pydata_dir, "confirmed_cases.json"),
+            path_join(pydata_dir, "population_data.json")));
     }
     return success();
 }