diff --git a/Cargo.lock b/Cargo.lock index 07986bbb8e..9fb7490aa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,6 +775,7 @@ dependencies = [ "abstutil", "anyhow", "csv", + "elevation", "fs-err", "geom", "kml", @@ -1321,6 +1322,14 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "elevation" +version = "0.1.0" +source = "git+https://github.com/dabreegster/elevation#e8d0fd4e11774b6204474be2ecb5ad7d05636f2e" +dependencies = [ + "georaster", +] + [[package]] name = "embedded-hal" version = "0.2.7" @@ -1959,6 +1968,14 @@ dependencies = [ "serde_json", ] +[[package]] +name = "georaster" +version = "0.1.0" +source = "git+https://github.com/pka/georaster#dca2d8b7c4e40c6d3b83f29352af5c04083a366b" +dependencies = [ + "tiff", +] + [[package]] name = "geozero" version = "0.9.9" @@ -2670,6 +2687,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" + [[package]] name = "js-sys" version = "0.3.65" @@ -5031,6 +5054,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiff" +version = "0.9.0" +source = "git+https://github.com/image-rs/image-tiff#4a2e76858b976ae1e9838f4b20772543e3940944" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "tiny-skia" version = "0.6.6" diff --git a/apps/game/src/edit/mod.rs b/apps/game/src/edit/mod.rs index d46ead1280..f7d6d687d8 100644 --- a/apps/game/src/edit/mod.rs +++ b/apps/game/src/edit/mod.rs @@ -18,7 +18,7 @@ pub use self::roads::RoadEditor; pub use self::routes::RouteEditor; pub use self::stop_signs::StopSignEditor; pub use self::traffic_signals::TrafficSignalEditor; -pub use self::validate::{check_sidewalk_connectivity}; +pub use self::validate::check_sidewalk_connectivity; use crate::app::{App, Transition}; use crate::common::{tool_panel, CommonState, Warping}; use crate::debug::DebugMode; diff --git a/cloud/worker_script.sh b/cloud/worker_script.sh index 703cdca39c..0299a4499d 100755 --- a/cloud/worker_script.sh +++ b/cloud/worker_script.sh @@ -32,22 +32,13 @@ mv abstreet-importer/dev/data/input data/input rm -rf abstreet-importer find data/input -name '*.gz' -print -exec gunzip '{}' ';' -# Set up Docker, for the elevation data -sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -echo \ - "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +# Install GDAL sudo apt-get update -# Also sneak GDAL in there -sudo apt-get install -y docker-ce docker-ce-cli containerd.io libgdal-dev +sudo apt-get install -y libgdal-dev # Now do the big import! rm -fv data/input/us/seattle/raw_maps/huge_seattle.bin data/input/us/seattle/popdat.bin -# Run this as root so Docker works. We could add the current user to the group, -# but then we have to fiddle with the shell a weird way to pick up the change -# immediately. -sudo ./target/release/cli regenerate-everything --shard-num=$WORKER_NUM --num-shards=$NUM_WORKERS +./target/release/cli regenerate-everything --shard-num=$WORKER_NUM --num-shards=$NUM_WORKERS # Upload the results ./target/release/updater incremental-upload --version=$EXPERIMENT_TAG diff --git a/convert_osm/Cargo.toml b/convert_osm/Cargo.toml index 73d9d166b2..e4c1f1193e 100644 --- a/convert_osm/Cargo.toml +++ b/convert_osm/Cargo.toml @@ -9,6 +9,7 @@ abstio = { path = "../abstio" } abstutil = { path = "../abstutil" } anyhow = { workspace = true } csv = { workspace = true } +elevation = { git = "https://github.com/dabreegster/elevation" } fs-err = { workspace = true } geom = { workspace = true } kml = { path = "../kml" } diff --git a/convert_osm/src/elevation.rs b/convert_osm/src/elevation.rs index fb391d792a..d48fab9798 100644 --- a/convert_osm/src/elevation.rs +++ b/convert_osm/src/elevation.rs @@ -1,148 +1,39 @@ -use std::io::{BufRead, BufReader, BufWriter, Write}; -use std::process::Command; +use std::collections::HashMap; +use std::io::BufReader; use anyhow::Result; +use elevation::GeoTiffElevation; use fs_err::File; -use osm2streets::IntersectionID; - use geom::Distance; -use raw_map::RawMap; - -pub fn add_data(map: &mut RawMap) -> Result<()> { - let input = format!("elevation_input_{}", map.name.as_filename()); - let output = format!("elevation_output_{}", map.name.as_filename()); - - // TODO It'd be nice to include more timing breakdown here, but if we bail out early, - // it's tedious to call timer.stop(). - let ids = generate_input(&input, map)?; - - fs_err::create_dir_all(&output)?; - fs_err::create_dir_all(abstio::path_shared_input("elevation"))?; - let pwd = std::env::current_dir()?.display().to_string(); - // Because elevation_lookups has so many dependencies, just depend on Docker. - // TODO This is only going to run on Linux, unless we can also build images for other OSes. - // TODO On Linux, data/input/shared/elevation files wind up being owned by root, due to how - // docker runs. For the moment, one workaround is to manually fix the owner afterwards: - // find data/ -user root -exec sudo chown $USER:$USER '{}' \; - let status = Command::new("docker") - .arg("run") - // Bind the input directory to the temporary place we just created - .arg("--mount") - .arg(format!( - "type=bind,source={pwd}/{input},target=/elevation/input,readonly" - )) - // We want to cache the elevation data sources in A/B Street's S3 bucket, so bind to - // our data/input/shared directory. - .arg("--mount") - .arg(format!( - "type=bind,source={},target=/elevation/data", - // Docker requires absolute paths - format!("{}/{}", pwd, abstio::path_shared_input("elevation")) - )) - .arg("--mount") - .arg(format!( - "type=bind,source={pwd}/{output},target=/elevation/output", - )) - .arg("-t") - // https://hub.docker.com/r/abstreet/elevation_lookups - .arg("abstreet/elevation_lookups") - .arg("python3") - .arg("main.py") - .arg("query") - // TODO How to tune this? Pretty machine dependant, and using ALL available cores may - // melt memory. - .arg("--n_threads=1") - .status()?; - if !status.success() { - bail!("Command failed: {}", status); - } - - scrape_output(&output, map, ids)?; - - // Clean up temporary files - fs_err::remove_file(format!("{input}/query"))?; - fs_err::remove_dir(input)?; - fs_err::remove_file(format!("{output}/query"))?; - fs_err::remove_dir(output)?; - Ok(()) -} +use abstutil::Timer; +use raw_map::RawMap; -fn generate_input(input: &str, map: &RawMap) -> Result> { - fs_err::create_dir_all(input)?; - let mut f = BufWriter::new(File::create(format!("{input}/query"))?); - let mut ids = Vec::new(); +pub fn add_data(map: &mut RawMap, path: &str, timer: &mut Timer) -> Result<()> { + // Get intersection points from road endpoints, to reduce the number of elevation lookups + let mut intersection_points = HashMap::new(); for r in map.streets.roads.values() { - ids.push((r.src_i, r.dst_i)); - // Sample points along the road. Smaller step size gives more detail, but is slower. - let mut pts = Vec::new(); - for (pt, _) in r - .reference_line - .step_along(Distance::meters(5.0), Distance::ZERO) - { - pts.push(pt); - } - // Always ask for the intersection - if *pts.last().unwrap() != r.reference_line.last_pt() { - pts.push(r.reference_line.last_pt()); + for (i, pt) in [ + (r.src_i, r.reference_line.first_pt()), + (r.dst_i, r.reference_line.last_pt()), + ] { + intersection_points.insert(i, pt.to_gps(&map.streets.gps_bounds)); } - for (idx, gps) in map - .streets - .gps_bounds - .convert_back(&pts) - .into_iter() - .enumerate() - { - write!(f, "{},{}", gps.x(), gps.y())?; - if idx != pts.len() - 1 { - write!(f, " ")?; - } - } - writeln!(f)?; } - Ok(ids) -} -fn scrape_output( - output: &str, - map: &mut RawMap, - ids: Vec<(IntersectionID, IntersectionID)>, -) -> Result<()> { - let num_ids = ids.len(); - let mut cnt = 0; - for (line, (src_i, dst_i)) in BufReader::new(File::open(format!("{output}/query"))?) - .lines() - .zip(ids) - { - cnt += 1; - let line = line?; - let mut values = Vec::new(); - for x in line.split('\t') { - if let Ok(x) = x.parse::() { - if !x.is_finite() { - // TODO Warn - continue; - } - if x < 0.0 { - // TODO Temporary - continue; - } - values.push(Distance::meters(x)); - } else { - // Blank lines mean the tool failed to figure out what happened + // TODO Download the file if needed? + let mut elevation = GeoTiffElevation::new(BufReader::new(File::open(path)?)); + + timer.start_iter("lookup elevation", intersection_points.len()); + for (i, gps) in intersection_points { + timer.next(); + if let Some(height) = elevation.get_height_for_lon_lat(gps.x() as f32, gps.y() as f32) { + if height < 0.0 { continue; } + map.elevation_per_intersection + .insert(i, Distance::meters(height.into())); } - if values.len() != 4 { - error!("Elevation output line \"{}\" doesn't have 4 numbers", line); - continue; - } - // TODO Also put total_climb and total_descent on the roads - map.elevation_per_intersection.insert(src_i, values[0]); - map.elevation_per_intersection.insert(dst_i, values[1]); - } - if cnt != num_ids { - bail!("Output had {} lines, but we made {} queries", cnt, num_ids); } // Calculate the incline for each road here, before the road gets trimmed for intersection diff --git a/convert_osm/src/lib.rs b/convert_osm/src/lib.rs index 6bd16bd32c..3eb850df87 100644 --- a/convert_osm/src/lib.rs +++ b/convert_osm/src/lib.rs @@ -1,6 +1,4 @@ #[macro_use] -extern crate anyhow; -#[macro_use] extern crate log; use std::collections::{HashMap, HashSet}; @@ -29,7 +27,8 @@ pub struct Options { pub extra_buildings: Option, /// Configure public transit using this URL to a static GTFS feed in .zip format. pub gtfs_url: Option, - pub elevation: bool, + /// Path to a GeoTIFF file in EPSG:4326 to use for elevation data + pub elevation_geotiff: Option, /// Only include crosswalks that match a `highway=crossing` OSM node. pub filter_crosswalks: bool, } @@ -43,7 +42,7 @@ impl Options { private_offstreet_parking: PrivateOffstreetParking::FixedPerBldg(1), extra_buildings: None, gtfs_url: None, - elevation: false, + elevation_geotiff: None, filter_crosswalks: false, } } @@ -138,9 +137,9 @@ pub fn convert( filter_crosswalks(&mut map, extract.crossing_nodes, pt_to_road, timer); } - if opts.elevation { + if let Some(ref path) = opts.elevation_geotiff { timer.start("add elevation data"); - if let Err(err) = elevation::add_data(&mut map) { + if let Err(err) = elevation::add_data(&mut map, path, timer) { error!("No elevation data: {}", err); } timer.stop("add elevation data"); diff --git a/importer/src/map_config.rs b/importer/src/map_config.rs index 15af0fd441..7428114895 100644 --- a/importer/src/map_config.rs +++ b/importer/src/map_config.rs @@ -76,6 +76,12 @@ pub fn config_for_map(name: &MapName) -> convert_osm::Options { None }, // We only have a few elevation sources working - elevation: name.city == CityName::new("us", "seattle") || name.city.country == "gb", + elevation_geotiff: if name.city == CityName::new("us", "seattle") { + Some("data/input/shared/elevation/king_county_2016_lidar.tif".to_string()) + } else if name.city.country == "gb" { + Some("data/input/shared/elevation/UK-dem-50m-4326.tif".to_string()) + } else { + None + }, } }