Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/fix bm #146

Merged
merged 8 commits into from
Feb 18, 2021
Merged

Feature/fix bm #146

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0e2f8bf
bug_fix bm
aleeciu Feb 10, 2021
acdc729
various
aleeciu Feb 15, 2021
42a5dc6
Merge branch 'develop' into feature/fix_bm
aleeciu Feb 15, 2021
fc91606
fix test
aleeciu Feb 15, 2021
40a6b86
requested changes and similar found elsewhere
aleeciu Feb 15, 2021
194c60a
un-comment
aleeciu Feb 15, 2021
4e7f79c
req_files brought back
aleeciu Feb 15, 2021
18cee61
remove white space and added TODO
aleeciu Feb 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion climada/entity/exposures/black_marble.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def set_countries(self, countries, ref_year=2016, res_km=None, from_hr=None,
independently of its year of acquisition.
admin_file (str): file name, admin_0_countries or admin_0_map_subunits
kwargs (optional): 'gdp' and 'inc_grp' dictionaries with keys the
country ISO_alpha3 code. 'poly_val' polynomial transformation
country ISO_alpha3 code. 'poly_val' list of polynomial coefficients
[1,x,x^2,...] to apply to nightlight (DEF_POLY_VAL used if not
provided). If provided, these are used.
"""
Expand Down
31 changes: 16 additions & 15 deletions climada/entity/exposures/nightlight.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from climada.util.constants import SYSTEM_DIR
from climada.util.files_handler import download_file
from climada.util.save import save
from .litpop import read_bm_file

Image.MAX_IMAGE_PIXELS = 1e9

Expand Down Expand Up @@ -228,7 +229,7 @@ def load_nightlight_nasa(bounds, req_files, year):

Parameters:
bounds (tuple): min_lon, min_lat, max_lon, max_lat
req_files (np.array): array with flags for NASA files needed
req_files (np.array): array with flags for NASA files needed
tovogt marked this conversation as resolved.
Show resolved Hide resolved
year (int): nightlight year

Returns:
Expand All @@ -253,12 +254,12 @@ def load_nightlight_nasa(bounds, req_files, year):
continue
extent = np.int64(np.clip(extent, 0, tile_size[None] - 1))

fname = SYSTEM_DIR.joinpath(fname.replace('*', str(year)))
with Image.open(fname, "r") as im_nl:
im_nl = im_nl.transpose(method=Image.FLIP_TOP_BOTTOM).getchannel(0)
im_nl = sparse.csc.csc_matrix(im_nl)
im_nl = im_nl[extent[0, 0]:extent[1, 0] + 1, extent[0, 1]:extent[1, 1] + 1]
nightlight.append((tile_coord, im_nl))
im_nl, _ = read_bm_file(SYSTEM_DIR, fname.replace('*', str(year)))
im_nl = np.flipud(im_nl)
im_nl = sparse.csc.csc_matrix(im_nl)
im_nl = im_nl[extent[0, 0]:extent[1, 0] + 1, extent[0, 1]:extent[1, 1] + 1]
nightlight.append((tile_coord, im_nl))

tile_coords = np.array([n[0] for n in nightlight])
shape = tile_coords.max(axis=0) - tile_coords.min(axis=0) + 1
nightlight = np.array([n[1] for n in nightlight]).reshape(shape, order='F')
Expand All @@ -281,17 +282,17 @@ def unzip_tif_to_py(file_gz):
sparse.csr_matrix (nightlight)
"""
LOGGER.info("Unzipping file %s.", file_gz)
file_path = Path(file_gz).stem
file_name = Path(Path(file_gz).stem)
with gzip.open(file_gz, 'rb') as f_in:
with file_path.open('wb') as f_out:
with file_name.open('wb') as f_out:
shutil.copyfileobj(f_in, f_out)
nightlight = sparse.csc.csc_matrix(plt.imread(file_path))
nightlight = sparse.csc.csc_matrix(plt.imread(file_name))
# flip X axis
nightlight.indices = -nightlight.indices + nightlight.shape[0] - 1
nightlight = nightlight.tocsr()
file_path.unlink()
file_name = file_path.stem + ".p"
save(file_name, nightlight)
file_name.unlink()
file_path = SYSTEM_DIR.joinpath(file_name.stem + ".p")
save(file_path, nightlight)

return file_name, nightlight

Expand Down Expand Up @@ -363,7 +364,7 @@ def load_nightlight_noaa(ref_year=2013, sat_name=None):
for pre_i in np.arange(ini_pre, end_pre, -1):
url = NOAA_SITE + 'F' + str(pre_i) + str(ref_year) + '.v4.tar'
try:
file_down = download_file(url)
file_down = download_file(url, download_dir=SYSTEM_DIR)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'm still not fully acquainted with the outcomes of the new config settings, but shouldn't file downloads etc. be configurable by setting the path in the config accordingly, as opposed to hard-coding system_dir as the option of choice here?

Copy link
Collaborator Author

@aleeciu aleeciu Feb 15, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's what I thought too.. but it was not the case for me when I ran it, it was saving the downloaded files in my working directory

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe @emanuel-schmid can comment on this?

I'm not sure what exactly is happening here, but the rest of this PR is fine from my side.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep, same from my side, all good, but would be nice if Emanuel could explain / help out on this one.

Copy link
Collaborator

@emanuel-schmid emanuel-schmid Feb 17, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The parameter download_dir of the download_file functions was introduced with the changes of the config settings.
(Before the changes, it used to download files into the working directory. Which was sometimes countered by undesired os.chdir() calls.)
The function's default directory is CONFIG.local_data.save_dir. The configuration's default value for this is ./results.

SYSTEM_DIR is a constant, but it's not exactly hard coded. Its value is defined as CONFIG.local_data.system.

I hope these explanations help clarify the behaviour?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SYSTEM_DIR is a constant, but it's not exactly hard coded. Its value is defined as CONFIG.local_data.system.

Thanks! I think that is the answer to @Evelyn-M's original question and solves this confusion.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks Emanuel, yes, it was creating a ./results folder in the working directory and saving files therein. Problem is, the loading function called after was expecting the files to be in SYSTEM_DIR, so that was the problem.

So maybe that also explaines why the same did not crash on Jenkins? (see #143) Can it be that there it is CONFIG.local_data.save_dir == CONFIG.local_data.system ? (my guess)

break
except ValueError:
pass
Expand All @@ -374,7 +375,7 @@ def load_nightlight_noaa(ref_year=2013, sat_name=None):
else:
url = NOAA_SITE + sat_name + str(ref_year) + '.v4.tar'
try:
file_down = download_file(url)
file_down = download_file(url, download_dir=SYSTEM_DIR)
except ValueError:
LOGGER.error('Nightlight intensities for year %s and satellite'
' %s do not exist.', ref_year, sat_name)
Expand Down
5 changes: 2 additions & 3 deletions climada/test/test_blackmarble.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def test_spain_pass(self):
cm.output[0])
self.assertIn("Processing country Spain.", cm.output[1])
self.assertIn("Generating resolution of approx 1 km.", cm.output[2])
self.assertTrue(np.isclose(ent.gdf.value.sum(), 1.362e+12 * (4 + 1), 4))
self.assertTrue(np.isclose(ent.gdf.value.sum(), 1.355e+12 * (4 + 1), 0.001))
self.assertTrue(equal_crs(ent.crs['init'], {'init': 'epsg:4326'}))
self.assertEqual(ent.meta['width'], 2699)
self.assertEqual(ent.meta['height'], 1938)
Expand All @@ -74,7 +74,7 @@ def test_sint_maarten_pass(self):
cm.output[0])
self.assertIn("Processing country Sint Maarten.", cm.output[1])
self.assertIn("Generating resolution of approx 0.2 km.", cm.output[2])
self.assertAlmostEqual(ent.gdf.value.sum(), 3.658e+08 * (4 + 1))
self.assertTrue(np.isclose(ent.gdf.value.sum(), 1.023e+09 * (4 + 1), 0.001))
self.assertTrue(equal_crs(ent.crs['init'], {'init': 'epsg:4326'}))

def test_anguilla_pass(self):
Expand Down Expand Up @@ -131,7 +131,6 @@ def test_from_hr_flag_pass(self):
print('MemoryError caught')
pass


ent = BlackMarble()
with self.assertLogs('climada.entity.exposures.black_marble', level='INFO') as cm:
ent.set_countries(country_name, 2012, res_km=5.0, from_hr=False)
Expand Down
687 changes: 404 additions & 283 deletions doc/tutorial/climada_entity_BlackMarble.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 0 additions & 4 deletions doc/tutorial/results/.gitignore
View file
Open in desktop

This file was deleted.