Skip to content

Commit

Permalink
separate metatile rawrtile buckets config override (#399)
Browse files Browse the repository at this point in the history
  • Loading branch information
@peitili
peitili authored Sep 30, 2021
1 parent 97c2a0f commit ff71a48
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 16 deletions.
12 changes: 12 additions & 0 deletions tilequeue/command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2529,6 +2529,9 @@ def command_fn(cfg, args):
subparser.add_argument('--store_name', required=False,
help='optional string of a list of tile store '
'names e.g. `["my-meta-tiles-us-east-1"]`')
subparser.add_argument('--rawr_store_name', required=False,
help='optional string of rawr tile store '
'names e.g. `"my-rawr-tiles-us-east-1"`')
subparser.add_argument('--store_date_prefix', required=False,
help='optional string of store bucket date prefix '
'e.g. `20210426`')
Expand Down Expand Up @@ -2564,6 +2567,9 @@ def command_fn(cfg, args):
help='optional string of db password e.g. `VHcDuAS0SYx2tlgTvtbuCXwlvO4pAtiGCuScJFjq7wersdfqwer`') # noqa
subparser.add_argument('--store_name', required=False,
help='optional string of a list of tile store names e.g. `["my-meta-tiles-us-east-1"]`') # noqa
subparser.add_argument('--rawr_store_name', required=False,
help='optional string of rawr tile store '
'names e.g. `"my-rawr-tiles-us-east-1"`')
subparser.add_argument('--store_date_prefix', required=False,
help='optional string of store bucket date prefix e.g. `20210426`') # noqa
subparser.add_argument('--batch_check_metafile_exists', required=False,
Expand Down Expand Up @@ -2601,8 +2607,13 @@ def command_fn(cfg, args):
subparser.add_argument('--store_name', required=False,
help='optional string of a list of tile store '
'names e.g. `["my-meta-tiles-us-east-1"]`')
subparser.add_argument('--rawr_store_name', required=False,
help='optional string of rawr tile store '
'names e.g. `"my-rawr-tiles-us-east-1"`')
subparser.add_argument('--store_date_prefix', required=False,
help='optional string of store bucket date prefix '
'which will override the prefix config'
' for meta tile and rawr tile s3 output'
'e.g. `20210426`')
subparser.add_argument('--batch_check_metafile_exists', required=False,
help='optional string of a boolean indicating '
Expand Down Expand Up @@ -2648,6 +2659,7 @@ def command_fn(cfg, args):
postgresql_dbnames=args.postgresql_dbnames, # noqa
postgresql_user=args.postgresql_user,
postgresql_password=args.postgresql_password, # noqa
rawr_store_name=args.rawr_store_name,
store_name=args.store_name,
store_date_prefix=args.store_date_prefix, # noqa
batch_check_metafile_exists=args.batch_check_metafile_exists) # noqa
Expand Down
36 changes: 20 additions & 16 deletions tilequeue/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,7 @@ def make_config_from_argparse(config_file_handle, default_yml=None,
postgresql_dbnames=None,
postgresql_user=None,
postgresql_password=None,
rawr_store_name=None,
store_name=None,
store_date_prefix=None,
batch_check_metafile_exists=None,
Expand Down Expand Up @@ -481,62 +482,65 @@ def make_config_from_argparse(config_file_handle, default_yml=None,

# override config values with explicit arguments if set
if postgresql_hosts is not None:
# for high/low zoom metatile
# overrides for tilequeue commands: meta-tile-low-zoom and meta-tile
keys = ['postgresql', 'host'] # attention non-plural form `host`
value = load(postgresql_hosts)
_override_cfg(cfg, keys, value)
# for RAWR tile
# overrides for tilequeue command: rawr-tile
keys = ['rawr', 'postgresql', 'host']
_override_cfg(cfg, keys, value)

if postgresql_dbnames is not None:
# for high/low zoom metatile
# overrides for tilequeue commands: meta-tile-low-zoom and meta-tile
keys = ['postgresql', 'dbnames']
value = load(postgresql_dbnames)
_override_cfg(cfg, keys, value)
# for RAWR
# overrides for tilequeue command: rawr-tile
keys = ['rawr', 'postgresql', 'dbname']
_override_cfg(cfg, keys, value[0])

if postgresql_user is not None:
# for high/low zoom metatile
# overrides for tilequeue commands: meta-tile-low-zoom and meta-tile
keys = ['postgresql', 'user']
value = load(postgresql_user)
_override_cfg(cfg, keys, value)
# for RAWR
# overrides for tilequeue command: rawr-tile
keys = ['rawr', 'postgresql', 'user']
_override_cfg(cfg, keys, value)

if postgresql_password is not None:
# for high/low zoom metatile
# overrides for tilequeue commands: meta-tile-low-zoom and meta-tile
keys = ['postgresql', 'password']
value = load(postgresql_password)
_override_cfg(cfg, keys, value)
# for RAWR
# overrides for tilequeue command: rawr-tile
keys = ['rawr', 'postgresql', 'password']
_override_cfg(cfg, keys, value)

if store_name is not None:
# for low zoom metatile
# overrides for tilequeue commands: meta-tile-low-zoom and meta-tile
keys = ['store', 'name']
value = load(store_name)
_override_cfg(cfg, keys, value)
# for RAWR

if rawr_store_name is not None:
value = load(rawr_store_name)
# overrides for tilequeue command: rawr-tile
keys = ['rawr', 'sink', 's3', 'bucket']
_override_cfg(cfg, keys, value[0])
# for high zoom metatile
_override_cfg(cfg, keys, value)
# overrides for tilequeue command: meta-tile
keys = ['rawr', 'source', 's3', 'bucket']
_override_cfg(cfg, keys, value[0])
_override_cfg(cfg, keys, value)

if store_date_prefix is not None:
# for low zoom metatile
# overrides for tilequeue command: meta-tile-low-zoom
keys = ['store', 'date-prefix']
value = load(store_date_prefix)
_override_cfg(cfg, keys, value)
# for RAWR
# overrides for tilequeue command: rawr-tile
keys = ['rawr', 'sink', 's3', 'prefix']
_override_cfg(cfg, keys, value)
# for high zoom metatile
# overrides for tilequeue command: meta-tile
keys = ['rawr', 'source', 's3', 'prefix']
_override_cfg(cfg, keys, value)

Expand Down

0 comments on commit ff71a48

Please sign in to comment.