Skip to content

Remove remaining wip pieces in tests #917

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 1, 2022
Merged

Remove remaining wip pieces in tests #917

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3d3b8a7
Remove remaining wip pieces in tests
dshemetov May 26, 2022
c71b3f4
Put long name test back
dshemetov May 26, 2022
789a404
Fix mocked tests in test_csv_to_database.py
dshemetov Aug 30, 2022
09013fa
Merge branch 'v4-schema-revisions-release-prep' into ds/remove-wip-tests
melange396 Sep 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 2 additions & 40 deletions integrations/acquisition/covidcast/test_csv_uploading.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,6 @@ def test_uploading(self):
log_file=log_file_directory +
"output.log",
data_dir=data_dir,
is_wip_override=False,
not_wip_override=False,
specific_issue_date=False)
uploader_column_rename = {"geo_id": "geo_value", "val": "value", "se": "stderr", "missing_val": "missing_value", "missing_se": "missing_stderr"}

Expand Down Expand Up @@ -232,42 +230,6 @@ def test_uploading(self):
self.setUp()


with self.subTest("Valid wip"):
values = pd.DataFrame({
"geo_id": ["me", "nd", "wa"],
"val": [10.0, 20.0, 30.0],
"se": [0.01, 0.02, 0.03],
"sample_size": [100.0, 200.0, 300.0],
"missing_val": [Nans.NOT_MISSING] * 3,
"missing_se": [Nans.NOT_MISSING] * 3,
"missing_sample_size": [Nans.NOT_MISSING] * 3
})
signal_name = "wip_prototype"
values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)

# upload CSVs
main(args)
dbjobs_main()
response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')

expected_values = pd.concat([values, pd.DataFrame({
"time_value": [20200419] * 3,
"signal": [signal_name] * 3,
"direction": [None] * 3
})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}

self.assertEqual(response, expected_response)
self.verify_timestamps_and_defaults()

# Verify that files were archived
path = data_dir + f'/archive/successful/src-name/20200419_state_wip_prototype.csv.gz'
self.assertIsNotNone(os.stat(path))

self.tearDown()
self.setUp()


with self.subTest("Valid signal with name length 32<x<64"):
values = pd.DataFrame({
"geo_id": ["pa"],
Expand All @@ -278,7 +240,7 @@ def test_uploading(self):
"missing_se": [Nans.NOT_MISSING],
"missing_sample_size": [Nans.NOT_MISSING]
})
signal_name = "wip_really_long_name_that_will_be_accepted"
signal_name = "really_long_name_that_will_be_accepted"
values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)

# upload CSVs
Expand Down Expand Up @@ -310,7 +272,7 @@ def test_uploading(self):
"missing_se": [Nans.NOT_MISSING],
"missing_sample_size": [Nans.NOT_MISSING]
})
signal_name = "wip_really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
signal_name = "really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)

# upload CSVs
Expand Down
4 changes: 2 additions & 2 deletions integrations/server/test_covidcast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,11 @@ def test_round_trip(self):
# `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`,
# `time_value`, `geo_value`, `value_updated_timestamp`,
# `value`, `stderr`, `sample_size`, `direction_updated_timestamp`,
# `direction`, `issue`, `lag`, `is_latest_issue`, `is_wip`,`missing_value`,
# `direction`, `issue`, `lag`, `is_latest_issue`, `missing_value`,
# `missing_stderr`,`missing_sample_size`)
# VALUES
# (0, 'src', 'sig', 'day', 'county', 20200414, '01234',
# 123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1, False,
# 123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1,
# {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
# ''')
# self.cnx.commit()
Expand Down
18 changes: 6 additions & 12 deletions tests/acquisition/covidcast/test_csv_to_database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,7 @@ def _path_details(self):
# a file with a data error
('path/b.csv', ('src_b', 'sig_b', 'week', 'msa', 202016, 202017, 1)),
# emulate a file that's named incorrectly
('path/c.csv', None),
# another good file w/ wip
('path/d.csv', ('src_d', 'wip_sig_d', 'week', 'msa', 202016, 202017, 1)),
('path/c.csv', None)
]

def test_collect_files(self):
Expand Down Expand Up @@ -65,15 +63,13 @@ def load_csv_impl(path, *args):
yield make_row('b1')
yield None
yield make_row('b3')
elif path == 'path/d.csv':
yield make_row('d1')
else:
# fail the test for any other path
raise Exception('unexpected path')

data_dir = 'data_dir'
mock_database = MagicMock()
mock_database.insert_or_update_bulk.return_value = 2
mock_database.insert_or_update_bulk.return_value = 1
mock_csv_importer = MagicMock()
mock_csv_importer.load_csv = load_csv_impl
mock_file_archiver = MagicMock()
Expand All @@ -87,30 +83,28 @@ def load_csv_impl(path, *args):
mock_logger,
csv_importer_impl=mock_csv_importer)

self.assertEqual(modified_row_count, 4)
self.assertEqual(modified_row_count, 1)
Copy link
Collaborator

@melange396 melange396 Aug 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i know this isnt exactly recent in your memory anymore, but why did this change to 1 instead of to 3??

Copy link
Contributor Author

@dshemetov dshemetov Aug 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good question, let me look around. since tests pass, there must've been some change to this test in v4.

Copy link
Contributor Author

@dshemetov dshemetov Aug 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for pointing this out! i made a fix, so it's 3 now. here's an explanation:

  • we have a database mock object, which overwrites insert_or_update_bulk
  • we didn't even count how many rows this function was passed, we just updated the mock return_value argument
  • so i rewrote this so insert_or_update_bulk = MagicMock(wraps=iter_len) where iter_len is just a function that counts the number of items in the iterable passed to insert_or_update_bulk. so now we correctly get modified_row_count == 3

# verify that appropriate rows were added to the database
self.assertEqual(mock_database.insert_or_update_bulk.call_count, 2)
self.assertEqual(mock_database.insert_or_update_bulk.call_count, 1)
call_args_list = mock_database.insert_or_update_bulk.call_args_list
actual_args = [[(a.source, a.signal, a.time_type, a.geo_type, a.time_value,
a.geo_value, a.value, a.stderr, a.sample_size, a.issue, a.lag)
for a in call.args[0]] for call in call_args_list]
expected_args = [
[('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a1', 'a1', 'a1', 'a1', 20200420, 1),
('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a2', 'a2', 'a2', 'a2', 20200420, 1),
('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)],
[('src_d', 'wip_sig_d', 'week', 'msa', 202016, 'd1', 'd1', 'd1', 'd1', 202017, 1)]
('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)]
]
self.assertEqual(actual_args, expected_args)

# verify that two files were successful (a, d) and two failed (b, c)
self.assertEqual(mock_file_archiver.archive_file.call_count, 4)
self.assertEqual(mock_file_archiver.archive_file.call_count, 3)
call_args_list = mock_file_archiver.archive_file.call_args_list
actual_args = [args for (args, kwargs) in call_args_list]
expected_args = [
('path', 'data_dir/archive/successful/src_a', 'a.csv', True),
('path', 'data_dir/archive/failed/src_b', 'b.csv', False),
('path', 'data_dir/archive/failed/unknown', 'c.csv', False),
('path', 'data_dir/archive/successful/src_d', 'd.csv', True),
]
self.assertEqual(actual_args, expected_args)

Expand Down