Closed
Description
Please checkout the F.A.Q page before creating a bug ticket to make sure it is not already addressed.
Describe the bug
An upstream provider sends us a JSON event with 3 sections: beforeImage, updates, allAfterImage. In the near future they will send us only the updates and allAfterImage to save on payload size. We will then attempt to generate the beforeImage on our own by using the allAfterImage and subtracting a Delta created from the updates. Note: we are taking a cautious approach and have chosen to specify the argument raise_errors=True on our Delta. However, in this one circumstance, we see that when we try to generate a beforeImage, the Delta throws an error. Likewise, if we change raise_errors=False then we see that our generated beforeImage does not match what was expected(provided).
To Reproduce
The below code can be used to demonstrate the problem using a Unit Test.
import deepdiff.serialization
import pytest
import json
import copy
from deepdiff import DeepDiff, Delta, serialization
from conv_deep_diff import write_dictionary_to_json
"""
Sample code (Unit Test) to demonstrate DeepDiff errors when trying
to generate a result dict from a AFTER dict - Delta
execute:
pytest -vv demo_deepdiff_fail.py
"""
def test_delta_failure():
false = False # Helps to translate between true JSON and Python Dictionary
true = True # Helps to translate between true JSON and Python Dictionary
null = None # Helps to translate between true JSON and Python Dictionary
event = {
"beforeImage": {
"postalAddresses": [
{
"country": "UNITED STATES",
"city": "BLUFFS",
"addressType": "US",
"postalCode": "626218032",
"usage": "Mailing",
"specialHandling": null,
"standardization": "YES",
"stateProvince": "IL",
"primaryIndicator": true,
"addressIdentifier": "Z8PDWBG42YC",
"addressLines": [
"871 PHILLIPS FERRY RD"
],
"specialHandlingType": null
},
{
"country": "UNITED STATES",
"city": "BLUFFS",
"addressType": "US",
"postalCode": "626218032",
"usage": "Residence",
"specialHandling": null,
"standardization": "YES",
"stateProvince": "IL",
"primaryIndicator": false,
"addressIdentifier": "Z8PDWBG42YC",
"addressLines": [
"871 PHILLIPS FERRY RD"
],
"specialHandlingType": null
},
{
"country": "UNITED STATES",
"city": "BLUFFS",
"addressType": "US",
"postalCode": "626218032",
"usage": "Mailing",
"specialHandling": null,
"standardization": null,
"stateProvince": "IL",
"primaryIndicator": false,
"addressIdentifier": "MHPP3BY0BYC",
"addressLines": [
"871 PHILLIPS FERRY RD",
"APT RV92"
],
"specialHandlingType": null
}
]
},
"allAfterImage": {
"postalAddresses": [
{
"country": "UNITED STATES",
"city": "BLUFFS",
"addressType": "US",
"postalCode": "626218032",
"usage": "Residence",
"specialHandling": null,
"standardization": "NO",
"stateProvince": "IL",
"primaryIndicator": false,
"addressIdentifier": "Z8PDWBG42YC",
"addressLines": [
"871 PHILLIPS FERRY RD"
],
"specialHandlingType": null
},
{
"country": "UNITED STATES",
"city": "BLUFFS",
"addressType": "US",
"postalCode": "626218032",
"usage": "Mailing",
"specialHandling": null,
"standardization": null,
"stateProvince": "IL",
"primaryIndicator": false,
"addressIdentifier": "MHPP3BY0BYC",
"addressLines": [
"871 PHILLIPS FERRY RD",
"APT RV92"
],
"specialHandlingType": null
},
{
"country": "UNITED STATES",
"city": "BLUFFS",
"addressType": "US",
"postalCode": "626218032",
"usage": "Mailing",
"specialHandling": null,
"standardization": "NO",
"stateProvince": "IL",
"primaryIndicator": true,
"addressIdentifier": "Z8PDWBG42YC",
"addressLines": [
"871 PHILLIPS FERRY RD"
],
"specialHandlingType": null
}
]
},
"updates": [
{
"path": [
"postalAddresses",
1,
"standardization"
],
"action": "values_changed",
"value": "NO",
"old_value": "YES"
},
{
"path": [
"postalAddresses",
0,
"standardization"
],
"action": "values_changed",
"value": "NO",
"old_value": "YES"
}
]
}
# Sanity Check, make sure a calculated_updates matches what was provided on the event.
temp_diff = DeepDiff(event.get('beforeImage'), event.get('allAfterImage'), ignore_order=True, report_repetition=True)
temp_delta = Delta(temp_diff, always_include_values=True, bidirectional=True, raise_errors=True)
calculated_updates = temp_delta.to_flat_dicts()
double_check = DeepDiff(calculated_updates, event.get('updates'))
assert {} == double_check
# If we get to here, then our event details were accurate, including the 'updates'
# Now see if we could generate a beforeImage by subtracting a Delta from our 'allAfterImage'
delta = Delta(flat_dict_list=event.get('updates'),
always_include_values=True, bidirectional=True, raise_errors=True)
generated_before_image = event.get('allAfterImage') - delta
"""
Errors on line above with:
msg = "Expected the old value for root['postalAddresses'][1]['standardization'] to be NO but it is None. Error found on: You have applied the delta to an object that has different values than the original object the delta was made from."
level = 'error'
def _raise_or_log(self, msg, level='error'):
if self.log_errors:
getattr(logger, level)(msg)
if self.raise_errors:
> raise DeltaError(msg)
E deepdiff.delta.DeltaError: Expected the old value for root['postalAddresses'][1]['standardization'] to be NO but it is None. Error found on: You have applied the delta to an object that has different values than the original object the delta was made from.
C:\DEV\Tools\Python\Python38\lib\site-packages\deepdiff\delta.py:190: DeltaError
"""
double_check = DeepDiff(generated_before_image, event.get('beforeImage'), ignore_order=True, report_repetition=True)
assert {} == double_check
Expected behavior
We would expect to be able to be able to generate a new beforeImage which exactly matches the provided event beforeImage.
OS, DeepDiff version and Python version (please complete the following information):
- OS: Win10
- Version [e.g. 20LTS]
- Python Version [e.g. 3.9.12]: Python 3.8.8
- DeepDiff Version [e.g. 5.8.0]: 6.7.0
Additional context
Add any other context about the problem here.