File tree 1 file changed +6
-7
lines changed
examples/incremental_processing
1 file changed +6
-7
lines changed Original file line number Diff line number Diff line change @@ -34,13 +34,12 @@ def process_files_with_delta():
34
34
This demonstrates incremental processing - only new files are processed.
35
35
"""
36
36
chain = (
37
- dc
38
- .read_storage ("test/" , update = True , delta = True , delta_on = "file.path" )
39
- .filter (C ("file.path" ).glob ("*.txt" ))
40
- .map (file_number = extract_file_number )
41
- .map (content = lambda file : file .read_text ())
42
- .map (processed_at = lambda : time .strftime ("%Y-%m-%d %H:%M:%S" ))
43
- .save (name = "test_files" )
37
+ dc .read_storage ("test/" , update = True , delta = True , delta_on = "file.path" )
38
+ .filter (C ("file.path" ).glob ("*.txt" ))
39
+ .map (file_number = extract_file_number )
40
+ .map (content = lambda file : file .read_text ())
41
+ .map (processed_at = lambda : time .strftime ("%Y-%m-%d %H:%M:%S" ))
42
+ .save (name = "test_files" )
44
43
)
45
44
46
45
# Show information about the dataset
You can’t perform that action at this time.
0 commit comments