3 parts

configs/dolma-v1_5r2/doc_dedupe/cc_en_tail_part1.yaml

@@ -0,0 +1,17 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/common-crawl/v1-c4-cleaned/documents/cc_en_tail/cc_en_tail-0*.json.gz
+
+dedupe:
+  name: dedupe_docs_v2
+  documents:
+    attribute_name: bff_duplicate_docs
+    key: $.text
+  skip_empty: true
+
+bloom_filter:
+  file: /tmp/cc_en_tail_dedupe_docs.bloom
+  read_only: false
+  estimated_doc_count: 30000000000 
+  desired_false_positive_rate: 1e-06
+
+processes: 188

configs/dolma-v1_5r2/doc_dedupe/cc_en_tail.yaml → configs/dolma-v1_5r2/doc_dedupe/cc_en_tail_part2.yaml

@@ -1,5 +1,5 @@
 documents:
-  - s3://ai2-llm/pretraining-data/sources/common-crawl/v1-c4-cleaned/documents/cc_en_tail/*.gz
+  - s3://ai2-llm/pretraining-data/sources/common-crawl/v1-c4-cleaned/documents/cc_en_tail/cc_en_tail-1*.json.gz
 
 dedupe:
   name: dedupe_docs_v2
@@ -9,9 +9,9 @@ dedupe:
   skip_empty: true
 
 bloom_filter:
-  file: /tmp/cc_en_head_dedupe_docs.bloom
+  file: /tmp/cc_en_tail_dedupe_docs.bloom
   read_only: false
-  estimated_doc_count: 60000000000
+  estimated_doc_count: 30000000000
   desired_false_positive_rate: 1e-06
 
 processes: 188

configs/dolma-v1_5r2/doc_dedupe/cc_en_tail_part3.yaml

@@ -0,0 +1,17 @@
+documents:
+  - s3://ai2-llm/pretraining-data/sources/common-crawl/v1-c4-cleaned/documents/cc_en_tail/cc_en_tail-2*.json.gz
+
+dedupe:
+  name: dedupe_docs_v2
+  documents:
+    attribute_name: bff_duplicate_docs
+    key: $.text
+  skip_empty: true
+
+bloom_filter:
+  file: /tmp/cc_en_tail_dedupe_docs.bloom
+  read_only: false
+  estimated_doc_count: 30000000000
+  desired_false_positive_rate: 1e-06
+
+processes: 188