add few fields to CITP pioneer study schema (#635)

* schema and test data for CITP's Pioneer study, fails test of bq_metadata_format

* add defaults, and prettify

* number -> integer

* add deletion and enrollment schemas from pioneer-debug

* add forgotten toplevel schemas

* namespace rename to pioneer-citp-news-disinfo, and ping to measurements

* add schema field for reporting generated survey id

* add field to pioneer-citp measurements

* add schema field for reporting generated survey id

* add field to pioneer-citp measurements

* deprecate and replace old field, add description for survey id field

* consistency updates for pioneer citp

Author: akohlbre

schemas/pioneer-citp-news-disinfo/measurements/measurements.1.schema.json

@@ -33,6 +33,9 @@
               },
               "timeOfDay": {
                 "type": "integer"
+              },
+              "visThreshold": {
+                "type": "integer"
               }
             },
             "type": "object"
@@ -157,6 +160,12 @@
                     "pageClassification": {
                       "type": "integer"
                     },
+                    "prevExposed": {
+                      "type": "integer"
+                    },
+                    "source": {
+                      "type": "string"
+                    },
                     "timeOfDay": {
                       "type": "integer"
                     },

templates/pioneer-citp-news-disinfo/measurements/measurements.1.schema.json

@@ -71,6 +71,8 @@
                       "type": "string",
                       "enum": ["public", "restricted", "unknown"]
                     },
+                    "prevExposed": { "type": "integer" },
+                    "source": { "type": "string" },
                     "numShares": { "type": "integer" },
                     "dayOfWeek": { "type": "integer" },
                     "timeOfDay": { "type": "integer" }
@@ -102,6 +104,7 @@
               "destinationDomain": { "type": "string" },
               "dayOfWeek": { "type": "integer" },
               "timeOfDay": { "type": "integer" },
+              "visThreshold": { "type": "integer" },
               "numExposures": { "type": "integer" },
               "laterSharedCount": { "type": "integer" },
               "laterVisitedCount": { "type": "integer" }

validation/pioneer-citp-news-disinfo/measurements.1.sample.pass.json

@@ -33,7 +33,9 @@
             "domain": "nytimes.com",
             "pageClassification": 0,
             "audience": "public",
+            "source": "page",
             "visitReferrer": "google.com",
+            "prevExposed": 1,
             "numShares": 5,
             "dayOfWeek": 2,
             "timeOfDay": 20
@@ -42,7 +44,9 @@
             "domain": "nytimes.com",
             "pageClassification": 1,
             "audience": "restricted",
+            "source": "",
             "visitReferrer": "google.com",
+            "prevExposed": 1,
             "numShares": 3,
             "dayOfWeek": 2,
             "timeOfDay": 20
@@ -58,6 +62,8 @@
             "pageClassification": 2,
             "visitReferrer": "google.com",
             "audience": "public",
+            "source": "",
+            "prevExposed": 0,
             "numShares": 2,
             "dayOfWeek": 2,
             "timeOfDay": 20
@@ -80,6 +86,7 @@
         "destinationDomain": "nytimes.com",
         "dayOfWeek": 1,
         "timeOfDay": 4,
+        "visThreshold": 3000,
         "numExposures": 10,
         "laterSharedCount": 3,
         "laterVisitedCount": 4
@@ -89,6 +96,7 @@
         "destinationDomain": "nytimes.com",
         "dayOfWeek": 2,
         "timeOfDay": 8,
+        "visThreshold": 1000,
         "numExposures": 3,
         "laterSharedCount": 3,
         "laterVisitedCount": 4