Infrastructure to expand 3rd-Party Tracker Loading Protection (duckduckgo#2143)

Task/Issue URL: https://app.asana.com/0/1179596784783755/1202464473778543/f

Description

This PR adds the infrastructure we need to expand our 3rd-Party Tracker Loading Protection to include Microsoft, such that Microsoft scripts on our list are blocked from loading except for requests from one domain (bat.bing.com) loading on some advertiser websites following DuckDuckGo ad clicks. For more information, please see https://help.duckduckgo.com/duckduckgo-help-pages/privacy/web-tracking-protections/#3rd-party-tracker-loading-protection

Author: anikiki

ad-click/ad-click-api/.gitignore

@@ -0,0 +1 @@
+/build

ad-click/ad-click-api/build.gradle

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2022 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    id 'java-library'
+    id 'kotlin'
+}
+
+apply from: "$rootProject.projectDir/spotless.gradle"
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_1_8
+    targetCompatibility = JavaVersion.VERSION_1_8
+}
+
+dependencies {
+    implementation Kotlin.stdlib.jdk7
+    implementation Google.dagger
+    implementation KotlinX.coroutines.core
+}

ad-click/ad-click-api/src/main/java/com/duckduckgo/adclick/api/AdClickFeatureName.kt

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2022 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.adclick.api
+
+enum class AdClickFeatureName(val value: String) {
+    AdClickAttributionFeatureName("adClickAttribution"),
+}

ad-click/ad-click-api/src/main/java/com/duckduckgo/adclick/api/AdClickManager.kt

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2022 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.adclick.api
+
+/** Public interface for the Ad Click feature which helps to measure ad conversions only when they are required */
+interface AdClickManager {
+
+    /**
+     * Sets the active tab. It takes as parameters:
+     * mandatory [tabId] - The id of the current visible tab.
+     * optional [url] - The url loaded in the current tab, null if no url was loaded (empty tab).
+     * optional [sourceTabId] - The id of the tab from which the current tab was opened, null if a new tab option was used.
+     * optional [sourceTabUrl] - The url loaded in the tab from which the current tab was opened, null if a new tab option was used.
+     */
+    fun setActiveTabId(tabId: String, url: String? = null, sourceTabId: String? = null, sourceTabUrl: String? = null)
+
+    /**
+     * Detects and registers the eTLD+1 if an ad link was clicked. It takes as parameters:
+     * optional [url] - The requested url, null if no url was requested.
+     * mandatory [isMainFrame] - True if the request is for mainframe, false otherwise.
+     */
+    fun detectAdClick(url: String?, isMainFrame: Boolean)
+
+    /**
+     * Detects and saves in memory the ad eTLD+1 domain from the url. It takes as parameters:
+     * mandatory [url] - The requested url.
+     */
+    fun detectAdDomain(url: String)
+
+    /**
+     * Removes any data kept in memory for the specified tab. It takes as parameters:
+     * mandatory [tabId] - The id of the active tab.
+     */
+    fun clearTabId(tabId: String)
+
+    /**
+     * Removes any data related to ad management that is kept in memory.
+     */
+    fun clearAll()
+
+    /**
+     * Removes any data related to ad management that is kept in memory. This is used asynchronously.
+     */
+    fun clearAllExpiredAsync()
+
+    /**
+     * Detects if there is an existing exemption based on the document url and the url requested. It takes as parameters:
+     * mandatory [documentUrl] - The initially requested url, potentially leading to the advertiser page.
+     * mandatory [url] - The requested url, potentially a tracker used for ad attribution.
+     * @return `true` if there is an existing exemption for this combination of [documentUrl] and [url], false otherwise.
+     */
+    fun isExemption(documentUrl: String, url: String): Boolean
+}

ad-click/ad-click-impl/.gitignore

@@ -0,0 +1 @@
+/build

ad-click/ad-click-impl/build.gradle

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2022 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    id 'com.android.library'
+    id 'kotlin-android'
+    id 'com.squareup.anvil'
+    id 'kotlin-android-extensions'
+}
+
+apply from: "$rootProject.projectDir/gradle/android-library.gradle"
+
+android {
+    anvil {
+        generateDaggerFactories = true // default is false
+    }
+    namespace 'com.duckduckgo.adclick.impl'
+}
+
+dependencies {
+    anvil project(path: ':anvil-compiler')
+    implementation project(path: ':anvil-annotations')
+    implementation project(path: ':ad-click-api')
+    implementation project(path: ':di')
+    implementation project(path: ':common')
+    implementation project(path: ':statistics')
+    implementation project(path: ':privacy-config-api')
+    implementation project(path: ':feature-toggles-api')
+    implementation project(path: ':app-build-config-api')
+    implementation project(path: ':ad-click-store')
+
+    implementation Kotlin.stdlib.jdk7
+
+    implementation AndroidX.core.ktx
+
+    implementation AndroidX.appCompat
+    implementation Google.android.material
+    implementation AndroidX.constraintLayout
+
+    implementation AndroidX.lifecycle.viewModelKtx
+    implementation AndroidX.lifecycle.runtimeKtx
+    implementation AndroidX.lifecycle.commonJava8
+    implementation AndroidX.lifecycle.liveDataKtx
+
+    // WorkManager
+    implementation AndroidX.work.runtimeKtx
+    testImplementation AndroidX.work.testing
+    implementation AndroidX.work.rxJava2
+
+    // Room
+    implementation AndroidX.room.runtime
+    implementation AndroidX.room.rxJava2
+    implementation AndroidX.room.ktx
+    testImplementation AndroidX.room.testing
+
+    // Dagger
+    implementation Google.dagger
+
+    implementation KotlinX.coroutines.core
+
+    testImplementation(KotlinX.coroutines.test) {
+        // https://github.com/Kotlin/kotlinx.coroutines/issues/2023
+        // conflicts with mockito due to direct inclusion of byte buddy
+        exclude group: "org.jetbrains.kotlinx", module: "kotlinx-coroutines-debug"
+    }
+
+    implementation "com.jakewharton.threetenabp:threetenabp:_"
+    testImplementation "org.threeten:threetenbp:_"
+
+    implementation JakeWharton.timber
+
+    implementation Square.retrofit2.converter.moshi
+    implementation Square.okHttp3.okHttp
+
+    // Testing dependencies
+    testImplementation "org.mockito.kotlin:mockito-kotlin:_"
+    testImplementation Testing.junit4
+    testImplementation AndroidX.test.ext.junit
+    testImplementation AndroidX.archCore.testing
+    testImplementation 'app.cash.turbine:turbine:_'
+    testImplementation Testing.robolectric
+
+    testImplementation project(path: ':common-test')
+}
+
+fulladleModuleConfig {
+    maxTestShards.set(1)
+}
+
+tasks.register('androidTestsBuild') {
+    dependsOn 'assembleDebugAndroidTest'
+}

ad-click/ad-click-impl/src/main/java/com/duckduckgo/adclick/impl/AdClickAttributionFeature.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2022 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.adclick.impl
+
+data class AdClickAttributionFeature(
+    val state: String,
+    val settings: AdClickAttributionSettings,
+    val exceptions: List<AdClickAttributionException>
+)
+
+data class AdClickAttributionSettings(
+    val linkFormats: List<AdClickAttributionLinkFormat>,
+    val allowlist: List<AdClickAttributionAllowlist>,
+    val navigationExpiration: Long,
+    val totalExpiration: Long,
+    val heuristicDetection: String?,
+    val domainDetection: String?
+)
+
+data class AdClickAttributionLinkFormat(
+    val url: String,
+    val parameterName: String?,
+    val parameterValue: String?,
+    val adDomainParameterName: String?
+)
+
+data class AdClickAttributionAllowlist(
+    val blocklistEntry: String?,
+    val host: String?
+)
+
+data class AdClickAttributionException(val domain: String, val reason: String)

ad-click/ad-click-impl/src/main/java/com/duckduckgo/adclick/impl/AdClickAttributionPlugin.kt

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2022 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.adclick.impl
+
+import com.duckduckgo.adclick.api.AdClickFeatureName
+import com.duckduckgo.adclick.store.AdClickAttributionAllowlistEntity
+import com.duckduckgo.adclick.store.AdClickAttributionDetectionEntity
+import com.duckduckgo.adclick.store.AdClickAttributionExpirationEntity
+import com.duckduckgo.adclick.store.AdClickAttributionLinkFormatEntity
+import com.duckduckgo.adclick.store.AdClickAttributionRepository
+import com.duckduckgo.adclick.store.AdClickFeatureToggleRepository
+import com.duckduckgo.adclick.store.AdClickFeatureToggles
+import com.duckduckgo.di.scopes.AppScope
+import com.duckduckgo.privacy.config.api.PrivacyFeaturePlugin
+import com.squareup.anvil.annotations.ContributesMultibinding
+import com.squareup.moshi.JsonAdapter
+import com.squareup.moshi.Moshi
+import javax.inject.Inject
+
+@ContributesMultibinding(AppScope::class)
+class AdClickAttributionPlugin @Inject constructor(
+    private val adClickAttributionRepository: AdClickAttributionRepository,
+    private val adClickFeatureTogglesRepository: AdClickFeatureToggleRepository
+) : PrivacyFeaturePlugin {
+
+    override fun store(featureName: String, jsonString: String): Boolean {
+        val adClickFeature = adClickFeatureValueOf(featureName) ?: return false
+        if (adClickFeature.value == this.featureName) {
+            val moshi = Moshi.Builder().build()
+            val jsonAdapter: JsonAdapter<AdClickAttributionFeature> =
+                moshi.adapter(AdClickAttributionFeature::class.java)
+
+            val linkFormats = mutableListOf<AdClickAttributionLinkFormatEntity>()
+            val allowList = mutableListOf<AdClickAttributionAllowlistEntity>()
+            val expirations = mutableListOf<AdClickAttributionExpirationEntity>()
+            val detections = mutableListOf<AdClickAttributionDetectionEntity>()
+
+            val adClickAttributionFeature: AdClickAttributionFeature? = jsonAdapter.fromJson(jsonString)
+
+            adClickAttributionFeature?.settings?.linkFormats?.map {
+                linkFormats.add(
+                    AdClickAttributionLinkFormatEntity(
+                        url = it.url,
+                        parameterName = it.parameterName.orEmpty(),
+                        parameterValue = it.parameterValue.orEmpty(),
+                        adDomainParameterName = it.adDomainParameterName.orEmpty()
+                    )
+                )
+            }
+
+            adClickAttributionFeature?.settings?.allowlist?.map {
+                if (it.blocklistEntry != null && it.host != null) {
+                    allowList.add(AdClickAttributionAllowlistEntity(blocklistEntry = it.blocklistEntry, host = it.host))
+                }
+            }
+
+            adClickAttributionFeature?.settings?.let {
+                expirations.add(
+                    AdClickAttributionExpirationEntity(
+                        navigationExpiration = it.navigationExpiration,
+                        totalExpiration = it.totalExpiration
+                    )
+                )
+                detections.add(
+                    AdClickAttributionDetectionEntity(
+                        heuristicDetection = it.heuristicDetection.orEmpty(),
+                        domainDetection = it.domainDetection.orEmpty()
+                    )
+                )
+            }
+
+            adClickAttributionRepository.updateAll(linkFormats, allowList, expirations, detections)
+            val isEnabled = adClickAttributionFeature?.state == "enabled"
+            adClickFeatureTogglesRepository.insert(AdClickFeatureToggles(adClickFeature, isEnabled, null))
+            return true
+        }
+        return false
+    }
+
+    override val featureName: String = AdClickFeatureName.AdClickAttributionFeatureName.value
+}