Add large scale https updates with bloom filter

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "submodules/ios-js-support"]
 	path = submodules/ios-js-support
 	url = https://github.com/duckduckgo/ios-js-support.git
+[submodule "submodules/bloom_cpp"]
+	path = submodules/bloom_cpp
+	url = https://github.com/duckduckgo/bloom_cpp

Cartfile

@@ -11,6 +11,8 @@ github "duckduckgo/Alamofire" ~>  4.4.0
 
 github "duckduckgo/OHHTTPStubs" ~> 6.0.0
 
+github "duckduckgo/CommonCrypto" ~> 1.1.0
+
 github "duckduckgo/SimulatorStatusMagic" "tags/1.9.5"
 
 github "duckduckgo/swifter" == 1.4.0

Cartfile.resolved

@@ -1,4 +1,5 @@
 github "duckduckgo/Alamofire" "4.6.0"
+github "duckduckgo/CommonCrypto" "v1.1.0"
 github "duckduckgo/Device.swift" "1.1.2"
 github "duckduckgo/Kingfisher" "3.13.1"
 github "duckduckgo/OHHTTPStubs" "6.1.0"

Core/AppUrls.swift

@@ -28,18 +28,21 @@ public struct AppUrls {
         }
 
         static let base = ProcessInfo.processInfo.environment["BASE_URL", default: "https://duckduckgo.com"]
-        static let favicon = "\(base)/favicon.ico"
         static let autocomplete = "\(base)/ac/"
         static let disconnectMeBlockList = "\(base)/contentblocking.js?l=disconnect"
         static let easylistBlockList = "\(base)/contentblocking.js?l=easylist"
         static let easylistPrivacyBlockList = "\(base)/contentblocking.js?l=easyprivacy"
-        static let httpsUpgradeList = "\(base)/contentblocking.js?l=https2"
         static let trackersWhitelist = "\(base)/contentblocking/trackers-whitelist.txt"
         static let surrogates = "\(base)/contentblocking.js?l=surrogates"
         static let atb = "\(base)/atb.js\(devMode)"
         static let exti = "\(base)/exti/\(devMode)"
         static let feedback = "\(base)/feedback.js?type=app-feedback"
         static let faviconService = "\(base)/ip3/%@.ico"
+
+        static let staticBase = "https://staticcdn.duckduckgo.com"
+        static let httpsBloomFilter = "\(staticBase)/https/https-mobile-bloom.bin"
+        static let httpsBloomFilterSpec = "\(staticBase)/https/https-mobile-bloom-spec.json"
+        static let httpsWhitelist = "\(staticBase)/https/https-mobile-whitelist.json"
 
         static let pixelBase = ProcessInfo.processInfo.environment["PIXEL_BASE_URL", default: "https://improving.duckduckgo.com"]
         static let pixel = "\(pixelBase)/t/%@_ios_%@"
@@ -65,9 +68,9 @@ public struct AppUrls {
     public var base: URL {
         return URL(string: Url.base)!
     }
-
-    public var favicon: URL {
-        return URL(string: Url.favicon)!
+    
+    public func autocompleteUrl(forText text: String) -> URL {
+        return URL(string: Url.autocomplete)!.addParam(name: Param.search, value: text)
     }
 
     public var disconnectMeBlockList: URL {
@@ -81,11 +84,7 @@ public struct AppUrls {
     public var easylistPrivacyBlockList: URL {
         return URL(string: Url.easylistPrivacyBlockList)!
     }
-
-    public var httpsUpgradeList: URL {
-        return URL(string: Url.httpsUpgradeList)!
-    }
-
+
     public var trackersWhitelist: URL {
         return URL(string: Url.trackersWhitelist)!
     }
@@ -97,6 +96,11 @@ public struct AppUrls {
     public var feedback: URL {
         return URL(string: Url.feedback)!
     }
+
+    public func faviconUrl(forDomain domain: String) -> URL {
+        let urlString = String(format: Url.faviconService, domain)
+        return URL(string: urlString)!
+    }
 
     public var atb: URL {
         var url = URL(string: Url.atb)!
@@ -132,11 +136,6 @@ public struct AppUrls {
         return extiUrl.addParam(name: Param.atb, value: atb)
     }
 
-    public func faviconUrl(forDomain domain: String) -> URL {
-        let urlString = String(format: Url.faviconService, domain)
-        return URL(string: urlString)!
-    }
-
     /**
      Generates a search url with the source (t) https://duck.co/help/privacy/t
      and cohort (atb) https://duck.co/help/privacy/atb
@@ -145,6 +144,12 @@ public struct AppUrls {
         let searchUrl = base.addParam(name: Param.search, value: text)
         return applyStatsParams(for: searchUrl)
     }
+
+    public func isDuckDuckGoSearch(url: URL) -> Bool {
+        if !isDuckDuckGo(url: url) { return false }
+        guard url.getParam(name: Param.search) != nil else { return false }
+        return true
+    }
 
     public func applyStatsParams(for url: URL) -> URL {
         var searchUrl = url.addParam(name: Param.source, value: ParamValue.source)
@@ -155,16 +160,6 @@ public struct AppUrls {
         return searchUrl
     }
 
-    public func autocompleteUrl(forText text: String) -> URL {
-        return URL(string: Url.autocomplete)!.addParam(name: Param.search, value: text)
-    }
-
-    public func isDuckDuckGoSearch(url: URL) -> Bool {
-        if !isDuckDuckGo(url: url) { return false }
-        guard url.getParam(name: Param.search) != nil else { return false }
-        return true
-    }
-
     public func hasCorrectMobileStatsParams(url: URL) -> Bool {
         guard let source = url.getParam(name: Param.source), source == ParamValue.source  else { return false }
         if let atbWithVariant = statisticsStore.atbWithVariant {
@@ -173,10 +168,21 @@ public struct AppUrls {
         return true
     }
 
+    public var httpsBloomFilter: URL {
+        return URL(string: Url.httpsBloomFilter)!
+    }
+
+    public var httpsBloomFilterSpec: URL {
+        return URL(string: Url.httpsBloomFilterSpec)!
+    }
+
+    public var httpsWhitelist: URL {
+        return URL(string: Url.httpsWhitelist)!
+    }
+
     public func pixelUrl(forPixelNamed pixelName: String, formFactor: String) -> URL {
         var url = URL(string: Url.pixel.format(arguments: pixelName, formFactor))!
         url = url.addParam(name: Param.atb, value: statisticsStore.atbWithVariant ?? "")
         return url
     }
-
 }

Core/BloomFilterWrapper.h

@@ -0,0 +1,26 @@
+//
+//  BloomFilterWrapper.h
+//  DuckDuckGo
+//
+//  Copyright © 2018 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+#import <Foundation/Foundation.h>
+
+@interface BloomFilterWrapper : NSObject
+- (instancetype)initFromPath:(NSString*)path withTotalItems:(int)count;
+- (instancetype)initWithTotalItems:(int)count errorRate:(double)errorRate;
+- (void)add:(NSString*) entry;
+- (BOOL)contains:(NSString*) entry;
+@end

Core/BloomFilterWrapper.mm

@@ -0,0 +1,60 @@
+//
+//  BloomFilterWrapper.m
+//  DuckDuckGo
+//
+//  Copyright © 2018 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+#import "BloomFilterWrapper.h"
+#import "BloomFilter.hpp"
+
+@interface BloomFilterWrapper() {
+    BloomFilter *filter;
+}
+@end
+
+@implementation BloomFilterWrapper
+
+- (instancetype)initFromPath:(NSString*)path withTotalItems:(int)count {
+    self = [super init];
+    if (self != nil) {
+        NSLog(@"Bloom: Importing data from %@", path);
+        filter = new BloomFilter([path cStringUsingEncoding: NSString.defaultCStringEncoding], count);
+    }
+    return self;
+}
+
+- (instancetype)initWithTotalItems:(int)count errorRate:(double)errorRate {
+    self = [super init];
+    if (self != nil) {
+        filter = new BloomFilter(count, errorRate);
+    }
+    return self;
+}
+
+- (void)add:(NSString*)entry {
+    if (filter != nil) {
+        filter->add([entry UTF8String]);
+    }
+}
+
+- (BOOL)contains:(NSString*)entry {
+    if (filter == nil || entry == nil) {
+        return false;
+    }
+    return filter->contains([entry UTF8String]);
+}
+
+@end