diff --git a/Source/DiscoveryV1/Discovery.swift b/Source/DiscoveryV1/Discovery.swift index 0766292b8..c45647fbe 100644 --- a/Source/DiscoveryV1/Discovery.swift +++ b/Source/DiscoveryV1/Discovery.swift @@ -3679,6 +3679,7 @@ public class Discovery { - `box` indicates the credentials are used to connect an instance of Enterprise Box. - `salesforce` indicates the credentials are used to connect to Salesforce. - `sharepoint` indicates the credentials are used to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the credentials are used to perform a web crawl. - parameter credentialDetails: Object containing details of the stored credentials. Obtain credentials for your source from the administrator of the source. - parameter headers: A dictionary of request headers to be sent with this request. @@ -3794,6 +3795,7 @@ public class Discovery { - `box` indicates the credentials are used to connect an instance of Enterprise Box. - `salesforce` indicates the credentials are used to connect to Salesforce. - `sharepoint` indicates the credentials are used to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the credentials are used to perform a web crawl. - parameter credentialDetails: Object containing details of the stored credentials. Obtain credentials for your source from the administrator of the source. - parameter headers: A dictionary of request headers to be sent with this request. diff --git a/Source/DiscoveryV1/Models/CredentialDetails.swift b/Source/DiscoveryV1/Models/CredentialDetails.swift index 6bd9b8e9d..3664b0858 100644 --- a/Source/DiscoveryV1/Models/CredentialDetails.swift +++ b/Source/DiscoveryV1/Models/CredentialDetails.swift @@ -27,12 +27,25 @@ public struct CredentialDetails: Codable, Equatable { the **source_type**. The following combinations are possible: - `"source_type": "box"` - valid `credential_type`s: `oauth2` - `"source_type": "salesforce"` - valid `credential_type`s: `username_password` - - `"source_type": "sharepoint"` - valid `credential_type`s: `saml`. + - `"source_type": "sharepoint"` - valid `credential_type`s: `saml` with **source_version** of `online`, or + `ntml_v1` with **source_version** of `2016` + - `"source_type": "web_crawl"` - valid `credential_type`s: `noauth` or `basic`. */ public enum CredentialType: String { case oauth2 = "oauth2" case saml = "saml" case usernamePassword = "username_password" + case noauth = "noauth" + case basic = "basic" + case ntmlV1 = "ntml_v1" + } + + /** + The type of Sharepoint repository to connect to. Only valid, and required, with a **source_type** of `sharepoint`. + */ + public enum SourceVersion: String { + case online = "online" + case sp2016 = "2016" } /** @@ -40,7 +53,9 @@ public struct CredentialDetails: Codable, Equatable { the **source_type**. The following combinations are possible: - `"source_type": "box"` - valid `credential_type`s: `oauth2` - `"source_type": "salesforce"` - valid `credential_type`s: `username_password` - - `"source_type": "sharepoint"` - valid `credential_type`s: `saml`. + - `"source_type": "sharepoint"` - valid `credential_type`s: `saml` with **source_version** of `online`, or + `ntml_v1` with **source_version** of `2016` + - `"source_type": "web_crawl"` - valid `credential_type`s: `noauth` or `basic`. */ public var credentialType: String? @@ -58,13 +73,13 @@ public struct CredentialDetails: Codable, Equatable { /** The **url** of the source that these credentials connect to. Only valid, and required, with a **credential_type** - of `username_password`. + of `username_password`, `noauth`, and `basic`. */ public var url: String? /** The **username** of the source that these credentials connect to. Only valid, and required, with a - **credential_type** of `saml` and `username_password`. + **credential_type** of `saml`, `username_password`, `basic`, or `ntml_v1`. */ public var username: String? @@ -110,13 +125,36 @@ public struct CredentialDetails: Codable, Equatable { /** The **password** of the source that these credentials connect to. Only valid, and required, with - **credential_type**s of `saml` and `username_password`. + **credential_type**s of `saml`, `username_password`, `basic`, or `ntml_v1`. **Note:** When used with a **source_type** of `salesforce`, the password consists of the Salesforce password and a valid Salesforce security token concatenated. This value is never returned and is only used when creating or modifying **credentials**. */ public var password: String? + /** + The ID of the **gateway** to be connected through (when connecting to intranet sites). Only valid with a + **credential_type** of `noauth`, `basic`, or `ntml_v1`. Gateways are created using the + `/v1/environments/{environment_id}/gateways` methods. + */ + public var gatewayID: String? + + /** + The type of Sharepoint repository to connect to. Only valid, and required, with a **source_type** of `sharepoint`. + */ + public var sourceVersion: String? + + /** + SharePoint OnPrem WebApplication URL. Only valid, and required, with a **source_version** of `2016`. + */ + public var webApplicationURL: String? + + /** + The domain used to log in to your OnPrem SharePoint account. Only valid, and required, with a **source_version** of + `2016`. + */ + public var domain: String? + // Map each property name to the key that shall be used for encoding/decoding. private enum CodingKeys: String, CodingKey { case credentialType = "credential_type" @@ -131,6 +169,10 @@ public struct CredentialDetails: Codable, Equatable { case privateKey = "private_key" case passphrase = "passphrase" case password = "password" + case gatewayID = "gateway_id" + case sourceVersion = "source_version" + case webApplicationURL = "web_application_url" + case domain = "domain" } /** @@ -140,15 +182,17 @@ public struct CredentialDetails: Codable, Equatable { specified must be supported by the **source_type**. The following combinations are possible: - `"source_type": "box"` - valid `credential_type`s: `oauth2` - `"source_type": "salesforce"` - valid `credential_type`s: `username_password` - - `"source_type": "sharepoint"` - valid `credential_type`s: `saml`. + - `"source_type": "sharepoint"` - valid `credential_type`s: `saml` with **source_version** of `online`, or + `ntml_v1` with **source_version** of `2016` + - `"source_type": "web_crawl"` - valid `credential_type`s: `noauth` or `basic`. - parameter clientID: The **client_id** of the source that these credentials connect to. Only valid, and required, with a **credential_type** of `oauth2`. - parameter enterpriseID: The **enterprise_id** of the Box site that these credentials connect to. Only valid, and required, with a **source_type** of `box`. - parameter url: The **url** of the source that these credentials connect to. Only valid, and required, with a - **credential_type** of `username_password`. + **credential_type** of `username_password`, `noauth`, and `basic`. - parameter username: The **username** of the source that these credentials connect to. Only valid, and - required, with a **credential_type** of `saml` and `username_password`. + required, with a **credential_type** of `saml`, `username_password`, `basic`, or `ntml_v1`. - parameter organizationURL: The **organization_url** of the source that these credentials connect to. Only valid, and required, with a **credential_type** of `saml`. - parameter siteCollectionPath: The **site_collection.path** of the source that these credentials connect to. @@ -166,10 +210,19 @@ public struct CredentialDetails: Codable, Equatable { required, with a **credential_type** of `oauth2`. This value is never returned and is only used when creating or modifying **credentials**. - parameter password: The **password** of the source that these credentials connect to. Only valid, and - required, with **credential_type**s of `saml` and `username_password`. + required, with **credential_type**s of `saml`, `username_password`, `basic`, or `ntml_v1`. **Note:** When used with a **source_type** of `salesforce`, the password consists of the Salesforce password and a valid Salesforce security token concatenated. This value is never returned and is only used when creating or modifying **credentials**. + - parameter gatewayID: The ID of the **gateway** to be connected through (when connecting to intranet sites). + Only valid with a **credential_type** of `noauth`, `basic`, or `ntml_v1`. Gateways are created using the + `/v1/environments/{environment_id}/gateways` methods. + - parameter sourceVersion: The type of Sharepoint repository to connect to. Only valid, and required, with a + **source_type** of `sharepoint`. + - parameter webApplicationURL: SharePoint OnPrem WebApplication URL. Only valid, and required, with a + **source_version** of `2016`. + - parameter domain: The domain used to log in to your OnPrem SharePoint account. Only valid, and required, with + a **source_version** of `2016`. - returns: An initialized `CredentialDetails`. */ @@ -185,7 +238,11 @@ public struct CredentialDetails: Codable, Equatable { publicKeyID: String? = nil, privateKey: String? = nil, passphrase: String? = nil, - password: String? = nil + password: String? = nil, + gatewayID: String? = nil, + sourceVersion: String? = nil, + webApplicationURL: String? = nil, + domain: String? = nil ) { self.credentialType = credentialType @@ -200,6 +257,10 @@ public struct CredentialDetails: Codable, Equatable { self.privateKey = privateKey self.passphrase = passphrase self.password = password + self.gatewayID = gatewayID + self.sourceVersion = sourceVersion + self.webApplicationURL = webApplicationURL + self.domain = domain } } diff --git a/Source/DiscoveryV1/Models/Credentials.swift b/Source/DiscoveryV1/Models/Credentials.swift index 423224c5e..5cb6312fe 100644 --- a/Source/DiscoveryV1/Models/Credentials.swift +++ b/Source/DiscoveryV1/Models/Credentials.swift @@ -26,11 +26,13 @@ public struct Credentials: Codable, Equatable { - `box` indicates the credentials are used to connect an instance of Enterprise Box. - `salesforce` indicates the credentials are used to connect to Salesforce. - `sharepoint` indicates the credentials are used to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the credentials are used to perform a web crawl. */ public enum SourceType: String { case box = "box" case salesforce = "salesforce" case sharepoint = "sharepoint" + case webCrawl = "web_crawl" } /** @@ -43,6 +45,7 @@ public struct Credentials: Codable, Equatable { - `box` indicates the credentials are used to connect an instance of Enterprise Box. - `salesforce` indicates the credentials are used to connect to Salesforce. - `sharepoint` indicates the credentials are used to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the credentials are used to perform a web crawl. */ public var sourceType: String? @@ -67,6 +70,7 @@ public struct Credentials: Codable, Equatable { - `box` indicates the credentials are used to connect an instance of Enterprise Box. - `salesforce` indicates the credentials are used to connect to Salesforce. - `sharepoint` indicates the credentials are used to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the credentials are used to perform a web crawl. - parameter credentialDetails: Object containing details of the stored credentials. Obtain credentials for your source from the administrator of the source. diff --git a/Source/DiscoveryV1/Models/Source.swift b/Source/DiscoveryV1/Models/Source.swift index 0ef87bbe3..71e040948 100644 --- a/Source/DiscoveryV1/Models/Source.swift +++ b/Source/DiscoveryV1/Models/Source.swift @@ -26,11 +26,13 @@ public struct Source: Codable, Equatable { - `box` indicates the configuration is to connect an instance of Enterprise Box. - `salesforce` indicates the configuration is to connect to Salesforce. - `sharepoint` indicates the configuration is to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the configuration is to perform a web page crawl. */ public enum TypeEnum: String { case box = "box" case salesforce = "salesforce" case sharepoint = "sharepoint" + case webCrawl = "web_crawl" } /** @@ -38,6 +40,7 @@ public struct Source: Codable, Equatable { - `box` indicates the configuration is to connect an instance of Enterprise Box. - `salesforce` indicates the configuration is to connect to Salesforce. - `sharepoint` indicates the configuration is to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the configuration is to perform a web page crawl. */ public var type: String? @@ -73,6 +76,7 @@ public struct Source: Codable, Equatable { - `box` indicates the configuration is to connect an instance of Enterprise Box. - `salesforce` indicates the configuration is to connect to Salesforce. - `sharepoint` indicates the configuration is to connect to Microsoft SharePoint Online. + - `web_crawl` indicates the configuration is to perform a web page crawl. - parameter credentialID: The **credential_id** of the credentials to use to connect to the source. Credentials are defined using the **credentials** method. The **source_type** of the credentials used must match the **type** field specified in this object. diff --git a/Source/DiscoveryV1/Models/SourceOptions.swift b/Source/DiscoveryV1/Models/SourceOptions.swift index 43e8129a1..74ad6e3cd 100644 --- a/Source/DiscoveryV1/Models/SourceOptions.swift +++ b/Source/DiscoveryV1/Models/SourceOptions.swift @@ -39,11 +39,18 @@ public struct SourceOptions: Codable, Equatable { */ public var siteCollections: [SourceOptionsSiteColl]? + /** + Array of Web page URLs to begin crawling the web from. Only valid and required when the **type** field of the + **source** object is set to `web_crawl`. + */ + public var urls: [SourceOptionsWebCrawl]? + // Map each property name to the key that shall be used for encoding/decoding. private enum CodingKeys: String, CodingKey { case folders = "folders" case objects = "objects" case siteCollections = "site_collections" + case urls = "urls" } /** @@ -56,18 +63,22 @@ public struct SourceOptions: Codable, Equatable { - parameter siteCollections: Array of Microsoft SharePointoint Online site collections to crawl from the SharePoint source. Only valid and required when the **type** field of the **source** object is set to `sharepoint`. + - parameter urls: Array of Web page URLs to begin crawling the web from. Only valid and required when the + **type** field of the **source** object is set to `web_crawl`. - returns: An initialized `SourceOptions`. */ public init( folders: [SourceOptionsFolder]? = nil, objects: [SourceOptionsObject]? = nil, - siteCollections: [SourceOptionsSiteColl]? = nil + siteCollections: [SourceOptionsSiteColl]? = nil, + urls: [SourceOptionsWebCrawl]? = nil ) { self.folders = folders self.objects = objects self.siteCollections = siteCollections + self.urls = urls } } diff --git a/Source/DiscoveryV1/Models/SourceOptionsWebCrawl.swift b/Source/DiscoveryV1/Models/SourceOptionsWebCrawl.swift new file mode 100644 index 000000000..874b9a612 --- /dev/null +++ b/Source/DiscoveryV1/Models/SourceOptionsWebCrawl.swift @@ -0,0 +1,129 @@ +/** + * Copyright IBM Corporation 2018 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +import Foundation + +/** + Object defining which URL to crawl and how to crawl it. + */ +public struct SourceOptionsWebCrawl: Codable, Equatable { + + /** + The number of concurrent URLs to fetch. `gentle` means one URL is fetched at a time with a delay between each call. + `normal` means as many as two URLs are fectched concurrently with a short delay between fetch calls. `aggressive` + means that up to ten URLs are fetched concurrently with a short delay between fetch calls. + */ + public enum CrawlSpeed: String { + case gentle = "gentle" + case normal = "normal" + case aggressive = "aggressive" + } + + /** + The starting URL to crawl. + */ + public var url: String + + /** + When `true`, crawls of the specified URL are limited to the host part of the **url** field. + */ + public var limitToStartingHosts: Bool? + + /** + The number of concurrent URLs to fetch. `gentle` means one URL is fetched at a time with a delay between each call. + `normal` means as many as two URLs are fectched concurrently with a short delay between fetch calls. `aggressive` + means that up to ten URLs are fetched concurrently with a short delay between fetch calls. + */ + public var crawlSpeed: String? + + /** + When `true`, allows the crawl to interact with HTTPS sites with SSL certificates with untrusted signers. + */ + public var allowUntrustedCertificate: Bool? + + /** + The maximum number of hops to make from the initial URL. When a page is crawled each link on that page will also be + crawled if it is within the **maximum_hops** from the initial URL. The first page crawled is 0 hops, each link + crawled from the first page is 1 hop, each link crawled from those pages is 2 hops, and so on. + */ + public var maximumHops: Int? + + /** + The maximum milliseconds to wait for a response from the web server. + */ + public var requestTimeout: Int? + + /** + When `true`, the crawler will ignore any `robots.txt` encountered by the crawler. This should only ever be done + when crawling a web site the user owns. This must be be set to `true` when a **gateway_id** is specied in the + **credentials**. + */ + public var overrideRobotsTxt: Bool? + + // Map each property name to the key that shall be used for encoding/decoding. + private enum CodingKeys: String, CodingKey { + case url = "url" + case limitToStartingHosts = "limit_to_starting_hosts" + case crawlSpeed = "crawl_speed" + case allowUntrustedCertificate = "allow_untrusted_certificate" + case maximumHops = "maximum_hops" + case requestTimeout = "request_timeout" + case overrideRobotsTxt = "override_robots_txt" + } + + /** + Initialize a `SourceOptionsWebCrawl` with member variables. + + - parameter url: The starting URL to crawl. + - parameter limitToStartingHosts: When `true`, crawls of the specified URL are limited to the host part of the + **url** field. + - parameter crawlSpeed: The number of concurrent URLs to fetch. `gentle` means one URL is fetched at a time with + a delay between each call. `normal` means as many as two URLs are fectched concurrently with a short delay + between fetch calls. `aggressive` means that up to ten URLs are fetched concurrently with a short delay between + fetch calls. + - parameter allowUntrustedCertificate: When `true`, allows the crawl to interact with HTTPS sites with SSL + certificates with untrusted signers. + - parameter maximumHops: The maximum number of hops to make from the initial URL. When a page is crawled each + link on that page will also be crawled if it is within the **maximum_hops** from the initial URL. The first page + crawled is 0 hops, each link crawled from the first page is 1 hop, each link crawled from those pages is 2 hops, + and so on. + - parameter requestTimeout: The maximum milliseconds to wait for a response from the web server. + - parameter overrideRobotsTxt: When `true`, the crawler will ignore any `robots.txt` encountered by the crawler. + This should only ever be done when crawling a web site the user owns. This must be be set to `true` when a + **gateway_id** is specied in the **credentials**. + + - returns: An initialized `SourceOptionsWebCrawl`. + */ + public init( + url: String, + limitToStartingHosts: Bool? = nil, + crawlSpeed: String? = nil, + allowUntrustedCertificate: Bool? = nil, + maximumHops: Int? = nil, + requestTimeout: Int? = nil, + overrideRobotsTxt: Bool? = nil + ) + { + self.url = url + self.limitToStartingHosts = limitToStartingHosts + self.crawlSpeed = crawlSpeed + self.allowUntrustedCertificate = allowUntrustedCertificate + self.maximumHops = maximumHops + self.requestTimeout = requestTimeout + self.overrideRobotsTxt = overrideRobotsTxt + } + +} diff --git a/WatsonDeveloperCloud.xcodeproj/project.pbxproj b/WatsonDeveloperCloud.xcodeproj/project.pbxproj index 9609eec96..bae330cdf 100644 --- a/WatsonDeveloperCloud.xcodeproj/project.pbxproj +++ b/WatsonDeveloperCloud.xcodeproj/project.pbxproj @@ -487,6 +487,7 @@ 92261EA421EFFFE800A1A620 /* GatewayDelete.swift in Sources */ = {isa = PBXBuildFile; fileRef = 92261EA021EFFFE800A1A620 /* GatewayDelete.swift */; }; 92261EA521EFFFE800A1A620 /* GatewayName.swift in Sources */ = {isa = PBXBuildFile; fileRef = 92261EA121EFFFE800A1A620 /* GatewayName.swift */; }; 92261EA621EFFFE800A1A620 /* GatewayList.swift in Sources */ = {isa = PBXBuildFile; fileRef = 92261EA221EFFFE800A1A620 /* GatewayList.swift */; }; + 92261EA821F1007800A1A620 /* SourceOptionsWebCrawl.swift in Sources */ = {isa = PBXBuildFile; fileRef = 92261EA721F1007500A1A620 /* SourceOptionsWebCrawl.swift */; }; 924008B92152FD3E005EC1F1 /* AssistantV2Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 924008B82152FD3D005EC1F1 /* AssistantV2Tests.swift */; }; 924EE7BA21505D7A0048C0E5 /* Shared.swift in Sources */ = {isa = PBXBuildFile; fileRef = 92BF69AD213F129A00FE6325 /* Shared.swift */; }; 924EE7CB21505D7A0048C0E5 /* RestKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 92048D8F212F7C16004FD822 /* RestKit.framework */; }; @@ -1235,6 +1236,7 @@ 92261EA021EFFFE800A1A620 /* GatewayDelete.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GatewayDelete.swift; sourceTree = ""; }; 92261EA121EFFFE800A1A620 /* GatewayName.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GatewayName.swift; sourceTree = ""; }; 92261EA221EFFFE800A1A620 /* GatewayList.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GatewayList.swift; sourceTree = ""; }; + 92261EA721F1007500A1A620 /* SourceOptionsWebCrawl.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SourceOptionsWebCrawl.swift; sourceTree = ""; }; 924008B82152FD3D005EC1F1 /* AssistantV2Tests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = AssistantV2Tests.swift; path = AssistantV2Tests/AssistantV2Tests.swift; sourceTree = ""; }; 924EE77D21505C2C0048C0E5 /* AssistantV2.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AssistantV2.h; path = Source/SupportingFiles/AssistantV2.h; sourceTree = ""; }; 924EE7D321505D7A0048C0E5 /* AssistantV2.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = AssistantV2.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -1849,6 +1851,7 @@ 68AF8EAA206968CC00D552E3 /* Models */ = { isa = PBXGroup; children = ( + 92261EA721F1007500A1A620 /* SourceOptionsWebCrawl.swift */, 68AF8EE8206968CC00D552E3 /* AggregationResult.swift */, 68AF8ED5206968CC00D552E3 /* Calculation.swift */, 68AF8EBA206968CC00D552E3 /* Collection.swift */, @@ -3948,6 +3951,7 @@ 68AF8F962069690300D552E3 /* DiskUsage.swift in Sources */, 682CBD2C209051BA0049EE9F /* SegmentSettings.swift in Sources */, 68AF8F752069690300D552E3 /* QueryRelationsRelationship.swift in Sources */, + 92261EA821F1007800A1A620 /* SourceOptionsWebCrawl.swift in Sources */, 68AF8F932069690300D552E3 /* QueryEntitiesEntity.swift in Sources */, 68AF8F772069690300D552E3 /* CollectionDiskUsage.swift in Sources */, 68AF8F642069690300D552E3 /* QueryEntitiesResponse.swift in Sources */,