diff --git a/plugins/arcgis/service/src/ArcGISConfig.ts b/plugins/arcgis/service/src/ArcGISConfig.ts index 5d3bb6977..ceded02c6 100644 --- a/plugins/arcgis/service/src/ArcGISConfig.ts +++ b/plugins/arcgis/service/src/ArcGISConfig.ts @@ -8,26 +8,11 @@ export interface FeatureServiceConfig { */ url: string - /** - * Username and password for ArcGIS authentication - */ - auth?: ArcGISAuthConfig - - /** - * Create layers that don't exist - */ - createLayers?: boolean - - /** - * The administration url to the arc feature service. - */ - adminUrl?: string - - /** - * Administration access token - */ - adminToken?: string - + /** + * Serialized ArcGISIdentityManager + */ + identityManager: string + /** * The feature layers. */ @@ -49,10 +34,6 @@ export interface FeatureLayerConfig { */ geometryType?: string - /** - * Access token - */ - token?: string // TODO - can this be removed? Will Layers have a token too? /** * The event ids or names that sync to this arc feature layer. */ @@ -67,86 +48,8 @@ export interface FeatureLayerConfig { * Delete editable layer fields missing from form fields */ deleteFields?: boolean - -} - -export enum AuthType { - Token = 'token', - UsernamePassword = 'usernamePassword', - OAuth = 'oauth' - } - - -/** - * Contains token-based authentication configuration. - */ -export interface TokenAuthConfig { - type: AuthType.Token - token: string - authTokenExpires?: string -} - -/** - * Contains username and password for ArcGIS server authentication. - */ -export interface UsernamePasswordAuthConfig { - type: AuthType.UsernamePassword - /** - * The username for authentication. - */ - username: string - - /** - * The password for authentication. - */ - password: string } -/** - * Contains OAuth authentication configuration. - */ -export interface OAuthAuthConfig { - - type: AuthType.OAuth - - /** - * The Client Id for OAuth - */ - clientId: string - - /** - * The redirectUri for OAuth - */ - redirectUri?: string - - /** - * The temporary auth token for OAuth - */ - authToken?: string - - /** - * The expiration date for the temporary token - */ - authTokenExpires?: number - - /** - * The Refresh token for OAuth - */ - refreshToken?: string - - /** - * The expiration date for the Refresh token - */ - refreshTokenExpires?: number -} - -/** - * Union type for authentication configurations. - */ -export type ArcGISAuthConfig = - | TokenAuthConfig - | UsernamePasswordAuthConfig - | OAuthAuthConfig /** * Attribute configurations diff --git a/plugins/arcgis/service/src/ArcGISIdentityManagerFactory.ts b/plugins/arcgis/service/src/ArcGISIdentityManagerFactory.ts deleted file mode 100644 index ee5d17cf7..000000000 --- a/plugins/arcgis/service/src/ArcGISIdentityManagerFactory.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" -import { ArcGISAuthConfig, AuthType, FeatureServiceConfig, OAuthAuthConfig, TokenAuthConfig, UsernamePasswordAuthConfig } from './ArcGISConfig' - -interface ArcGISIdentityManagerFactory { - create(portal: string, server: string, config: ArcGISAuthConfig): Promise -} - -const OAuthIdentityManagerFactory: ArcGISIdentityManagerFactory = { - async create(portal: string, server: string, auth: OAuthAuthConfig): Promise { - console.debug('Client ID provided for authentication') - const { clientId, authToken, authTokenExpires, refreshToken, refreshTokenExpires } = auth - - if (authToken && new Date(authTokenExpires || 0) > new Date()) { - return ArcGISIdentityManager.fromToken({ - clientId: clientId, - token: authToken, - tokenExpires: new Date(authTokenExpires || 0), - portal: portal, - server: server - }) - } else if (refreshToken && new Date(refreshTokenExpires || 0) > new Date()) { - // TODO: find a way without using constructor nor httpClient - const url = `${portal}/oauth2/token?client_id=${clientId}&refresh_token=${refreshToken}&grant_type=refresh_token` - try { - const response = await request(url, { - httpMethod: 'GET' - }); - - // TODO Factory should not handle config changes - // Update authToken to new token - // const config = await processor.safeGetConfig(); - // let service = config.featureServices.find(service => service.url === portal)?.auth as OAuthAuthConfig; - // const date = new Date(); - // date.setSeconds(date.getSeconds() + response.expires_in || 0); - // service = { - // ...service, - // authToken: response.access_token, - // authTokenExpires: date.getTime() - // } - // await processor.putConfig(config) - - // return ArcGISIdentityManager.fromToken({ - // clientId: clientId, - // token: response.access_token, - // tokenExpires: date, - // portal: portal - // }); - - throw new Error('TODO Unsupported') - } catch (error) { - throw new Error('Error occurred when using refresh token') - } - - } else { - // TODO the config, we need to let the user know UI side they need to authenticate again - throw new Error('Refresh token missing or expired') - } - } -} - -const TokenIdentityManagerFactory: ArcGISIdentityManagerFactory = { - async create(portal: string, server: string, auth: TokenAuthConfig): Promise { - console.debug('Token provided for authentication') - const identityManager = await ArcGISIdentityManager.fromToken({ - token: auth.token, - portal: portal, - server: server, - // TODO: what do we really want to do here? esri package seems to need this optional parameter. - // Use authTokenExpires if defined, otherwise set to now plus a day - tokenExpires: auth.authTokenExpires ? new Date(auth.authTokenExpires) : new Date(Date.now() + 24 * 60 * 60 * 1000) - }) - return identityManager - } -} - -const UsernamePasswordIdentityManagerFactory: ArcGISIdentityManagerFactory = { - async create(portal: string, server: string, auth: UsernamePasswordAuthConfig): Promise { - console.debug('console and password provided for authentication, username:' + auth?.username) - const identityManager = await ArcGISIdentityManager.signIn({ username: auth?.username, password: auth?.password, portal }) - return identityManager - } -} - -const authConfigMap: { [key: string]: ArcGISIdentityManagerFactory } = { - [AuthType.OAuth]: OAuthIdentityManagerFactory, - [AuthType.Token]: TokenIdentityManagerFactory, - [AuthType.UsernamePassword]: UsernamePasswordIdentityManagerFactory -} - -export function getIdentityManager( - config: FeatureServiceConfig -): Promise { - const auth = config.auth - const authType = config.auth?.type - if (!auth || !authType) { - throw new Error('Auth type is undefined') - } - const factory = authConfigMap[authType] - if (!factory) { - throw new Error(`No factory found for type ${authType}`) - } - return factory.create(getPortalUrl(config.url), getServerUrl(config.url), auth) -} - - -export function getPortalUrl(featureService: FeatureServiceConfig | string): string { - const url = getFeatureServiceUrl(featureService) - return `https://${url.hostname}/arcgis/sharing/rest` -} - -export function getServerUrl(featureService: FeatureServiceConfig | string): string { - const url = getFeatureServiceUrl(featureService) - return `https://${url.hostname}/arcgis` -} - -export function getFeatureServiceUrl(featureService: FeatureServiceConfig | string): URL { - const url = typeof featureService === 'string' ? featureService : featureService.url - return new URL(url) -} \ No newline at end of file diff --git a/plugins/arcgis/service/src/ArcGISPluginConfig.ts b/plugins/arcgis/service/src/ArcGISPluginConfig.ts index 831ce8b9a..78f8a8a26 100644 --- a/plugins/arcgis/service/src/ArcGISPluginConfig.ts +++ b/plugins/arcgis/service/src/ArcGISPluginConfig.ts @@ -145,7 +145,7 @@ export const defaultArcGISPluginConfig = Object.freeze({ textAreaFieldLength: 256, observationIdField: 'description', idSeparator: '-', - // eventIdField: 'event_id', + eventIdField: 'event_id', lastEditedDateField: 'last_edited_date', eventNameField: 'event_name', userIdField: 'user_id', diff --git a/plugins/arcgis/service/src/ArcGISService.ts b/plugins/arcgis/service/src/ArcGISService.ts new file mode 100644 index 000000000..c85989174 --- /dev/null +++ b/plugins/arcgis/service/src/ArcGISService.ts @@ -0,0 +1,57 @@ +import { ArcGISIdentityManager } from '@esri/arcgis-rest-request' +import { FeatureServiceConfig } from './ArcGISConfig' +import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' + +export interface ArcGISIdentityService { + getIdentityManager(featureService: FeatureServiceConfig): Promise + updateIndentityManagers(): Promise +} + +export function createArcGISIdentityService( + stateRepo: PluginStateRepository +): ArcGISIdentityService { + const identityManagerCache: Map> = new Map() + + return { + async getIdentityManager(featureService: FeatureServiceConfig): Promise { + let cached = await identityManagerCache.get(featureService.url) + if (!cached) { + const identityManager = ArcGISIdentityManager.deserialize(featureService.identityManager) + const promise = identityManager.getUser().then(() => identityManager) + identityManagerCache.set(featureService.url, promise) + return promise + } else { + return cached + } + }, + async updateIndentityManagers() { + const config = await stateRepo.get() + for (let [url, persistedIdentityManagerPromise] of identityManagerCache) { + const persistedIdentityManager = await persistedIdentityManagerPromise + const featureService: FeatureServiceConfig | undefined = config.featureServices.find((service: FeatureServiceConfig) => service.url === url) + if (featureService) { + const identityManager = ArcGISIdentityManager.deserialize(featureService.identityManager) + if (identityManager.token !== persistedIdentityManager.token || identityManager.refreshToken !== persistedIdentityManager.refreshToken) { + featureService.identityManager = persistedIdentityManager.serialize() + await stateRepo.put(config) + } + } + } + } + } +} + +export function getPortalUrl(featureService: FeatureServiceConfig | string): string { + const url = getFeatureServiceUrl(featureService) + return `https://${url.hostname}/arcgis/sharing/rest` +} + +export function getServerUrl(featureService: FeatureServiceConfig | string): string { + const url = getFeatureServiceUrl(featureService) + return `https://${url.hostname}/arcgis` +} + +export function getFeatureServiceUrl(featureService: FeatureServiceConfig | string): URL { + const url = typeof featureService === 'string' ? featureService : featureService.url + return new URL(url) +} \ No newline at end of file diff --git a/plugins/arcgis/service/src/FeatureQuerier.ts b/plugins/arcgis/service/src/FeatureQuerier.ts index 5cf43893d..395f085d0 100644 --- a/plugins/arcgis/service/src/FeatureQuerier.ts +++ b/plugins/arcgis/service/src/FeatureQuerier.ts @@ -63,7 +63,8 @@ export class FeatureQuerier { this._console.info('ArcGIS query: ' + queryUrl) const queryResponse = await request(queryUrl.toString(), { - authentication: this._identityManager + authentication: this._identityManager, + params: { f: 'json' } }); this._console.info('ArcGIS response for ' + queryUrl + ' ' + queryResponse.toString) @@ -107,10 +108,9 @@ export class FeatureQuerier { queryUrl.searchParams.set('outFields', this.outFields([field])); queryUrl.searchParams.set('returnGeometry', 'false'); this._console.info('ArcGIS query: ' + queryUrl) - + const queryResponse = await request(queryUrl.toString(), { authentication: this._identityManager - }); this._console.info('ArcGIS response for ' + queryUrl + ' ' + queryResponse) const result = queryResponse as QueryObjectResult diff --git a/plugins/arcgis/service/src/FeatureService.ts b/plugins/arcgis/service/src/FeatureService.ts index d5d4adf18..aabf76738 100644 --- a/plugins/arcgis/service/src/FeatureService.ts +++ b/plugins/arcgis/service/src/FeatureService.ts @@ -1,7 +1,3 @@ -import { LayerInfoResult } from "./LayerInfoResult"; -import { FeatureServiceResult } from "./FeatureServiceResult"; -import { HttpClient } from "./HttpClient"; -import { getIdentityManager } from "./ArcGISIdentityManagerFactory" import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" import { queryFeatures, applyEdits, IQueryFeaturesOptions } from "@esri/arcgis-rest-feature-service"; import { FeatureServiceConfig } from "./ArcGISConfig"; @@ -11,28 +7,13 @@ import { FeatureServiceConfig } from "./ArcGISConfig"; */ export class FeatureService { - /** - * Used to make the get request about the feature layer. - */ - // private _httpClient: HttpClient; - - /** - * Used to log messages. - */ private _console: Console; + private _config: FeatureServiceConfig; + private _identityManager: ArcGISIdentityManager; - private _config: FeatureServiceConfig; - private _identityManager: ArcGISIdentityManager; - - /** - * Constructor. - * @param console Used to log messages. - * @param token The access token. - */ constructor(console: Console, config: FeatureServiceConfig, identityManager: ArcGISIdentityManager) { - this._config = config; - this._identityManager = identityManager; - // this._httpClient = new HttpClient(console, token); + this._config = config; + this._identityManager = identityManager; this._console = console; } @@ -40,7 +21,7 @@ export class FeatureService { // By finishing this class, we can transition from low-level generic requests that leverage ArcGISIdentityManager for auth to higher-level strongly typed requests. - // Query features using arcgis-rest-js's queryFeatures + // Query features using arcgis-rest-js's queryFeatures async queryFeatureService(whereClause: string): Promise { const queryParams = { url: this._config.url, diff --git a/plugins/arcgis/service/src/FeatureServiceAdmin.ts b/plugins/arcgis/service/src/FeatureServiceAdmin.ts index 977452ad1..bdf0c6a94 100644 --- a/plugins/arcgis/service/src/FeatureServiceAdmin.ts +++ b/plugins/arcgis/service/src/FeatureServiceAdmin.ts @@ -7,315 +7,295 @@ import { ObservationsTransformer } from "./ObservationsTransformer" import { LayerInfoResult, LayerField } from "./LayerInfoResult" import FormData from 'form-data' import { request } from '@esri/arcgis-rest-request' -import { getIdentityManager } from './ArcGISIdentityManagerFactory' +import { ArcGISIdentityService, getFeatureServiceUrl } from "./ArcGISService" /** * Administers hosted feature services such as layer creation and updates. */ export class FeatureServiceAdmin { - - /** - * ArcGIS configuration. - */ - private _config: ArcGISPluginConfig - - /** - * Used to log to the console. - */ - private _console: Console - - /** - * Constructor. - * @param config The plugins configuration. - * @param console Used to log to the console. - */ - constructor(config: ArcGISPluginConfig, console: Console) { - this._config = config - this._console = console - } - - /** - * Create the layer - * @param service feature service - * @param featureLayer feature layer - * @param nextId next service layer id - * @param eventRepo event repository - * @returns layer id - */ - async createLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, nextId: number, eventRepo: MageEventRepository): Promise { - - const layer = { type: 'Feature Layer' } as Layer - - const layerIdentifier = featureLayer.layer - const layerIdentifierNumber = Number(layerIdentifier) - if (isNaN(layerIdentifierNumber)) { - layer.name = String(layerIdentifier) - layer.id = nextId - } else { - layer.id = layerIdentifierNumber - } - - const events = await this.layerEvents(featureLayer, eventRepo) - - if (layer.name == null) { - layer.name = this.layerName(events) - } - - if (featureLayer.geometryType != null) { - layer.geometryType = featureLayer.geometryType - } else { - layer.geometryType = 'esriGeometryPoint' - } - - layer.fields = this.fields(events) - - // TODO What other layer properties are needed or required? - // https://developers.arcgis.com/rest/services-reference/online/add-to-definition-feature-service-.htm#GUID-63F2BD08-DCF4-485D-A3E6-C7116E17DDD8 - - this.create(service, layer) - - return layer.id - } - - /** - * Update the layer fields - * @param service feature service - * @param featureLayer feature layer - * @param layerInfo layer info - * @param eventRepo event repository - */ - async updateLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, eventRepo: MageEventRepository) { - - const events = await this.layerEvents(featureLayer, eventRepo) - - const eventFields = this.fields(events) - const layerFields = layerInfo.fields - - // TODO - better naming: addFields is a boolean, array of fields, and a method. Ditto for deleteFields - if (featureLayer.addFields) { - - const layerFieldSet = new Set() - for (const field of layerFields) { - layerFieldSet.add(field.name) - } - - const addFields = [] - for (const field of eventFields) { - if (!layerFieldSet.has(field.name)) { - addFields.push(field) - const layerField = {} as LayerField - layerField.name = field.name - layerField.editable = true - layerFields.push(layerField) - } - } - - if (addFields.length > 0) { - this.addFields(service, featureLayer, addFields) - } - - } - - if (featureLayer.deleteFields) { - - const eventFieldSet = new Set() - for (const field of eventFields) { - eventFieldSet.add(field.name) - } - - const deleteFields = [] - const remainingFields = [] - for (const field of layerFields) { - if (field.editable && !eventFieldSet.has(field.name)) { - deleteFields.push(field) - } else { - remainingFields.push(field) - } - } - - if (deleteFields.length > 0) { - layerInfo.fields = remainingFields - this.deleteFields(service, featureLayer, deleteFields) - } - - } - - } - - /** - * Get the layer events - * @param layer feature layer - * @param eventRepo event repository - * @returns layer events - */ - private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise { - - const layerEvents: Set = new Set() - if (layer.events != null) { - for (const layerEvent of layer.events) { - layerEvents.add(layerEvent) - } - } - - let mageEvents - if (layerEvents.size > 0) { - mageEvents = await eventRepo.findAll() - } else { - mageEvents = await eventRepo.findActiveEvents() - } - - const events: MageEvent[] = [] - for (const mageEvent of mageEvents) { - if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) { - const event = await eventRepo.findById(mageEvent.id) - if (event != null) { - events.push(event) - } - } - } - - return events - } - - /** - * Create a layer name - * @param events layer events - * @returns layer name - */ - private layerName(events: MageEvent[]): string { - let layerName = '' - for (let i = 0; i < events.length; i++) { - if (i > 0) { - layerName += ', ' - } - layerName += events[i].name - } - return layerName - } - - /** - * Builder the layer fields - * @param events layer events - * @returns fields - */ - private fields(events: MageEvent[]): Field[] { - - const fields: Field[] = [] - - fields.push(this.createTextField(this._config.observationIdField, false)) - if (this._config.eventIdField != null) { - fields.push(this.createIntegerField(this._config.eventIdField, false)) - } - if (this._config.eventNameField != null) { - fields.push(this.createTextField(this._config.eventNameField, true)) - } - if (this._config.userIdField != null) { - fields.push(this.createTextField(this._config.userIdField, true)) - } - if (this._config.usernameField != null) { - fields.push(this.createTextField(this._config.usernameField, true)) - } - if (this._config.userDisplayNameField != null) { - fields.push(this.createTextField(this._config.userDisplayNameField, true)) - } - if (this._config.deviceIdField != null) { - fields.push(this.createTextField(this._config.deviceIdField, true)) - } - if (this._config.createdAtField != null) { - fields.push(this.createDateTimeField(this._config.createdAtField, true)) - } - if (this._config.lastModifiedField != null) { - fields.push(this.createDateTimeField(this._config.lastModifiedField, true)) - } - if (this._config.geometryType != null) { - fields.push(this.createTextField(this._config.geometryType, true)) - } - - const fieldNames = new Set() - for (const field of fields) { - fieldNames.add(field.name) - } - - this.eventsFields(events, fields, fieldNames) - - return fields - } - - /** - * Create a field - * @param name field name - * @param type form field type - * @param nullable nullable flag - * @param integer integer flag when numeric - * @returns field - */ - private createField(name: string, type: FormFieldType, nullable: boolean, integer?: boolean): Field { - let field = this.initField(type, integer) as Field - if (field != null) { - field.name = ObservationsTransformer.replaceSpaces(name) - field.alias = field.name - field.nullable = nullable - field.editable = true - } - return field - } - - /** - * Create a text field - * @param name field name - * @param nullable nullable flag - * @returns field - */ - private createTextField(name: string, nullable: boolean): Field { - return this.createField(name, FormFieldType.Text, nullable) - } - - /** - * Create a numeric field - * @param name field name - * @param nullable nullable flag - * @param integer integer flag - * @returns field - */ - private createNumericField(name: string, nullable: boolean, integer?: boolean): Field { - return this.createField(name, FormFieldType.Numeric, nullable, integer) - } - - /** - * Create an integer field - * @param name field name - * @param nullable nullable flag - * @returns field - */ - private createIntegerField(name: string, nullable: boolean): Field { - return this.createNumericField(name, nullable, true) - } - - /** - * Create a date time field - * @param name field name - * @param nullable nullable flag - * @returns field - */ - private createDateTimeField(name: string, nullable: boolean): Field { - return this.createField(name, FormFieldType.DateTime, nullable) - } - - /** - * Build fields from the layer events - * @param events layer events - * @param fields created fields - * @param fieldNames set of all field names - */ - private eventsFields(events: MageEvent[], fields: Field[], fieldNames: Set) { - - const forms = new Set() - - for (const event of events) { - this.eventFields(event, forms, fields, fieldNames) - } - - } + private _config: ArcGISPluginConfig + private _identityService: ArcGISIdentityService + private _console: Console + + constructor(config: ArcGISPluginConfig, identityService: ArcGISIdentityService, console: Console) { + this._config = config + this._identityService = identityService + this._console = console + } + + /** + * Create the layer + * @param service feature service + * @param featureLayer feature layer + * @param nextId next service layer id + * @param eventRepo event repository + * @returns layer id + */ + async createLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, nextId: number, eventRepo: MageEventRepository): Promise { + const layer = { type: 'Feature Layer' } as Layer + + const layerIdentifier = featureLayer.layer + const layerIdentifierNumber = Number(layerIdentifier) + if (isNaN(layerIdentifierNumber)) { + layer.name = String(layerIdentifier) + layer.id = nextId + } else { + layer.id = layerIdentifierNumber + } + + const events = await this.layerEvents(featureLayer, eventRepo) + + if (layer.name == null) { + layer.name = this.layerName(events) + } + + if (featureLayer.geometryType != null) { + layer.geometryType = featureLayer.geometryType + } else { + layer.geometryType = 'esriGeometryPoint' + } + + layer.fields = this.fields(events) + + // TODO What other layer properties are needed or required? + // https://developers.arcgis.com/rest/services-reference/online/add-to-definition-feature-service-.htm#GUID-63F2BD08-DCF4-485D-A3E6-C7116E17DDD8 + + this.create(service, layer) + + return layer.id + } + + /** + * Update the layer fields + * @param service feature service + * @param featureLayer feature layer + * @param layerInfo layer info + * @param eventRepo event repository + */ + async updateLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, eventRepo: MageEventRepository) { + const events = await this.layerEvents(featureLayer, eventRepo) + + const eventFields = this.fields(events) + const layerFields = layerInfo.fields + + // TODO - better naming: addFields is a boolean, array of fields, and a method. Ditto for deleteFields + if (featureLayer.addFields) { + + const layerFieldSet = new Set() + for (const field of layerFields) { + layerFieldSet.add(field.name) + } + + const addFields = [] + for (const field of eventFields) { + if (!layerFieldSet.has(field.name)) { + addFields.push(field) + const layerField = {} as LayerField + layerField.name = field.name + layerField.editable = true + layerFields.push(layerField) + } + } + + if (addFields.length > 0) { + this.addFields(service, featureLayer, addFields) + } + + } + + if (featureLayer.deleteFields) { + const eventFieldSet = new Set() + for (const field of eventFields) { + eventFieldSet.add(field.name) + } + + const deleteFields = [] + const remainingFields = [] + for (const field of layerFields) { + if (field.editable && !eventFieldSet.has(field.name)) { + deleteFields.push(field) + } else { + remainingFields.push(field) + } + } + + if (deleteFields.length > 0) { + layerInfo.fields = remainingFields + this.deleteFields(service, featureLayer, deleteFields) + } + } + } + + /** + * Get the layer events + * @param layer feature layer + * @param eventRepo event repository + * @returns layer events + */ + private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise { + const layerEvents: Set = new Set() + if (layer.events != null) { + for (const layerEvent of layer.events) { + layerEvents.add(layerEvent) + } + } + + let mageEvents + if (layerEvents.size > 0) { + mageEvents = await eventRepo.findAll() + } else { + mageEvents = await eventRepo.findActiveEvents() + } + + const events: MageEvent[] = [] + for (const mageEvent of mageEvents) { + if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) { + const event = await eventRepo.findById(mageEvent.id) + if (event != null) { + events.push(event) + } + } + } + + return events + } + + /** + * Create a layer name + * @param events layer events + * @returns layer name + */ + private layerName(events: MageEvent[]): string { + let layerName = '' + for (let i = 0; i < events.length; i++) { + if (i > 0) { + layerName += ', ' + } + layerName += events[i].name + } + return layerName + } + + /** + * Builder the layer fields + * @param events layer events + * @returns fields + */ + private fields(events: MageEvent[]): Field[] { + const fields: Field[] = [] + + fields.push(this.createTextField(this._config.observationIdField, false)) + if (this._config.eventIdField != null) { + fields.push(this.createIntegerField(this._config.eventIdField, false)) + } + if (this._config.eventNameField != null) { + fields.push(this.createTextField(this._config.eventNameField, true)) + } + if (this._config.userIdField != null) { + fields.push(this.createTextField(this._config.userIdField, true)) + } + if (this._config.usernameField != null) { + fields.push(this.createTextField(this._config.usernameField, true)) + } + if (this._config.userDisplayNameField != null) { + fields.push(this.createTextField(this._config.userDisplayNameField, true)) + } + if (this._config.deviceIdField != null) { + fields.push(this.createTextField(this._config.deviceIdField, true)) + } + if (this._config.createdAtField != null) { + fields.push(this.createDateTimeField(this._config.createdAtField, true)) + } + if (this._config.lastModifiedField != null) { + fields.push(this.createDateTimeField(this._config.lastModifiedField, true)) + } + if (this._config.geometryType != null) { + fields.push(this.createTextField(this._config.geometryType, true)) + } + + const fieldNames = new Set() + for (const field of fields) { + fieldNames.add(field.name) + } + + this.eventsFields(events, fields, fieldNames) + + return fields + } + + /** + * Create a field + * @param name field name + * @param type form field type + * @param nullable nullable flag + * @param integer integer flag when numeric + * @returns field + */ + private createField(name: string, type: FormFieldType, nullable: boolean, integer?: boolean): Field { + let field = this.initField(type, integer) as Field + if (field != null) { + field.name = ObservationsTransformer.replaceSpaces(name) + field.alias = field.name + field.nullable = nullable + field.editable = true + } + return field + } + + /** + * Create a text field + * @param name field name + * @param nullable nullable flag + * @returns field + */ + private createTextField(name: string, nullable: boolean): Field { + return this.createField(name, FormFieldType.Text, nullable) + } + + /** + * Create a numeric field + * @param name field name + * @param nullable nullable flag + * @param integer integer flag + * @returns field + */ + private createNumericField(name: string, nullable: boolean, integer?: boolean): Field { + return this.createField(name, FormFieldType.Numeric, nullable, integer) + } + + /** + * Create an integer field + * @param name field name + * @param nullable nullable flag + * @returns field + */ + private createIntegerField(name: string, nullable: boolean): Field { + return this.createNumericField(name, nullable, true) + } + + /** + * Create a date time field + * @param name field name + * @param nullable nullable flag + * @returns field + */ + private createDateTimeField(name: string, nullable: boolean): Field { + return this.createField(name, FormFieldType.DateTime, nullable) + } + + /** + * Build fields from the layer events + * @param events layer events + * @param fields created fields + * @param fieldNames set of all field names + */ + private eventsFields(events: MageEvent[], fields: Field[], fieldNames: Set) { + const forms = new Set() + + for (const event of events) { + this.eventFields(event, forms, fields, fieldNames) + } + } /** * Build fields from the layer event @@ -324,205 +304,193 @@ export class FeatureServiceAdmin { * @param fields created fields * @param fieldNames set of all field names */ - private eventFields(event: MageEvent, forms: Set, fields: Field[], fieldNames: Set) { - - for (const form of event.activeForms) { - - if (!forms.has(form.id)) { - - forms.add(form.id) - - for (const formField of form.fields) { - if (formField.archived == null || !formField.archived) { - this.createFormField(form, formField, fields, fieldNames) - } - } - - } - } - - } - - /** - * Build a field from the form field - * @param form form - * @param formField form field - * @param fields created fields - * @param fieldNames set of all field names - */ - private createFormField(form: Form, formField: FormField, fields: Field[], fieldNames: Set) { - const field = this.initField(formField.type) - - if (field != null) { - const sanitizedName = ObservationsTransformer.replaceSpaces(formField.title) - const sanitizedFormName = ObservationsTransformer.replaceSpaces(form.name) - const name = `${sanitizedFormName}_${sanitizedName}` - - fieldNames.add(name) - - field.name = name - field.alias = field.name - field.nullable = !formField.required - field.editable = true - field.defaultValue = formField.value - - fields.push(field) - } - } - - /** - * Initialize a field by type - * @param type form field type - * @param integer numeric integer field type - * @return field or null - */ - private initField(type: FormFieldType, integer?: boolean): Field | null { - - let field = {} as Field - - switch (type) { - case FormFieldType.CheckBox: - case FormFieldType.Dropdown: - case FormFieldType.Email: - case FormFieldType.MultiSelectDropdown: - case FormFieldType.Password: - case FormFieldType.Radio: - case FormFieldType.Text: - field.type = 'esriFieldTypeString' - field.actualType = 'nvarchar' - field.sqlType = 'sqlTypeNVarchar' - field.length = this._config.textFieldLength - break; - case FormFieldType.TextArea: - field.type = 'esriFieldTypeString' - field.actualType = 'nvarchar' - field.sqlType = 'sqlTypeNVarchar' - field.length = this._config.textAreaFieldLength - break; - case FormFieldType.DateTime: - field.type = 'esriFieldTypeDate' - field.sqlType = 'sqlTypeOther' - field.length = 10 - break; - case FormFieldType.Numeric: - if (integer) { - field.type = 'esriFieldTypeInteger' - field.actualType = 'int' - field.sqlType = 'sqlTypeInteger' - } else { - field.type = 'esriFieldTypeDouble' - field.actualType = 'float' - field.sqlType = 'sqlTypeFloat' - } - break; - case FormFieldType.Geometry: - case FormFieldType.Attachment: - case FormFieldType.Hidden: - default: - break - } - - return field.type != null ? field : null - } - - /** - * Create the layer - * @param service feature service - * @param layer layer - */ - private async create(service: FeatureServiceConfig, layer: Layer) { - - const identityManager = await getIdentityManager(service) - const url = this.adminUrl(service) + 'addToDefinition' - - this._console.info('ArcGIS feature service addToDefinition (create layer) url ' + url) - - const form = new FormData() - form.append('addToDefinition', JSON.stringify(layer)) - - const postResponse = request(url, { - authentication: identityManager, - httpMethod: 'POST', - params: form - }); - console.log('Response: ' + postResponse) - - } - - /** - * Add fields to the layer - * @param service feature service - * @param featureLayer feature layer - * @param fields fields to add - */ - private async addFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: Field[]) { - - const layer = { fields: fields} as Layer - - const identityManager = await getIdentityManager(service) - const url = this.adminUrl(service) + featureLayer.layer.toString() + '/addToDefinition' - - this._console.info('ArcGIS feature layer addToDefinition (add fields) url ' + url) - - await request(url, { - authentication: identityManager, - params: { - addToDefinition: JSON.stringify(layer), - f: "json" - } - }).then((postResponse) => { - console.log('Response: ' + postResponse) - }).catch((error) => { - console.log('Error: ' + error) - }); - } - - /** - * Delete fields from the layer - * @param service feature service - * @param featureLayer feature layer - * @param fields fields to delete - */ - private async deleteFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: LayerField[]) { - - const deleteFields = [] - for (const layerField of fields) { - const field = {} as Field - field.name = layerField.name - deleteFields.push(field) - } - - const layer = {} as Layer - layer.fields = deleteFields - - const identityManager = await getIdentityManager(service) - const url = this.adminUrl(service) + featureLayer.layer.toString() + '/deleteFromDefinition' - - this._console.info('ArcGIS feature layer deleteFromDefinition (delete fields) url ' + url) - - const postResponse = request(url, { - authentication: identityManager, - httpMethod: 'POST', - params: { - deleteFromDefinition: JSON.stringify(layer) - } - }); - console.log('Response: ' + postResponse) - } - - /** - * Get the administration url - * @param service feature service - * @returns url - */ - private adminUrl(service: FeatureServiceConfig): String { - let url = service.adminUrl - if (url == null) { - url = service.url.replace('/services/', '/admin/services/') - } - if (!url.endsWith('/')) { - url += '/' - } - return url - } -} + private eventFields(event: MageEvent, forms: Set, fields: Field[], fieldNames: Set) { + for (const form of event.activeForms) { + if (!forms.has(form.id)) { + forms.add(form.id) + + for (const formField of form.fields) { + if (formField.archived == null || !formField.archived) { + this.createFormField(form, formField, fields, fieldNames) + } + } + } + } + } + + /** + * Build a field from the form field + * @param form form + * @param formField form field + * @param fields created fields + * @param fieldNames set of all field names + */ + private createFormField(form: Form, formField: FormField, fields: Field[], fieldNames: Set) { + const field = this.initField(formField.type) + + if (field != null) { + const sanitizedName = ObservationsTransformer.replaceSpaces(formField.title) + const sanitizedFormName = ObservationsTransformer.replaceSpaces(form.name) + const name = `${sanitizedFormName}_${sanitizedName}` + + fieldNames.add(name) + + field.name = name + field.alias = field.name + field.nullable = !formField.required + field.editable = true + field.defaultValue = formField.value + + fields.push(field) + } + } + + /** + * Initialize a field by type + * @param type form field type + * @param integer numeric integer field type + * @return field or null + */ + private initField(type: FormFieldType, integer?: boolean): Field | null { + let field = {} as Field + + switch (type) { + case FormFieldType.CheckBox: + case FormFieldType.Dropdown: + case FormFieldType.Email: + case FormFieldType.MultiSelectDropdown: + case FormFieldType.Password: + case FormFieldType.Radio: + case FormFieldType.Text: + field.type = 'esriFieldTypeString' + field.actualType = 'nvarchar' + field.sqlType = 'sqlTypeNVarchar' + field.length = this._config.textFieldLength + break; + case FormFieldType.TextArea: + field.type = 'esriFieldTypeString' + field.actualType = 'nvarchar' + field.sqlType = 'sqlTypeNVarchar' + field.length = this._config.textAreaFieldLength + break; + case FormFieldType.DateTime: + field.type = 'esriFieldTypeDate' + field.sqlType = 'sqlTypeOther' + field.length = 10 + break; + case FormFieldType.Numeric: + if (integer) { + field.type = 'esriFieldTypeInteger' + field.actualType = 'int' + field.sqlType = 'sqlTypeInteger' + } else { + field.type = 'esriFieldTypeDouble' + field.actualType = 'float' + field.sqlType = 'sqlTypeFloat' + } + break; + case FormFieldType.Geometry: + case FormFieldType.Attachment: + case FormFieldType.Hidden: + default: + break + } + + return field.type != null ? field : null + } + + /** + * Create the layer + * @param service feature service + * @param layer layer + */ + private async create(service: FeatureServiceConfig, layer: Layer) { + const url = this.adminUrl(service) + 'addToDefinition' + + this._console.info('ArcGIS feature service addToDefinition (create layer) url ' + url) + + const form = new FormData() + form.append('addToDefinition', JSON.stringify(layer)) + + const identityManager = await this._identityService.getIdentityManager(service) + const postResponse = await request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: form + }); + console.log('Response: ' + JSON.stringify(postResponse)) + } + + /** + * Add fields to the layer + * @param service feature service + * @param featureLayer feature layer + * @param fields fields to add + */ + private async addFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: Field[]) { + const layer = { fields: fields} as Layer + + const url = this.adminUrl(service) + featureLayer.layer.toString() + '/addToDefinition' + + this._console.info('ArcGIS feature layer addToDefinition (add fields) url ' + url) + + const identityManager = await this._identityService.getIdentityManager(service) + await request(url, { + authentication: identityManager, + params: { + addToDefinition: JSON.stringify(layer), + f: "json" + } + }).then((postResponse) => { + console.log('Response: ' + postResponse) + }).catch((error) => { + console.log('Error: ' + error) + }); + } + + /** + * Delete fields from the layer + * @param service feature service + * @param featureLayer feature layer + * @param fields fields to delete + */ + private async deleteFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: LayerField[]) { + const deleteFields = [] + for (const layerField of fields) { + const field = {} as Field + field.name = layerField.name + deleteFields.push(field) + } + + const layer = {} as Layer + layer.fields = deleteFields + + const url = this.adminUrl(service) + featureLayer.layer.toString() + '/deleteFromDefinition' + + this._console.info('ArcGIS feature layer deleteFromDefinition (delete fields) url ' + url) + + const identityManager = await this._identityService.getIdentityManager(service) + const postResponse = request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: { + deleteFromDefinition: JSON.stringify(layer) + } + }); + console.log('Response: ' + postResponse) + } + + /** + * Get the administration url + * @param service feature service + * @returns url + */ + private adminUrl(service: FeatureServiceConfig): String { + let url = service.url.replace('/services/', '/admin/services/') + if (!url.endsWith('/')) { + url += '/' + } + + return url + } +} \ No newline at end of file diff --git a/plugins/arcgis/service/src/LayerInfo.ts b/plugins/arcgis/service/src/LayerInfo.ts index 931937cf4..4918d7153 100644 --- a/plugins/arcgis/service/src/LayerInfo.ts +++ b/plugins/arcgis/service/src/LayerInfo.ts @@ -37,9 +37,8 @@ export class LayerInfo { * @param layerInfo The layer info. * @param token The access token. */ - constructor(url: string, events: string[], layerInfo: LayerInfoResult, token?: string) { + constructor(url: string, events: string[], layerInfo: LayerInfoResult) { this.url = url - this.token = token if (events != undefined && events != null && events.length == 0) { this.events.add('nothing to sync') } diff --git a/plugins/arcgis/service/src/ObservationProcessor.ts b/plugins/arcgis/service/src/ObservationProcessor.ts index 045541a1a..23379aff5 100644 --- a/plugins/arcgis/service/src/ObservationProcessor.ts +++ b/plugins/arcgis/service/src/ObservationProcessor.ts @@ -14,11 +14,11 @@ import { EventTransform } from './EventTransform'; import { GeometryChangedHandler } from './GeometryChangedHandler'; import { EventDeletionHandler } from './EventDeletionHandler'; import { EventLayerProcessorOrganizer } from './EventLayerProcessorOrganizer'; -import { FeatureServiceConfig, FeatureLayerConfig, AuthType } from "./ArcGISConfig" +import { FeatureServiceConfig, FeatureLayerConfig } from "./ArcGISConfig" import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' import { FeatureServiceAdmin } from './FeatureServiceAdmin'; -import { getIdentityManager } from "./ArcGISIdentityManagerFactory" import { request } from '@esri/arcgis-rest-request'; +import { ArcGISIdentityService } from './ArcGISService'; /** * Class that wakes up at a certain configured interval and processes any new observations that can be @@ -26,424 +26,398 @@ import { request } from '@esri/arcgis-rest-request'; */ export class ObservationProcessor { - /** - * True if the processor is currently active, false otherwise. - */ - private _isRunning = false; - - /** - * The next timeout, use this to cancel the next one if the processor is stopped. - */ - private _nextTimeout: NodeJS.Timeout | undefined; - - /** - * Used to get all the active events. - */ - private _eventRepo: MageEventRepository; - - /** - * Used to get new observations. - */ - private _obsRepos: ObservationRepositoryForEvent; - - /** - * Used to get user information. - */ - private _userRepo: UserRepository; - - /** - * Used to log to the console. - */ - private _console: Console; - - /** - * Used to convert observations to json string that can be sent to an arcgis server. - */ - private _transformer: ObservationsTransformer; - - /** - * Contains the different feature layers to send observations too. - */ - private _stateRepo: PluginStateRepository; - - /** - * The previous plugins configuration JSON. - */ - private _previousConfig?: string; - - /** - * Sends observations to a single feature layer. - */ - private _layerProcessors: FeatureLayerProcessor[] = []; - - /** - * True if this is a first run at updating arc feature layers. If so we need to make sure the layers are - * all up to date. - */ - private _firstRun: boolean; - - /** - * Handles removing observation from previous layers when an observation geometry changes. - */ - private _geometryChangeHandler: GeometryChangedHandler; - - /** - * Handles removing observations when an event is deleted. - */ - private _eventDeletionHandler: EventDeletionHandler; - - /** - * Maps the events to the processor they are synching data for. - */ - private _organizer: EventLayerProcessorOrganizer; - - /** - * Constructor. - * @param stateRepo The plugins configuration. - * @param eventRepo Used to get all the active events. - * @param obsRepo Used to get new observations. - * @param userRepo Used to get user information. - * @param console Used to log to the console. - */ - constructor(stateRepo: PluginStateRepository, eventRepo: MageEventRepository, obsRepos: ObservationRepositoryForEvent, userRepo: UserRepository, console: Console) { - this._stateRepo = stateRepo; - this._eventRepo = eventRepo; - this._obsRepos = obsRepos; - this._userRepo = userRepo; - this._console = console; - this._firstRun = true; - this._organizer = new EventLayerProcessorOrganizer(); - this._transformer = new ObservationsTransformer(defaultArcGISPluginConfig, console); - this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); - this._eventDeletionHandler = new EventDeletionHandler(this._console, defaultArcGISPluginConfig); - } - - /** - * Gets the current configuration from the database. - * @returns The current configuration from the database. - */ - public async safeGetConfig(): Promise { - const state = await this._stateRepo.get(); - if (!state) return await this._stateRepo.put(defaultArcGISPluginConfig); - return await this._stateRepo.get().then((state) => state ? state : this._stateRepo.put(defaultArcGISPluginConfig)); - } - - /** - * Puts a new confguration in the state repo. - * @param newConfig The new config to put into the state repo. - */ - public async putConfig(newConfig: ArcGISPluginConfig): Promise { - return await this._stateRepo.put(newConfig); - } - - /** - * Updates the confguration in the state repo. - * @param newConfig The new config to put into the state repo. - */ - public async patchConfig(newConfig: ArcGISPluginConfig): Promise { - return await this._stateRepo.patch(newConfig); - } - - /** - * Gets the current configuration and updates the processor if needed - * @returns The current configuration from the database. - */ - private async updateConfig(): Promise { - const config = await this.safeGetConfig() - const configJson = JSON.stringify(config) - if (this._previousConfig == null || this._previousConfig != configJson) { - this._transformer = new ObservationsTransformer(config, console); - this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); - this._eventDeletionHandler = new EventDeletionHandler(this._console, config); - this._layerProcessors = []; - this.getFeatureServiceLayers(config); - this._previousConfig = configJson - this._firstRun = true; - } - return config - } - - /** - * Starts the processor. - */ - async start() { - this._isRunning = true; - this._firstRun = true; - this.processAndScheduleNext(); - } - - /** - * Stops the processor. - */ - stop() { - this._isRunning = false; - clearTimeout(this._nextTimeout); - } - - /** - * Gets information on all the configured features service layers. - * @param config The plugins configuration. - */ - private async getFeatureServiceLayers(config: ArcGISPluginConfig) { - // TODO: What is the impact of what this is doing? Do we need to account for usernamePassword auth type services? - for (const service of config.featureServices) { - - const services: FeatureServiceConfig[] = [] - - if (service.auth?.type !== AuthType.Token || service.auth?.token == null) { - const tokenServices = new Map() - const nonTokenLayers = [] - for (const layer of service.layers) { - if (layer.token != null) { - let serv = tokenServices.get(layer.token) - if (serv == null) { - serv = { url: service.url, token: layer.token, layers: [] } - tokenServices.set(layer.token, serv) - services.push(serv) - } - serv.layers.push(layer) - } else { - nonTokenLayers.push(layer) - } - } - if (services.length > 0) { - service.layers = nonTokenLayers - } - } - - if (service.layers.length > 0) { - services.push(service) - } - - for (const serv of services) { - try { - const identityManager = await getIdentityManager(serv) - const response = await request(serv.url, { - authentication: identityManager - }) as FeatureServiceResult - this.handleFeatureService(response, serv, config); - } catch (err) { - console.error(err) - // res.status(500).json({ message: 'Could not get ArcGIS layer info', error: err }) - } - - - } - } - } - - /** - * Called when information on a feature service is returned from an arc server. - * @param featureService The feature service. - * @param featureServiceConfig The feature service config. - * @param config The plugin configuration. - */ - private async handleFeatureService(featureService: FeatureServiceResult, featureServiceConfig: FeatureServiceConfig, config: ArcGISPluginConfig) { - - if (featureService.layers != null) { - - const serviceLayers = new Map() - const admin = new FeatureServiceAdmin(config, this._console) - - let maxId = -1 - for (const layer of featureService.layers) { - serviceLayers.set(layer.id, layer) - serviceLayers.set(layer.name, layer) - maxId = Math.max(maxId, layer.id) - } - - for (const featureLayer of featureServiceConfig.layers) { - // Initiate the feature layer and event fields to sync with the feature service - featureLayer.addFields = true - - if (featureLayer.token == null) { - featureLayer.token = featureServiceConfig.auth?.type == AuthType.Token ? featureServiceConfig.auth.token : "" - } - - const eventNames: string[] = [] - const events = featureLayer.events - if (events != null) { - for (const event of events) { - const eventId = Number(event); - if (isNaN(eventId)) { - eventNames.push(String(event)); - } else { - const mageEvent = await this._eventRepo.findById(eventId) - if (mageEvent != null) { - eventNames.push(mageEvent.name); - } - } - } - } - if (eventNames.length > 0) { - featureLayer.events = eventNames - } - - const layer = serviceLayers.get(featureLayer.layer) - - let layerId = undefined - if (layer != null) { - layerId = layer.id - } else if (featureServiceConfig.createLayers) { - layerId = await admin.createLayer(featureServiceConfig, featureLayer, maxId + 1, this._eventRepo) - maxId = Math.max(maxId, layerId) - } - - if (layerId != null) { - featureLayer.layer = layerId - const identityManager = await getIdentityManager(featureServiceConfig) - const featureService = new FeatureService(console, featureServiceConfig, identityManager) - const layerInfo = await featureService.queryLayerInfo(layerId); - const url = `${featureServiceConfig.url}/${layerId}`; - this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config); - } - - } - - } - } - - /** - * Called when information on a feature layer is returned from an arc server. - * @param url The layer url. - * @param featureServiceConfig The feature service config. - * @param featureLayer The feature layer configuration. - * @param layerInfo The information on a layer. - * @param config The plugins configuration. - */ - private async handleLayerInfo(url: string, featureServiceConfig: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, config: ArcGISPluginConfig) { - if (layerInfo.geometryType != null) { - const events = featureLayer.events as string[] - if (featureLayer.addFields || featureLayer.deleteFields) { - const admin = new FeatureServiceAdmin(config, this._console) - await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo) - } - const info = new LayerInfo(url, events, layerInfo, featureLayer.token) - const identityManager = await getIdentityManager(featureServiceConfig) - const layerProcessor = new FeatureLayerProcessor(info, config, identityManager,this._console); - this._layerProcessors.push(layerProcessor); - // clearTimeout(this._nextTimeout); // TODO why is this needed? - // this.scheduleNext(config); // TODO why is this needed when processAndScheduleNext is called upstream and ends with scheduleNext() This causes a query before updateLayer. - } - } - - /** - * Processes any new observations and then schedules its next run if it hasn't been stopped. - */ - private async processAndScheduleNext() { - const config = await this.updateConfig(); - if (this._isRunning) { - if (config.enabled && this._layerProcessors.length > 0) { - this._console.info('ArcGIS plugin checking for any pending updates or adds'); - for (const layerProcessor of this._layerProcessors) { - layerProcessor.processPendingUpdates(); - } - this._console.info('ArcGIS plugin processing new observations...'); - const activeEvents = await this._eventRepo.findActiveEvents(); - this._eventDeletionHandler.checkForEventDeletion(activeEvents, this._layerProcessors, this._firstRun); - const eventsToProcessors = this._organizer.organize(activeEvents, this._layerProcessors); - const nextQueryTime = Date.now(); - for (const pair of eventsToProcessors) { - this._console.info('ArcGIS getting newest observations for event ' + pair.event.name); - const obsRepo = await this._obsRepos(pair.event.id); - const pagingSettings = { - pageSize: config.batchSize, - pageIndex: 0, - includeTotalCount: true - } - let morePages = true; - let numberLeft = 0; - while (morePages) { - numberLeft = await this.queryAndSend(config, pair.featureLayerProcessors, obsRepo, pagingSettings, numberLeft); - morePages = numberLeft > 0; - } - } - - for (const layerProcessor of this._layerProcessors) { - layerProcessor.lastTimeStamp = nextQueryTime; - } - - this._firstRun = false; - } - this.scheduleNext(config); - } - } - - private scheduleNext(config: ArcGISPluginConfig) { - if (this._isRunning) { - let interval = config.intervalSeconds; - if (this._firstRun && config.featureServices.length > 0) { - interval = config.startupIntervalSeconds; - } else { - for (const layerProcessor of this._layerProcessors) { - if (layerProcessor.hasPendingUpdates()) { - interval = config.updateIntervalSeconds; - break; - } - } - } - this._nextTimeout = setTimeout(() => { this.processAndScheduleNext() }, interval * 1000); - } - } - - /** - * Queries for new observations and sends them to any configured arc servers. - * @param config The plugin configuration. - * @param layerProcessors The layer processors to use when processing arc objects. - * @param obsRepo The observation repo for an event. - * @param pagingSettings Current paging settings. - * @param numberLeft The number of observations left to query and send to arc. - * @returns The number of observations still needing to be queried and sent to arc. - */ - private async queryAndSend(config: ArcGISPluginConfig, layerProcessors: FeatureLayerProcessor[], obsRepo: EventScopedObservationRepository, pagingSettings: PagingParameters, numberLeft: number): Promise { - let newNumberLeft = numberLeft; - - let queryTime = -1; - for (const layerProcessor of layerProcessors) { - if (queryTime == -1 || layerProcessor.lastTimeStamp < queryTime) { - queryTime = layerProcessor.lastTimeStamp; - } - } - - let latestObs = await obsRepo.findLastModifiedAfter(queryTime, pagingSettings); - if (latestObs != null && latestObs.totalCount != null && latestObs.totalCount > 0) { - if (pagingSettings.pageIndex == 0) { - this._console.info('ArcGIS newest observation count ' + latestObs.totalCount); - newNumberLeft = latestObs.totalCount; - } - const observations = latestObs.items - const mageEvent = await this._eventRepo.findById(obsRepo.eventScope) - const eventTransform = new EventTransform(config, mageEvent) - const arcObjects = new ArcObjects() - this._geometryChangeHandler.checkForGeometryChange(observations, arcObjects, layerProcessors, this._firstRun); - for (let i = 0; i < observations.length; i++) { - const observation = observations[i] - let deletion = false - if (observation.states.length > 0) { - deletion = observation.states[0].name.startsWith('archive') - } - if (deletion) { - const arcObservation = this._transformer.createObservation(observation) - arcObjects.deletions.push(arcObservation) - } else { - let user = null - if (observation.userId != null) { - user = await this._userRepo.findById(observation.userId) - } - const arcObservation = this._transformer.transform(observation, eventTransform, user) - arcObjects.add(arcObservation) - } - } - arcObjects.firstRun = this._firstRun; - for (const layerProcessor of layerProcessors) { - layerProcessor.processArcObjects(JSON.parse(JSON.stringify(arcObjects))); - } - newNumberLeft -= latestObs.items.length; - pagingSettings.pageIndex++; - } else { - this._console.info('ArcGIS no new observations') - } - - return newNumberLeft; - } + /** + * True if the processor is currently active, false otherwise. + */ + private _isRunning = false; + + /** + * The next timeout, use this to cancel the next one if the processor is stopped. + */ + private _nextTimeout: NodeJS.Timeout | undefined; + + /** + * Used to get all the active events. + */ + private _eventRepo: MageEventRepository; + + /** + * Used to get new observations. + */ + private _obsRepos: ObservationRepositoryForEvent; + + /** + * Used to get user information. + */ + private _userRepo: UserRepository; + + /** + * Used to manager ArcGIS user identities + */ + private _identityService: ArcGISIdentityService + + /** + * Used to log to the console. + */ + private _console: Console; + + /** + * Used to convert observations to json string that can be sent to an arcgis server. + */ + private _transformer: ObservationsTransformer; + + /** + * Contains the different feature layers to send observations too. + */ + private _stateRepo: PluginStateRepository; + + /** + * The previous plugins configuration JSON. + */ + private _previousConfig?: string; + + /** + * Sends observations to a single feature layer. + */ + private _layerProcessors: FeatureLayerProcessor[] = []; + + /** + * True if this is a first run at updating arc feature layers. If so we need to make sure the layers are + * all up to date. + */ + private _firstRun: boolean; + + /** + * Handles removing observation from previous layers when an observation geometry changes. + */ + private _geometryChangeHandler: GeometryChangedHandler; + + /** + * Handles removing observations when an event is deleted. + */ + private _eventDeletionHandler: EventDeletionHandler; + + /** + * Maps the events to the processor they are synching data for. + */ + private _organizer: EventLayerProcessorOrganizer; + + /** + * Constructor. + * @param stateRepo The plugins configuration. + * @param eventRepo Used to get all the active events. + * @param obsRepo Used to get new observations. + * @param userRepo Used to get user information. + * @param console Used to log to the console. + */ + constructor( + stateRepo: PluginStateRepository, + eventRepo: MageEventRepository, + obsRepos: ObservationRepositoryForEvent, + userRepo: UserRepository, + identityService: ArcGISIdentityService, + console: Console + ) { + this._stateRepo = stateRepo; + this._eventRepo = eventRepo; + this._obsRepos = obsRepos; + this._userRepo = userRepo; + this._identityService = identityService + this._console = console; + this._firstRun = true; + this._organizer = new EventLayerProcessorOrganizer(); + this._transformer = new ObservationsTransformer(defaultArcGISPluginConfig, console); + this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); + this._eventDeletionHandler = new EventDeletionHandler(this._console, defaultArcGISPluginConfig); + } + + /** + * Gets the current configuration from the database. + * @returns The current configuration from the database. + */ + public async safeGetConfig(): Promise { + const state = await this._stateRepo.get(); + if (!state) return await this._stateRepo.put(defaultArcGISPluginConfig); + return await this._stateRepo.get().then((state) => state ? state : this._stateRepo.put(defaultArcGISPluginConfig)); + } + + /** + * Puts a new confguration in the state repo. + * @param newConfig The new config to put into the state repo. + */ + public async putConfig(newConfig: ArcGISPluginConfig): Promise { + return await this._stateRepo.put(newConfig); + } + + /** + * Updates the confguration in the state repo. + * @param newConfig The new config to put into the state repo. + */ + public async patchConfig(newConfig: ArcGISPluginConfig): Promise { + return await this._stateRepo.patch(newConfig); + } + + /** + * Gets the current configuration and updates the processor if needed + * @returns The current configuration from the database. + */ + private async updateConfig(): Promise { + const config = await this.safeGetConfig() + const configJson = JSON.stringify(config) + if (this._previousConfig == null || this._previousConfig != configJson) { + this._transformer = new ObservationsTransformer(config, console); + this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); + this._eventDeletionHandler = new EventDeletionHandler(this._console, config); + this._layerProcessors = []; + this.getFeatureServiceLayers(config); + this._previousConfig = configJson + this._firstRun = true; + } + return config + } + + /** + * Starts the processor. + */ + async start() { + this._isRunning = true; + this._firstRun = true; + this.processAndScheduleNext(); + } + + /** + * Stops the processor. + */ + stop() { + this._isRunning = false; + clearTimeout(this._nextTimeout); + } + + /** + * Gets information on all the configured features service layers. + * @param config The plugins configuration. + */ + private async getFeatureServiceLayers(config: ArcGISPluginConfig) { + for (const service of config.featureServices) { + try { + const identityManager = await this._identityService.getIdentityManager(service) + const response = await request(service.url, { authentication: identityManager }) + this.handleFeatureService(response, service, config) + } catch (err) { + console.error(err) + } + } + } + + /** + * Called when information on a feature service is returned from an arc server. + * @param featureService The feature service. + * @param featureServiceConfig The feature service config. + * @param config The plugin configuration. + */ + private async handleFeatureService(featureService: FeatureServiceResult, featureServiceConfig: FeatureServiceConfig, config: ArcGISPluginConfig) { + + if (featureService.layers != null) { + + const serviceLayers = new Map() + const admin = new FeatureServiceAdmin(config, this._identityService, this._console) + + let maxId = -1 + for (const layer of featureService.layers) { + serviceLayers.set(layer.id, layer) + serviceLayers.set(layer.name, layer) + maxId = Math.max(maxId, layer.id) + } + + for (const featureLayer of featureServiceConfig.layers) { + // Initiate the feature layer and event fields to sync with the feature service + featureLayer.addFields = true + + const eventNames: string[] = [] + const events = featureLayer.events + if (events != null) { + for (const event of events) { + const eventId = Number(event); + if (isNaN(eventId)) { + eventNames.push(String(event)); + } else { + const mageEvent = await this._eventRepo.findById(eventId) + if (mageEvent != null) { + eventNames.push(mageEvent.name); + } + } + } + } + if (eventNames.length > 0) { + featureLayer.events = eventNames + } + + const layer = serviceLayers.get(featureLayer.layer) + + let layerId = undefined + if (layer != null) { + layerId = layer.id + } else { + layerId = await admin.createLayer(featureServiceConfig, featureLayer, maxId + 1, this._eventRepo) + maxId = Math.max(maxId, layerId) + } + + if (layerId != null) { + featureLayer.layer = layerId + const identityManager = await this._identityService.getIdentityManager(featureServiceConfig) + const featureService = new FeatureService(console, featureServiceConfig, identityManager) + const layerInfo = await featureService.queryLayerInfo(layerId); + const url = `${featureServiceConfig.url}/${layerId}`; + this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config); + } + } + } + } + + /** + * Called when information on a feature layer is returned from an arc server. + * @param url The layer url. + * @param featureServiceConfig The feature service config. + * @param featureLayer The feature layer configuration. + * @param layerInfo The information on a layer. + * @param config The plugins configuration. + */ + private async handleLayerInfo(url: string, featureServiceConfig: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, config: ArcGISPluginConfig) { + if (layerInfo.geometryType != null) { + const events = featureLayer.events as string[] + if (featureLayer.addFields || featureLayer.deleteFields) { + const admin = new FeatureServiceAdmin(config, this._identityService, this._console) + await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo) + } + const info = new LayerInfo(url, events, layerInfo) + const identityManager = await this._identityService.getIdentityManager(featureServiceConfig) + const layerProcessor = new FeatureLayerProcessor(info, config, identityManager, this._console); + this._layerProcessors.push(layerProcessor); + // clearTimeout(this._nextTimeout); // TODO why is this needed? + // this.scheduleNext(config); // TODO why is this needed when processAndScheduleNext is called upstream and ends with scheduleNext() This causes a query before updateLayer. + } + } + + /** + * Processes any new observations and then schedules its next run if it hasn't been stopped. + */ + private async processAndScheduleNext() { + const config = await this.updateConfig(); + if (this._isRunning) { + if (config.enabled && this._layerProcessors.length > 0) { + this._console.info('ArcGIS plugin checking for any pending updates or adds'); + for (const layerProcessor of this._layerProcessors) { + layerProcessor.processPendingUpdates(); + } + this._console.info('ArcGIS plugin processing new observations...'); + const activeEvents = await this._eventRepo.findActiveEvents(); + this._eventDeletionHandler.checkForEventDeletion(activeEvents, this._layerProcessors, this._firstRun); + const eventsToProcessors = this._organizer.organize(activeEvents, this._layerProcessors); + const nextQueryTime = Date.now(); + for (const pair of eventsToProcessors) { + this._console.info('ArcGIS getting newest observations for event ' + pair.event.name); + const obsRepo = await this._obsRepos(pair.event.id); + const pagingSettings = { + pageSize: config.batchSize, + pageIndex: 0, + includeTotalCount: true + } + let morePages = true; + let numberLeft = 0; + while (morePages) { + numberLeft = await this.queryAndSend(config, pair.featureLayerProcessors, obsRepo, pagingSettings, numberLeft); + morePages = numberLeft > 0; + } + } + + for (const layerProcessor of this._layerProcessors) { + layerProcessor.lastTimeStamp = nextQueryTime; + } + + this._firstRun = false; + + // ArcGISIndentityManager access tokens may have been updated check and save + this._identityService.updateIndentityManagers() + } + this.scheduleNext(config); + } + } + + private scheduleNext(config: ArcGISPluginConfig) { + if (this._isRunning) { + let interval = config.intervalSeconds; + if (this._firstRun && config.featureServices.length > 0) { + interval = config.startupIntervalSeconds; + } else { + for (const layerProcessor of this._layerProcessors) { + if (layerProcessor.hasPendingUpdates()) { + interval = config.updateIntervalSeconds; + break; + } + } + } + this._nextTimeout = setTimeout(() => { this.processAndScheduleNext() }, interval * 1000); + } + } + + /** + * Queries for new observations and sends them to any configured arc servers. + * @param config The plugin configuration. + * @param layerProcessors The layer processors to use when processing arc objects. + * @param obsRepo The observation repo for an event. + * @param pagingSettings Current paging settings. + * @param numberLeft The number of observations left to query and send to arc. + * @returns The number of observations still needing to be queried and sent to arc. + */ + private async queryAndSend(config: ArcGISPluginConfig, layerProcessors: FeatureLayerProcessor[], obsRepo: EventScopedObservationRepository, pagingSettings: PagingParameters, numberLeft: number): Promise { + let newNumberLeft = numberLeft; + + let queryTime = -1; + for (const layerProcessor of layerProcessors) { + if (queryTime == -1 || layerProcessor.lastTimeStamp < queryTime) { + queryTime = layerProcessor.lastTimeStamp; + } + } + + let latestObs = await obsRepo.findLastModifiedAfter(queryTime, pagingSettings); + if (latestObs != null && latestObs.totalCount != null && latestObs.totalCount > 0) { + if (pagingSettings.pageIndex == 0) { + this._console.info('ArcGIS newest observation count ' + latestObs.totalCount); + newNumberLeft = latestObs.totalCount; + } + const observations = latestObs.items + const mageEvent = await this._eventRepo.findById(obsRepo.eventScope) + const eventTransform = new EventTransform(config, mageEvent) + const arcObjects = new ArcObjects() + this._geometryChangeHandler.checkForGeometryChange(observations, arcObjects, layerProcessors, this._firstRun); + for (let i = 0; i < observations.length; i++) { + const observation = observations[i] + let deletion = false + if (observation.states.length > 0) { + deletion = observation.states[0].name.startsWith('archive') + } + if (deletion) { + const arcObservation = this._transformer.createObservation(observation) + arcObjects.deletions.push(arcObservation) + } else { + let user = null + if (observation.userId != null) { + user = await this._userRepo.findById(observation.userId) + } + const arcObservation = this._transformer.transform(observation, eventTransform, user) + arcObjects.add(arcObservation) + } + } + arcObjects.firstRun = this._firstRun; + for (const layerProcessor of layerProcessors) { + layerProcessor.processArcObjects(JSON.parse(JSON.stringify(arcObjects))); + } + newNumberLeft -= latestObs.items.length; + pagingSettings.pageIndex++; + } else { + this._console.info('ArcGIS no new observations') + } + + return newNumberLeft; + } } \ No newline at end of file diff --git a/plugins/arcgis/service/src/index.ts b/plugins/arcgis/service/src/index.ts index 657e21e51..10ede0275 100644 --- a/plugins/arcgis/service/src/index.ts +++ b/plugins/arcgis/service/src/index.ts @@ -4,14 +4,12 @@ import { ObservationRepositoryToken } from '@ngageoint/mage.service/lib/plugins. import { MageEventRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.events' import { UserRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.users' import { SettingPermission } from '@ngageoint/mage.service/lib/entities/authorization/entities.permissions' -import { ArcGISPluginConfig } from './ArcGISPluginConfig' -import { AuthType } from './ArcGISConfig' import { ObservationProcessor } from './ObservationProcessor' import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" -import { FeatureServiceConfig, OAuthAuthConfig } from './ArcGISConfig' +import { FeatureServiceConfig } from './ArcGISConfig' import { URL } from "node:url" import express from 'express' -import { getIdentityManager, getPortalUrl } from './ArcGISIdentityManagerFactory' +import { createArcGISIdentityService, getPortalUrl } from './ArcGISService' const logPrefix = '[mage.arcgis]' const logMethods = ['log', 'debug', 'info', 'warn', 'error'] as const @@ -37,17 +35,9 @@ const InjectedServices = { const pluginWebRoute = "plugins/@ngageoint/mage.arcgis.service" -const sanitizeFeatureService = (config: FeatureServiceConfig, type: AuthType): FeatureServiceConfig => { - if (type === AuthType.OAuth) { - const newAuth = Object.assign({}, config.auth) as OAuthAuthConfig; - delete newAuth.refreshToken; - delete newAuth.refreshTokenExpires; - return { - ...config, - auth: newAuth - } - } - return config; +const sanitizeFeatureService = (config: FeatureServiceConfig): Omit => { + const { identityManager, ...sanitized } = config; + return sanitized; } /** @@ -65,11 +55,9 @@ const arcgisPluginHooks: InitPluginHook = { init: async (services): Promise => { console.info('Intializing ArcGIS plugin...') const { stateRepo, eventRepo, obsRepoForEvent, userRepo } = services - // TODO - // - Update layer token to get token from identity manager - // - Move plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.ts addLayer to helper file and use instead of encodeURIComponent - const processor = new ObservationProcessor(stateRepo, eventRepo, obsRepoForEvent, userRepo, console) + const identityService = createArcGISIdentityService(stateRepo) + const processor = new ObservationProcessor(stateRepo, eventRepo, obsRepoForEvent, userRepo, identityService, console) processor.start() return { webRoutes: { @@ -117,27 +105,24 @@ const arcgisPluginHooks: InitPluginHook = { ArcGISIdentityManager.exchangeAuthorizationCode(creds, code).then(async (idManager: ArcGISIdentityManager) => { let service = config.featureServices.find(service => service.url === state.url) if (!service) { - service = { url: state.url, layers: [] } - config.featureServices.push(service) + service = { + url: state.url, + identityManager: idManager.serialize(), + layers: [] + } + } else { + service.identityManager = idManager.serialize() } - service.auth = { - ...service.auth, - type: AuthType.OAuth, - clientId: state.clientId, - authToken: idManager.token, - authTokenExpires: idManager.tokenExpires.getTime(), - refreshToken: idManager.refreshToken, - refreshTokenExpires: idManager.refreshTokenExpires.getTime() - } + config.featureServices.push(service) await processor.putConfig(config) - // TODO: This seems like a bad idea to send the access tokens to the front end. It has no use for them and could potentially be a security concern + const sanitizedService = sanitizeFeatureService(service) res.send(` @@ -163,14 +148,26 @@ const arcgisPluginHooks: InitPluginHook = { .get(async (req, res, next) => { console.info('Getting ArcGIS plugin config...') const config = await processor.safeGetConfig() - config.featureServices = config.featureServices.map((service) => sanitizeFeatureService(service, AuthType.OAuth)); - res.json(config) + const { featureServices, ...remaining } = config + res.json({ ...remaining, featureServices: featureServices.map((service) => sanitizeFeatureService(service)) }) }) .put(async (req, res, next) => { console.info('Applying ArcGIS plugin config...') - const arcConfig = req.body as ArcGISPluginConfig - const configString = JSON.stringify(arcConfig) - processor.patchConfig(arcConfig) + const config = await stateRepo.get() + const { featureServices: updatedServices, ...updateConfig } = req.body + + // Map exisiting identityManager, client does not send this + const featureServices: FeatureServiceConfig[] = updatedServices.map((updateService: FeatureServiceConfig) => { + const existingService = config.featureServices.find((featureService: FeatureServiceConfig) => featureService.url === updateService.url) + return { + url: updateService.url, + layers: updateService.layers, + identityManager: existingService?.identityManager + } + }) + + await stateRepo.patch({ ...updateConfig, featureServices }) + res.sendStatus(200) }) @@ -183,25 +180,32 @@ const arcgisPluginHooks: InitPluginHook = { } let service: FeatureServiceConfig + let identityManager: ArcGISIdentityManager if (token) { - service = { url, layers: [], auth: { type: AuthType.Token, token } } + identityManager = await ArcGISIdentityManager.fromToken({ + token: token + }) + service = { url, layers: [], identityManager: identityManager.serialize() } } else if (username && password) { - service = { url, layers: [], auth: { type: AuthType.UsernamePassword, username, password } } + identityManager = await ArcGISIdentityManager.signIn({ + username: auth?.username, + password: auth?.password, + portal: getPortalUrl(url) + }) + service = { url, layers: [], identityManager: identityManager.serialize() } } else { return res.sendStatus(400) } try { - // Create the IdentityManager instance to validate credentials - await getIdentityManager(service) + // TODO can you validate existing services? let existingService = config.featureServices.find(service => service.url === url) - if (existingService) { - existingService = { ...existingService } - } else { + if (!existingService) { config.featureServices.push(service) } + await processor.patchConfig(config) - return res.send(sanitizeFeatureService(service, AuthType.OAuth)) + return res.send(sanitizeFeatureService(service)) } catch (err) { return res.send('Invalid credentials provided to communicate with feature service').status(400) } @@ -216,7 +220,7 @@ const arcgisPluginHooks: InitPluginHook = { } try { - const identityManager = await getIdentityManager(featureService) + const identityManager = await identityService.getIdentityManager(featureService) const response = await request(url, { authentication: identityManager })