Skip to content

Commit 56d7a9c

Browse files
QbertoQberto
committed
Merge remote-tracking branch 'refs/remotes/Esri/master'
2 parents 367f25f + 492c86a commit 56d7a9c

File tree

1 file changed

+4
-78
lines changed

1 file changed

+4
-78
lines changed

‎talks/uc2018/GeoAnalytics/GeoAnalytics_Aggregation_blog.ipynb‎

Lines changed: 4 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -224,104 +224,30 @@
224224
{
225225
"name": "stderr",
226226
"output_type": "stream",
227-
"text": [
228-
"Attaching log redirect\n",
229-
"Log level set to DEBUG\n"
230-
]
227+
"text": []
231228
},
232229
{
233230
"name": "stdout",
234231
"output_type": "stream",
235232
"text": [
236233
"Executing (CopyToDataStore): CopyToDataStore \"Record Set\" \"{\"serviceProperties\": {\"name\": \"CrimeDataset_38\", \"serviceUrl\": \"http://gpportal.esri.com/server/rest/services/Hosted/CrimeDataset_38/FeatureServer\"}, \"itemProperties\": {\"itemId\": \"15592be05cbc4b1a96512eab24414420\"}}\" \"{\"defaultAggregationStyles\": true}\"\n",
237234
"Start Time: Fri Jul 20 12:56:14 2018\n",
238-
"Attaching log redirect\n",
239-
"Log level set to DEBUG\n",
240-
"[DEBUG|12:56:14] Current Service Environment: GPServiceEnvironment(j3673f738ddcb4998806defc4db74bbcb,C:\\arcgis\\arcgisserver\\directories\\arcgisjobs\\system\\geoanalyticstools_gpserver\\j3673f738ddcb4998806defc4db74bbcb,Some(GPServiceRequest(Some(kXKoFor9uXQgSNDtx7eJj3o1oBSCOkyc0Qw4Dl5NSF0rihPqpW8cPLGDeJhIJzG4_KV-P7FVjVk8hZxcOLBVHBFhEMUlwnTQCjJss2wXEig.),None,Some(http))),System/GeoAnalyticsTools,GPServer,Map(maxAllowedComputeCoresPerJob -> 4, jobsVirtualDirectory -> /rest/directories/arcgisjobs, jobsDirectory -> C:\\arcgis\\arcgisserver\\directories\\arcgisjobs, virtualOutputDir -> /rest/directories/arcgisoutput, maxAllowedMemoryPerJobPerNode -> 6, showMessages -> Info, outputDir -> C:\\arcgis\\arcgisserver\\directories\\arcgisoutput, javaHeapSize -> 2048, maximumRecords -> 1000, toolbox -> ${AGSSERVER}/ArcToolBox/Services/GeoAnalytics Tools.tbx, executionType -> Asynchronous, renewTokens -> true, _debugModeEnabled -> true))\n",
241-
"[DEBUG|12:56:14] Running on GPPORTAL.ESRI.COM\n",
242-
"[DEBUG|12:56:16] Acquiring GAContext\n",
243-
"[DEBUG|12:56:16] Cached context is null, creating\n",
244-
"[DEBUG|12:56:16] Creating new context\n",
245-
"[DEBUG|12:56:16] Destroying any running contexts\n",
246-
"[DEBUG|12:56:16] Adding authentication info to SparkContext\n",
247-
"[DEBUG|12:56:16] \n",
248-
" spark.cores.max = 4,\n",
249-
" spark.executor.memory = 6g\n",
250-
" spark.dynamicAllocation.minExecutors = None\n",
251-
" spark.authenticate = Some(true)\n",
252-
" spark.ui.enabled = None\n",
253-
" spark.fileserver.port = Some(56540)\n",
254-
" spark.driver.port = Some(56541)\n",
255-
" spark.executor.port = Some(56542)\n",
256-
" spark.blockManager.port = Some(56543)\n",
257-
" \n",
258-
"[DEBUG|12:56:17] Initialized Spark 2.2.0\n",
259-
"[DEBUG|12:56:17] Attaching progress listener to SparkContext\n",
260-
"[DEBUG|12:56:17] Using status file 'C:\\arcgis\\arcgisserver\\directories\\arcgisjobs\\system\\geoanalyticstools_gpserver\\j3673f738ddcb4998806defc4db74bbcb\\status.dat' to track cancel\n",
261-
"[DEBUG|12:56:17] Executing function 'CopyToDataStore'\n",
262-
"[DEBUG|12:56:17] Client supplied query filter: fields=*;where=;extent=;interval=\n",
263-
"Using URL based GPRecordSet param: https://gpportal.esri.com/server/rest/services/DataStoreCatalogs/bigDataFileShares_pyTest/BigDataCatalogServer/ChicagoCrimes\n",
264-
"[DEBUG|12:56:17] Detected catalog server path (share=bigDataFileShares_pyTest,dataset=ChicagoCrimes)\n",
265-
"[DEBUG|12:56:17] Found data store ID: 7894077e-0250-4acd-8ee5-86586664d373\n",
266-
"[DEBUG|12:56:17] Using input handler: ManifestDataStoreLayerInputHandler\n",
267-
"[DEBUG|12:56:17] Loading data store factory\n",
268-
"[DEBUG|12:56:17] Attempting to load data store factory for '/bigDataFileShares/pyTest' of type 'fileShare'\n",
269-
"[DEBUG|12:56:17] Loading data store using [com.esri.arcgis.gae.ags.datastore.manifest.ManifestDataStoreFactory]\n",
270-
"[DEBUG|12:56:17] Initialized with qualified path '//qalab_server/pyunit/ArcGISOnline/GAX_bigDataFileShares/pyTest'\n",
271-
"[DEBUG|12:56:18] Loading dataset 'ChicagoCrimes' from datasource FileSystemDataSource\n",
272-
"[DEBUG|12:56:18] Loading file system dataset with path '//qalab_server/pyunit/ArcGISOnline/GAX_bigDataFileShares/pyTest/ChicagoCrimes'\n",
273-
"[DEBUG|12:56:18] Input layer has allocated 6 read task(s)\n",
274-
"[DEBUG|12:56:18] Input layer known upper bounds (countEstimate=271868,spatialExtent=N/A,temporalExtent=N/A)\n",
275-
"[DEBUG|12:56:18] Validating function inputs\n",
276-
"[DEBUG|12:56:18] Result service set to 'http://gpportal.esri.com/server/rest/services/Hosted/CrimeDataset_38/FeatureServer' (Portal item=15592be05cbc4b1a96512eab24414420)\n",
277-
"[DEBUG|12:56:18] Adding result layer\n",
278-
"[DEBUG|12:56:18] (6) WrappedFeatureSchemaRDD[4] at RDD at FeatureRDD.scala:38 []\n",
279-
" | (OperatorTrackFeatureIssues) BatchFeatureOperatorRDD[3] at RDD at BatchFeatureOperatorRDD.scala:17 []\n",
280-
" | WrappedFeatureSchemaRDD[2] at RDD at FeatureRDD.scala:38 []\n",
281-
" | MappedFieldsOverlayRDD[1] at RDD at MappedFieldsOverlayRDD.scala:18 []\n",
282-
" | ShapefileRDD[0] at RDD at ShapefileRDD.scala:35 []\n",
283-
"[DEBUG|12:56:18] Writing to managed data store (datastore=SpatiotemporalDataStore,dataset=gax2a2664922df7442b96f7cf01a4cb3512)\n",
284-
"[DEBUG|12:56:18] Discovered 1 ES machine(s). Setting result shard count to 1\n",
285-
"[DEBUG|12:56:18] Including default aggregation styles: [{\"style\":\"flatTriangle\",\"sr\":\"102100\",\"lods\":30},{\"style\":\"pointyTriangle\",\"sr\":\"102100\",\"lods\":30},{\"style\":\"square\",\"sr\":\"102100\",\"lods\":30}]\n",
286-
"[DEBUG|12:56:18] Creating data source: CreateRequest(gax2a2664922df7442b96f7cf01a4cb3512,esriGeometryPoint,BDSSchema([Lcom.esri.arcgis.bds.BDSField;@7c90b43b,Shape,INSTANT_DATETIME,null,null),0,1,60,esriTimeUnitsSeconds,true,false,true,0.0,ObjectId64Bit,Daily,0,1000,OBJECTID,globalid,-1,[Lcom.esri.arcgis.bds.EsriGeoHash;@157d6eac,false,1,Months,10000,false,50km,0.025,)\n",
287-
"[WARN|12:56:20] Write failures may occur because the number of potential concurrent writers is much larger than the number of nodes in the data store.\n",
288-
"{\"messageCode\":\"BD_101028\",\"message\":\"Starting new distributed job with 6 tasks.\",\"params\":{\"totalTasks\":\"6\"}}\n",
289-
"[DEBUG|12:56:21] [Spark] Job has 1 stages\n",
290-
"[DEBUG|12:56:21] [Spark] Submitted stage 'runJob at EsSpark.scala:107' with 6 tasks (attempt=0)\n",
291-
"{\"messageCode\":\"BD_101029\",\"message\":\"0/6 distributed tasks completed.\",\"params\":{\"completedTasks\":\"0\",\"totalTasks\":\"6\"}}\n",
292-
"[DEBUG|12:56:22] [Spark] Executor added (host=10.28.102.61,cores=4)\n",
293-
"[DEBUG|12:56:22] [Spark] Job has accepted first task and is no longer pending\n",
294-
"{\"messageCode\":\"BD_101029\",\"message\":\"1/6 distributed tasks completed.\",\"params\":{\"completedTasks\":\"1\",\"totalTasks\":\"6\"}}\n",
295-
"{\"messageCode\":\"BD_101029\",\"message\":\"3/6 distributed tasks completed.\",\"params\":{\"completedTasks\":\"3\",\"totalTasks\":\"6\"}}\n",
296-
"{\"messageCode\":\"BD_101029\",\"message\":\"5/6 distributed tasks completed.\",\"params\":{\"completedTasks\":\"5\",\"totalTasks\":\"6\"}}\n",
297-
"{\"messageCode\":\"BD_101029\",\"message\":\"6/6 distributed tasks completed.\",\"params\":{\"completedTasks\":\"6\",\"totalTasks\":\"6\"}}\n",
298-
"[DEBUG|12:58:29] [Spark] Completed stage 'runJob at EsSpark.scala:107' (attempt=0,gc-time=1724,disk-spill=0b,cpu-time=32656250000ns)\n",
299-
"[DEBUG|12:58:30] Updating metadata properties for gax2a2664922df7442b96f7cf01a4cb3512 after write\n",
300-
"[DEBUG|12:58:31] Results written in 131353ms (WriteResult(Some({\"xmin\":-87.93433083735326,\"ymin\":41.64472422641647,\"xmax\":-87.52468393341654,\"ymax\":42.022654058892584}),Some(Interval(MutableInstant(2014-01-01 00:00:00.000),MutableInstant(2014-12-31 23:58:00.000))),271868,0))\n",
301235
"{\"messageCode\":\"BD_101081\",\"message\":\"Finished writing results:\"}\n",
302236
"{\"messageCode\":\"BD_101082\",\"message\":\"* Count of features = 271868\",\"params\":{\"resultCount\":\"271868\"}}\n",
303237
"{\"messageCode\":\"BD_101083\",\"message\":\"* Spatial extent = {\\\"xmin\\\":-87.93433083735326,\\\"ymin\\\":41.64472422641647,\\\"xmax\\\":-87.52468393341654,\\\"ymax\\\":42.022654058892584}\",\"params\":{\"extent\":\"{\\\"xmin\\\":-87.93433083735326,\\\"ymin\\\":41.64472422641647,\\\"xmax\\\":-87.52468393341654,\\\"ymax\\\":42.022654058892584}\"}}\n",
304-
"{\"messageCode\":\"BD_101084\",\"message\":\"* Temporal extent = Interval(MutableInstant(2014-01-01 00:00:00.000),MutableInstant(2014-12-31 23:58:00.000))\",\"params\":{\"extent\":\"Interval(MutableInstant(2014-01-01 00:00:00.000),MutableInstant(2014-12-31 23:58:00.000))\"}}\n",
305-
"[DEBUG|12:58:31] Adding result layer definition\n"
238+
"{\"messageCode\":\"BD_101084\",\"message\":\"* Temporal extent = Interval(MutableInstant(2014-01-01 00:00:00.000),MutableInstant(2014-12-31 23:58:00.000))\",\"params\":{\"extent\":\"Interval(MutableInstant(2014-01-01 00:00:00.000),MutableInstant(2014-12-31 23:58:00.000))\"}}\n"
306239
]
307240
},
308241
{
309242
"name": "stderr",
310243
"output_type": "stream",
311-
"text": [
312-
"Detaching log redirect\n",
313-
"{\"messageCode\":\"BD_101051\",\"message\":\"Possible issues were found while reading 'inputLayer'.\",\"params\":{\"paramName\":\"inputLayer\"}}\n",
314-
"{\"messageCode\":\"BD_101054\",\"message\":\"Some records have either missing or invalid geometries.\"}\n"
315-
]
244+
"text": []
316245
},
317246
{
318247
"name": "stdout",
319248
"output_type": "stream",
320249
"text": [
321-
"[DEBUG|12:58:41] Running cleanup tasks for conditions (Success,Complete)\n",
322-
"[DEBUG|12:58:41] Running cleanup task [? @ DebugUtil.scala:53].\n",
323-
"Detaching log redirect\n",
324-
"{\"messageCode\":\"BD_101051\",\"message\":\"Possible issues were found while reading 'inputLayer'.\",\"params\":{\"paramName\":\"inputLayer\"}}\n",
250+
"{\"messageCode\":\"BD_101051\",\"message\":\"Possible issues were found while reading 'inputLayer'.\",\"params\":{\"paramName\":\"inputLayer\"}}\n",
325251
"{\"messageCode\":\"BD_101054\",\"message\":\"Some records have either missing or invalid geometries.\"}\n",
326252
"Succeeded at Fri Jul 20 12:58:41 2018 (Elapsed Time: 2 minutes 26 seconds)\n"
327253
]

0 commit comments

Comments
 (0)