diff --git a/lib/createGeoIndex.js b/lib/createGeoIndex.js index 898436db..eee84a6f 100644 --- a/lib/createGeoIndex.js +++ b/lib/createGeoIndex.js @@ -7,8 +7,6 @@ var _ = require('lodash'), mkdirp = require('mkdirp'), polygon = require('turf-polygon') -var DATA_DIR = './data' - var geoJsonReader = new jsts.io.GeoJSONReader(), geoJsonWriter = new jsts.io.GeoJSONWriter() @@ -34,35 +32,8 @@ var intersection = function(a, b) { var result = _a.intersection(_b) - try { - result = geoJsonWriter.write(result); - } catch(e) { - console.log('error interseting') - async.parallel([ - function(cb) { - var filePath = DATA_DIR + '/err-a.json', - writeStream = fs.createWriteStream(filePath) - - writeStream.end(JSON.stringify(a, null, 2), function(err) { - if(err) { throw err } - cb() - }) - }, function(cb) { - var filePath = DATA_DIR + '/err-b.json', - writeStream = fs.createWriteStream(filePath) - - writeStream.end(JSON.stringify(b, null, 2), function(err) { - if(err) { throw err } - cb() - }) - }], function() { - throw e - }) + result = geoJsonWriter.write(result); - return 'error' - - - } if(result.type === 'GeometryCollection' && result.geometries.length === 0) { return undefined } else { @@ -74,7 +45,7 @@ var intersection = function(a, b) { } } -module.exports = function(tzGeojson, callback) { +module.exports = function(tzGeojson, dataDir, callback) { console.log('indexing') @@ -129,7 +100,7 @@ module.exports = function(tzGeojson, callback) { cb() }) }) - }, 2) + }, 10) // create array and index lookup of timezone names for(i = 0; i < tzGeojson.features.length; i++) { @@ -142,29 +113,32 @@ module.exports = function(tzGeojson, callback) { expectedAtLevel = 4, curZones = [{ id: 'a', - bounds: [0, 0, 180, 90] + bounds: [0, 0, 179.9999, 89.9999] }, { id: 'b', - bounds: [-180, 0, 0, 90], + bounds: [-179.9999, 0, 0, 89.9999], }, { id: 'c', - bounds: [-180, -90, 0, 0], + bounds: [-179.9999, -89.9999, 0, 0], }, { id: 'd', - bounds: [0, -90, 180, 0] - }] + bounds: [0, -89.9999, 179.9999, 0] + }], + printMod - while(curPctIndexed < 0.90 && curZones.length < 40) { + while(curPctIndexed < 0.99) { var nextZones = [] console.log('*********************************************') console.log('level', curLevel, ' pct Indexed: ', curPctIndexed) console.log('*********************************************') - for(i = 0; i < curZones.length; i++) { + printMod = Math.round(curZones.length / 5) - if(i % 1000 == 0) { - console.log('inspecting index area ', i + 1, ' of ', curZones.length) + for (i = curZones.length - 1; i >= 0; i--) { + + if(i % printMod == 0) { + console.log('inspecting index area ', curZones.length - i, ' of ', curZones.length) } var curZone = curZones[i], @@ -177,17 +151,6 @@ module.exports = function(tzGeojson, callback) { [curBounds[0], curBounds[1]] ]]).geometry - var subZones = [], - debug = polygon([[ - [curBounds[0], curBounds[1]], - [curBounds[0], curBounds[3]], - [curBounds[2], curBounds[3]], - [curBounds[2], curBounds[1]], - [curBounds[0], curBounds[1]] - ]]) - - subZones.push(debug) - // calculate intersection with timezone boundaries var timezonesToInspect = [] @@ -196,7 +159,7 @@ module.exports = function(tzGeojson, callback) { timezonesToInspect = curZone.tzs } else { // first iteration, find all intersections in world - for (j = 0; j < tzGeojson.features.length; j++) { + for (var j = tzGeojson.features.length - 1; j >= 0; j--) { timezonesToInspect.push(j) } } @@ -205,13 +168,6 @@ module.exports = function(tzGeojson, callback) { intersectedZones = result.intersectedZones, foundExactMatch = result.foundExactMatch - /*for (j = 0; j < timezonesToInspect.length; j++) { - var zIntersect = intersection(tzGeojson.features[timezonesToInspect[j]].geometry, curBoundsGeoJson) - if(zIntersect) { - subZones.push(zIntersect) - } - }*/ - var zoneResult = -1 // defaults to no zones found // check the results @@ -279,12 +235,6 @@ module.exports = function(tzGeojson, callback) { _.set(data.lookup, curZone.id, zoneResult) - /*fileWritingQueue.push({ - folder: DATA_DIR, - filename: 'level' + curLevel + '-sz-' + i + '.json', - data: featurecollection(subZones), - })*/ - } // recalculate pct indexed after this round @@ -294,14 +244,18 @@ module.exports = function(tzGeojson, callback) { curLevel++ } + console.log('*********************************************') console.log('reached target index: ', curPctIndexed) + console.log('writing unindexable zone data') - var allSubZones = [] + printMod = Math.round(curZones.length / 5) // process remaining zones and write out individual geojson for each small region - for(i = 0; i < curZones.length; i++) { + for (i = curZones.length - 1; i >= 0; i--) { - console.log('writing zone data ', i ,'of', curZones.length) + if(i % printMod == 0) { + console.log('inspecting unindexable area ', curZones.length - i, ' of ', curZones.length) + } var curZone = curZones[i], curBounds = curZone.bounds, @@ -312,15 +266,14 @@ module.exports = function(tzGeojson, callback) { [curBounds[2], curBounds[1]], [curBounds[0], curBounds[1]] ]]).geometry - errFound = false - console.log(curZone.id) + //console.log('writing zone data `', curZone.id, '`', i ,'of', curZones.length) var result = inspectZones(curZone.tzs, curBoundsGeoJson), intersectedZones = result.intersectedZones, foundExactMatch = result.foundExactMatch - console.log('intersectedZones', intersectedZones.length, 'exact:', foundExactMatch) + //console.log('intersectedZones', intersectedZones.length, 'exact:', foundExactMatch) var zoneResult = -1 // defaults to no zones found @@ -334,27 +287,18 @@ module.exports = function(tzGeojson, callback) { for (j = intersectedZones.length - 1; j >= 0; j--) { var tzIdx = intersectedZones[j] - console.log('intersecting', tzGeojson.features[tzIdx].properties) + //console.log('intersecting', tzGeojson.features[tzIdx].properties) var intersectedArea = intersection(tzGeojson.features[tzIdx].geometry, curBoundsGeoJson) - if(intersectedArea === 'error') { - errFound = true - break + if(intersectedArea) { + intersectedArea.properties.TZID = data.timezones[tzIdx] + features.push(intersectedArea) } - - intersectedArea.properties.TZID = data.timezones[tzIdx] - features.push(intersectedArea) - allSubZones.push(intersectedArea) - } - - if(errFound) { - break } var areaGeoJson = featurecollection(features), - path = './data/' + curZone.id.replace(/\./g, '/') - + path = dataDir + '/' + curZone.id.replace(/\./g, '/') fileWritingQueue.push({ folder: path, filename: 'geo.json', data: areaGeoJson }) @@ -369,12 +313,11 @@ module.exports = function(tzGeojson, callback) { console.log('writing index file') fileWritingQueue.drain = function(err) { - console.log('drained') + console.log('done indexing') callback(err) } // write index data to file - fileWritingQueue.push({ folder: DATA_DIR, filename: 'index.json', data: data }) - fileWritingQueue.push({ folder: DATA_DIR, filename: 'finalSubzones.json', data: featurecollection(allSubZones) }) + fileWritingQueue.push({ folder: dataDir, filename: 'index.json', data: data }) } \ No newline at end of file diff --git a/lib/find.js b/lib/find.js index 018d9256..1dc529f2 100644 --- a/lib/find.js +++ b/lib/find.js @@ -77,7 +77,6 @@ var getTimezone = function(lat, lon) { // analyze result of current depth if(curTzData === 'f') { - console.log('looking up tz from file', quadPos) // exact boundaries saved in file // parse geojson for exact boundaries var filepath = quadPos.split('').join('/'), @@ -92,13 +91,11 @@ var getTimezone = function(lat, lon) { // not within subarea, therefore no valid timezone return null } else if(curTzData === -1) { - console.log('no timezone at index') // no timezone at this gps location return null } else if(typeof curTzData === 'number') { // exact match found - console.log('exact match at index') - return tzData.timezones[curTzData[nextQuad]] + return tzData.timezones[curTzData] } else if(typeof curTzData !== 'object') { // not another nested quad index, throw error var err = new Error('Unexpected data type') diff --git a/lib/update.js b/lib/update.js index 165d36ab..6c30c418 100644 --- a/lib/update.js +++ b/lib/update.js @@ -3,14 +3,13 @@ var fs = require('fs') var async = require('async'), Download = require('download'), downloadStatus = require('download-status'), + rimraf = require('rimraf'), shp = require('shpjs') var indexGeoJSON = require('./createGeoIndex.js') var MASTER_DL_URL = 'http://efele.net/maps/tz/world/tz_world_mp.zip', - MASTER_DL_SHA_URL = 'http://efele.net/maps/tz/world/tz_world_mp.zip.sha1', - MASTER_LOCAL_SHA_FILE = './data/tz_world_mp.zip.sha1', - OUTPUT_FILE = './data/tzgeo.json' + MASTER_DL_SHA_URL = 'http://efele.net/maps/tz/world/tz_world_mp.zip.sha1' /** * Copied from http://stackoverflow.com/a/12101012/269834 @@ -27,7 +26,7 @@ var toArrayBuffer = function(buffer) { var extractToGeoJson = function(callback) { shp(toArrayBuffer(fs.readFileSync('./downloads/tz_shp.zip'))) .then(function(geojson) { callback(null, geojson) }) - .catch(function(e){ throw e }) + .catch(function(e){ callback(e) }) } var downloadFile = function(url, rename, callback) { @@ -39,18 +38,15 @@ var downloadFile = function(url, rename, callback) { .run(callback) } -var updateShaFile = function(callback) { - fs.unlink(MASTER_LOCAL_SHA_FILE, function(err) { - if(err) { - return callback(err) - } - fs.rename('./downloads/file.sha1', MASTER_LOCAL_SHA_FILE, callback) - }) +var makeReadShaFileFn = function(filename) { + return function(results, cb) { + fs.readFile(filename, 'utf-8', cb ? cb : results) + } } module.exports = function(cfg, callback) { - var mainUrl, shaUrl + var mainUrl, shaUrl, dataDir, masterLocalShaFile if(!callback) { if(typeof cfg === 'function') { @@ -67,52 +63,52 @@ module.exports = function(cfg, callback) { shaUrl = cfg.shaUrl || MASTER_DL_SHA_URL } - indexGeoJSON(require('../data/tzgeo.json'), callback) - - // download sha file to see if master file is updated - /*downloadFile(shaUrl, 'file.sha1', function(err) { - - if(err) { + dataDir = cfg.dataDir || './data' + masterLocalShaFile = dataDir + '/tz_world_mp.zip.sha1' + + async.auto({ + // download sha file to see if master file is updated + downloadSha: function(cb) { + downloadFile(shaUrl, 'file.sha1', cb) + }, + // read local sha file + readLocalSha: makeReadShaFileFn(masterLocalShaFile), + // read downloaded sha file + readDownloadedSha: ['downloadSha', makeReadShaFileFn('./downloads/file.sha1')], + comparseShaFiles: ['readLocalSha', 'readDownloadedSha', function(results, cb) { + if(results.readLocalSha === results.readDownloadedSha) { + var message = 'remote file same as local, no need to download' + console.log(message) + return cb(message) + } + cb() + }], + unlinkLocalShaFile: ['comparseShaFiles', function(results, cb) { + fs.unlink(masterLocalShaFile, cb) + }], + updateShaFile: ['unlinkLocalShaFile', function(results, cb) { + fs.rename('./downloads/file.sha1', masterLocalShaFile, cb) + }], + deleteIndexFoldersAndFiles: ['comparseShaFiles', function(results, cb) { + async.each(['a', 'b', 'c', 'd', 'index.json'], function(fileOrFolder, eachCb) { + rimraf(dataDir + '/' + fileOrFolder, eachCb) + }, cb) + }], + downloadShapefile: ['comparseShaFiles', function(results, cb) { + downloadFile(mainUrl, 'tz_shp.zip', cb) + }], + extractShapefileToGeoJSON: ['downloadShapefile', function(results, cb) { + extractToGeoJson(cb) + }], + createIndex: ['deleteIndexFoldersAndFiles', 'extractShapefileToGeoJSON', function(results, cb) { + indexGeoJSON(results.extractShapefileToGeoJSON, dataDir, cb) + }] + + }, function(err, results) { + if(err && err !== 'remote file same as local, no need to download') { return callback(err) } + callback() + }) - async.map([MASTER_LOCAL_SHA_FILE, './downloads/file.sha1'], - function(file, mapCallback) { - fs.readFile(file, 'utf-8', mapCallback) - }, - function(err, results) { - if(err) { - return callback(err) - } - if(results[0] === results[1]) { - console.log('remote file same as local, no need to download') - callback() - } else { - // remote tz world shapefile is new - async.parallel([ - // download tz world shapefile - function(parallelCallback) { - downloadFile(mainUrl, 'tz_shp.zip', extractToGeoJson(function(err, geojson) { - if(err) { return parallelCallback(err) } - if(cfg.indexGeoJSON) { - indexGeoJSON(geojson, parallelCallback) - } else { - var writeStream = fs.createWriteStream(OUTPUT_FILE) - writeStream.end(JSON.stringify(geojson), function(err) { - if(err) { - return parallelCallback(err) - } - parallelCallback() - }) - } - } - )) - }, - // also update master sha file - updateShaFile - ], - callback) - } - }) - })*/ } \ No newline at end of file diff --git a/package.json b/package.json index 95ea5b1d..0f6724bd 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "codeclimate": "npm run cover && npm run codeclimate-send" }, "devDependencies": { - "async": "^1.5.2", + "async": "^2.0.0-rc.5", "chai": "^3.5.0", "codeclimate-test-reporter": "^0.3.1", "cross-env": "^1.0.7", @@ -34,7 +34,7 @@ "download-status": "^2.2.1", "fs-extra": "^0.30.0", "istanbul": "^0.4.2", - "jsts": "^1.1.1", + "jsts": "^1.1.2", "mkdirp": "^0.5.1", "mocha": "^2.4.5", "nock": "^8.0.0", diff --git a/scripts/updateData.js b/scripts/updateData.js index 9fd196fc..3ba3d416 100644 --- a/scripts/updateData.js +++ b/scripts/updateData.js @@ -1,6 +1,6 @@ var update = require('../lib/update.js') -update({ indexGeoJSON: true }, function(err) { +update(function(err) { if(err) { console.log('update unsuccessful. Error: ', err) } else {