Adding in tiled OSM data insert and server.

Author: iandees

insert_tiled_osm_data.py

@@ -0,0 +1,160 @@
+"""This program parses an OSM XML file and inserts the data in a
+MongoDB database"""
+
+import sys
+import os
+import time
+import pymongo
+from datetime import datetime
+from xml.sax import make_parser
+from xml.sax.handler import ContentHandler
+from pymongo import Connection
+
+from globalmaptiles import GlobalMercator
+
+class OsmHandler(ContentHandler):
+    """Base class for parsing OSM XML data"""
+    def __init__(self, client):
+        self.proj = GlobalMercator()
+        self.nodeRecords = []
+        self.wayRecords = []
+        self.relationRecords = []
+        self.record = {}
+        self.nodeLocations = {}
+        self.client = client
+        
+        self.stats = {'nodes': 0, 'ways': 0, 'relations': 0}
+        self.lastStatString = ""
+        self.statsCount = 0
+
+    def writeStatsToScreen(self):
+        for char in self.lastStatString:
+            sys.stdout.write('\b')
+        self.lastStatString = "%d nodes, %d ways, %d relations" % (self.stats['nodes'],
+                                                                     self.stats['ways'],
+                                                                     self.stats['relations'])
+        sys.stdout.write(self.lastStatString)
+
+    def fillDefault(self, attrs):
+        """Fill in default record values"""
+        self.record['_id'] = int(attrs['id'])
+        self.record['ts'] = self.isoToTimestamp(attrs['timestamp'])
+        self.record['tg'] = [] 
+        if attrs.has_key('user'):
+            self.record['u'] = attrs['user']
+        if attrs.has_key('uid'):
+            self.record['uid'] = int(attrs['uid'])
+        if attrs.has_key('version'):
+            self.record['v'] = int(attrs['version'])
+        if attrs.has_key('changeset'):
+            self.record['c'] = int(attrs['changeset'])
+
+    def isoToTimestamp(self, isotime):
+        """Parse a date and return a time tuple"""
+        t = datetime.strptime(isotime, "%Y-%m-%dT%H:%M:%SZ")
+        return time.mktime(t.timetuple())
+
+    def quadKey(self, lat, lon, zoom):
+        (mx, my) = self.proj.LatLonToMeters(lat, lon)
+        (tx, ty) = self.proj.MetersToTile(mx, my, zoom)
+        return self.proj.QuadTree(tx, ty, zoom)
+
+    def startElement(self, name, attrs):
+        """Parse the XML element at the start"""
+        if name == 'node':
+            self.fillDefault(attrs)
+            self.record['loc'] = {'lat': float(attrs['lat']),
+                                  'lon': float(attrs['lon'])}
+            self.record['qk'] = self.quadKey(float(attrs['lat']), float(attrs['lon']), 17)
+            self.nodeLocations[self.record['_id']] = self.record['qk']
+        elif name == 'changeset':
+            self.fillDefault(attrs)
+        elif name == 'tag':
+            k = attrs['k']
+            v = attrs['v']
+            # MongoDB doesn't let us have dots in the key names.
+            #k = k.replace('.', ',,')
+            self.record['tg'].append((k, v))
+        elif name == 'way':
+            self.fillDefault(attrs)
+            self.record['n'] = []
+            self.record['loc'] = []
+        elif name == 'relation':
+            self.fillDefault(attrs)
+            self.record['m'] = []
+        elif name == 'nd':
+            ref = int(attrs['ref'])
+            self.record['n'].append(ref)
+            refLoc = self.nodeLocations[ref]
+            if refLoc not in self.record['loc']:
+                self.record['loc'].append(refLoc)
+        elif name == 'member':
+            ref = int(attrs['ref'])
+            member = {'type': attrs['type'],
+                      'ref':  ref,
+                      'role': attrs['role']}
+            self.record['m'].append(member)
+            
+            if attrs['type'] == 'way':
+                ways2relations = self.client.osm.ways.find_one({ '_id' : ref})
+                if ways2relations:
+                    if 'relations' not in ways2relations:
+                        ways2relations['relations'] = []
+                    ways2relations['relations'].append(self.record['_id'])
+                    self.client.osm.ways.save(ways2relations)
+            elif attrs['type'] == 'node':
+                nodes2relations = self.client.osm.nodes.find_one({ '_id' : ref})
+                if nodes2relations:
+                    if 'relations' not in nodes2relations:
+                        nodes2relations['relations'] = []
+                    nodes2relations['relations'].append(self.record['_id'])
+                    self.client.osm.nodes.save(nodes2relations)
+        
+    def endElement(self, name):
+        """Finish parsing an element
+        (only really used with nodes, ways and relations)"""
+        if name == 'node':
+            self.nodeRecords.append(self.record)
+            if len(self.nodeRecords) > 1500:
+                self.client.osm.nodes.insert(self.nodeRecords)
+                self.nodeRecords = []
+                self.writeStatsToScreen()
+            self.record = {}
+            self.stats['nodes'] = self.stats['nodes'] + 1
+        elif name == 'way':
+            # Clean up any existing nodes
+            if len(self.nodeRecords) > 0:
+                self.client.osm.nodes.insert(self.nodeRecords)
+                self.nodeRecords = []
+
+            self.wayRecords.append(self.record)
+            if len(self.wayRecords) > 100:
+                self.client.osm.ways.insert(self.wayRecords)
+                self.wayRecords = []
+                self.writeStatsToScreen()
+            self.record = {}
+            self.stats['ways'] = self.stats['ways'] + 1
+        elif name == 'relation':
+            self.client.osm.relations.save(self.record)
+            self.record = {}
+            self.statsCount = self.statsCount + 1
+            if self.statsCount > 10:
+                self.writeStatsToScreen()
+                self.statsCount = 0
+            self.stats['relations'] = self.stats['relations'] + 1
+
+if __name__ == "__main__":
+    filename = sys.argv[1]
+
+    if not os.path.exists(filename):
+        print "Path %s doesn't exist." % (filename)
+        sys.exit(-1)
+
+    client = Connection()
+    parser = make_parser()
+    handler = OsmHandler(client)
+    parser.setContentHandler(handler)
+    parser.parse(open(filename))
+    client.disconnect()
+
+    print

tile_server.py

@@ -0,0 +1,180 @@
+import pymongo
+from pymongo import Connection
+from xml.sax.saxutils import escape
+import re
+
+from globalmaptiles import GlobalMercator
+
+class OsmApi:
+    def __init__(self):
+        self.client = Connection()
+        self.proj = GlobalMercator()
+
+    def getTile(self, zoom, x, y):
+        (x, y) = self.proj.GoogleTile(x,y,zoom)
+        quadkey = self.proj.QuadTree(x,y,zoom)
+        print "Querying for %s." % (quadkey,)
+        (minlat, minlon, maxlat, maxlon) = self.proj.TileLatLonBounds(x,y,zoom)
+
+        # Nodes in the tile
+        nodes = {}
+        cursor = self.client.osm.nodes.find({'qk': {'$regex': "^%s" % (quadkey,)} })
+        for row in cursor:
+            nodes[row['_id']] = row
+
+        # Ways with nodes in the tile
+        ways = {}
+        cursor = self.client.osm.ways.find({'loc': {'$regex': "^%s" % (quadkey,)} })
+        for row in cursor:
+            ways[row['_id']] = row
+
+        # Nodes on ways that extend beyond the bounding box
+        otherNids = set()
+        for way in ways.values():
+            for nid in way['nodes']:
+                otherNids.add(nid)
+        cursor = self.client.osm.nodes.find({'_id': {'$in': list(otherNids)} })
+        for row in cursor:
+            nodes[row['_id']] = row
+
+        # Relations that contain any of the above as members
+        relations = {}
+
+        # Sort the results by id
+        nodes = sorted(nodes.iteritems())
+        ways = sorted(ways.iteritems())
+        relations = sorted(relations.iteritems())
+
+        doc = {'bounds': {'minlat': minlat,
+                          'minlon': minlon,
+                          'maxlat': maxlat,
+                          'maxlon': maxlon},
+               'nodes': nodes,
+               'ways': ways,
+               'relations': relations}
+
+        return doc
+
+class OsmXmlOutput:
+    def addNotNullAttr(self, mappable, mappableElement, name, outName=None):
+        if not outName:
+            outName = name
+        if name in mappable:
+            mappableElement.setAttribute(escape(outName), escape(unicode(mappable[name])))
+
+    def defaultAttrs(self, mappableElement, mappable):
+        self.addNotNullAttr(mappable, mappableElement, "_id", "id")
+        self.addNotNullAttr(mappable, mappableElement, "v", "version")
+        self.addNotNullAttr(mappable, mappableElement, "user")
+
+    def tagNodes(self, doc, mappableElement, mappable):
+        for mappable in mappable['tags']:
+            k,v = mappable
+            tagElement = doc.createElement("tag")
+            tagElement.setAttribute("k", k)
+            tagElement.setAttribute("v", v)
+            mappableElement.appendChild(tagElement)
+
+    def iter(self, data):
+        from xml.dom.minidom import Document
+        doc = Document()
+
+        yield '<osm generator="%s" version="%s">\n' % ("tiled mongosm 0.1", "0.6")
+
+        if 'bounds' in data:
+            yield '<bounds minlat="%s" minlon="%s" maxlat="%s" maxlon="%s"/>\n' % (
+                    str(data['bounds']['minlat']),
+                    str(data['bounds']['minlon']),
+                    str(data['bounds']['maxlat']),
+                    str(data['bounds']['maxlon']))
+
+        if 'nodes' in data:
+            for (id, node) in data['nodes']:
+                nodeElem = doc.createElement("node")
+                nodeElem.setAttribute("lat", str(node['loc']['lat']))
+                nodeElem.setAttribute("lon", str(node['loc']['lon']))
+                self.defaultAttrs(nodeElem, node)
+                self.tagNodes(doc, nodeElem, node)
+                yield "%s\n" % (nodeElem.toxml('UTF-8'),)
+
+        if 'ways' in data:
+            for (id, way) in data['ways']:
+                wayElem = doc.createElement("way")
+                self.defaultAttrs(wayElem, way)
+                self.tagNodes(doc, wayElem, way)
+                for ref in way['nodes']:
+                    refElement = doc.createElement("nd")
+                    refElement.setAttribute("ref", str(ref))
+                    wayElem.appendChild(refElement)
+                yield "%s\n" % (wayElem.toxml('UTF-8'),)
+
+        if 'relations' in data:
+            for (id, relation) in data['relations']:
+                relationElem = doc.createElement("relation")
+                self.defaultAttrs(relationElem, relation)
+                self.tagNodes(doc, relationElem, relation)
+                for member in relation['members']:
+                    memberElem = doc.createElement("member")
+                    memberElem.setAttribute("type", member['type'])
+                    memberElem.setAttribute("ref", str(member['ref']))
+                    memberElem.setAttribute("role", member['role'])
+                    relationElem.appendChild(memberElem)
+                yield "%s\n" % (relationElem.toxml('UTF-8'),)
+
+        yield '</osm>\n'
+
+import time, sys
+import os
+import urlparse
+from werkzeug.wrappers import Request, Response
+from werkzeug.routing import Map, Rule
+from werkzeug.exceptions import HTTPException, NotFound
+
+class Mongosm(object):
+
+    def tileRequest(self, request, zoom, x, y):
+        #(minlon, minlat, maxlon, maxlat) = request.args['bbox'].split(',')
+        #bbox = [[float(minlat), float(minlon)],[float(maxlat), float(maxlon)]]
+
+        api = OsmApi()
+        data = api.getTile(int(zoom), int(x), int(y))
+
+        outputter = OsmXmlOutput()
+
+        return Response(outputter.iter(data), content_type='text/xml', direct_passthrough=True)
+
+    def capabilitiesRequest(self, request):
+        return Response("""
+            <osm version="0.6" generator="mongosm 0.1">
+                <api>
+                    <version minimum="0.6" maximum="0.6"/>
+                    <area maximum="0.5"/>
+                </api>
+            </osm>""")
+
+    def __init__(self):
+        self.url_map = Map([
+            Rule('/tiles/0.6/<zoom>/<x>/<y>', endpoint='tileRequest'),
+            Rule('/api/capabilities', endpoint='capabilitiesRequest'),
+        ])
+
+    def dispatch_request(self, request):
+        adapter = self.url_map.bind_to_environ(request.environ)
+        try:
+            endpoint, values = adapter.match()
+            return getattr(self, endpoint)(request, **values)
+        except HTTPException, e:
+            return e
+
+    def wsgi_app(self, environ, start_response):
+        request = Request(environ)
+        response = self.dispatch_request(request)
+        return response(environ, start_response)
+
+    def __call__(self, environ, start_response):
+        return self.wsgi_app(environ, start_response)
+
+if __name__ == '__main__':
+    from werkzeug.serving import run_simple
+    app = Mongosm()
+    run_simple('0.0.0.0', 5000, app, use_debugger=True, use_reloader=True)