Use MongoDB's 1.9 multi-location document indexing support to index ways.

Author: iandees

insert_osm_data.py

@@ -16,13 +16,12 @@ def __init__(self, client):
         self.records = []
         self.record = {}
         self.client = client
-        self.client.osm.nodes.ensure_index([('loc', pymongo.GEO2D),
-                                            ('id', pymongo.ASCENDING),
-                                            ('version', pymongo.DESCENDING)])
+        self.client.osm.nodes.ensure_index([('loc', pymongo.GEO2D)])
         self.client.osm.nodes.ensure_index([('id', pymongo.ASCENDING),
                                             ('version', pymongo.DESCENDING)])
         self.client.osm.ways.ensure_index([('id', pymongo.ASCENDING),
                                            ('version', pymongo.DESCENDING)])
+        self.client.osm.ways.ensure_index([('loc', pymongo.GEO2D)])
         self.client.osm.relations.ensure_index([('id', pymongo.ASCENDING),
                                                 ('version', pymongo.DESCENDING)])
     def fillDefault(self, attrs):
@@ -58,6 +57,11 @@ def startElement(self, name, attrs):
             k = k.replace('.', ',,')
             self.record['tags'][k] = attrs['v']
         elif name == 'way':
+            # Insert remaining nodes
+            if len(self.records) > 0:
+                self.client.osm.nodes.insert(self.records)
+                self.records = []
+
             self.fillDefault(attrs)
             self.record['nodes'] = []
         elif name == 'relation':
@@ -66,16 +70,6 @@ def startElement(self, name, attrs):
         elif name == 'nd':
             ref = long(attrs['ref'])
             self.record['nodes'].append(ref)
-
-            nodes2ways = self.client.osm.nodes.find_one({ 'id' : ref })
-            if nodes2ways:
-                if 'ways' not in nodes2ways:
-                    nodes2ways['ways'] = []
-                nodes2ways['ways'].append(self.record['id'])
-                self.client.osm.nodes.save(nodes2ways)
-            else:
-                print "Node %d ref'd by way %d not in file." % \
-                    (ref, self.record['id'])
         elif name == 'member':
             ref = long(attrs['ref'])
             member = {'type': attrs['type'],
@@ -108,7 +102,12 @@ def endElement(self, name):
                 self.records = []
             self.record = {}
         elif name == 'way':
-            self.client.osm.ways.save(self.record)
+            nodes = self.client.osm.nodes.find({ 'id': { '$in': self.record['nodes'] } }, { 'loc': 1, '_id': 0 })
+            self.record['loc'] = []
+            for node in nodes:
+                self.record['loc'].append(node['loc'])
+
+            self.client.osm.ways.insert(self.record, safe=True)
             self.record = {}
         elif name == 'relation':
             self.client.osm.relations.save(self.record)

map_server.py

@@ -15,20 +15,24 @@ def getNodesInBounds(self, box):
 
         return nodes
 
+    def getWaysInBounds(self, box):
+        cursor = self.client.osm.ways.find({'loc' : { '$within' : { '$box' : box } } }, { 'loc': 0 })
+        ways = []
+
+        for row in cursor:
+            ways.append(row)
+
+        return ways
+
+
     def getNodesFromWays(self, ways):
-        nodeIds = set()
+        nodeIds = set() 
 
         for way in ways:
             for nodeId in way['nodes']:
                 nodeIds.add(nodeId)
 
-        nodes = []
-        for nodeId in nodeIds:
-            node = self.client.osm.nodes.find_one({'id' : nodeId})
-            if node:
-                nodes.append(node)
-            else:
-                print "Error. couldn't find node id %d." % nodeId
+        nodes = self.client.osm.nodes.find({'id': {'$in': list(nodeIds)} })
 
         return nodes
 
@@ -118,20 +122,34 @@ def getRelationById(self, id):
     def getBbox(self, bbox):
         import time, sys
 
-        sys.stderr.write("<!-- Start %s -->\n" % time.time())
+        start = time.time()
+        
         nodes = self.getNodesInBounds(bbox)
-        sys.stderr.write("<!-- Get nodes %s -->\n" % time.time())
-        ways = self.getWaysFromNodes(nodes)
-        sys.stderr.write("<!-- Get ways %s -->\n" % time.time())
+
+        timeA = time.time()
+        sys.stderr.write("<!-- Get nodes in bbox %s -->\n" % (timeA - start))
+
+        ways = self.getWaysInBounds(bbox)
+
+        timeB = time.time()
+        sys.stderr.write("<!-- Get ways in bbox %s -->\n" % (timeB - timeA))
 
         wayNodes = self.getNodesFromWays(ways)
+
+        timeC = time.time()
+        sys.stderr.write("<!-- Get nodes from ways %s -->\n" % (timeC - timeB))
+
         for n in wayNodes:
             if n['id'] not in nodes:
                 nodes.append(n)
-        sys.stderr.write("<!-- Get nodes from ways %s -->\n" % time.time())
+
+        timeD = time.time()
+        sys.stderr.write("<!-- Collate nodes from ways %s -->\n" % (timeD - timeC))
 
         relations = self.getRelationsFromWays(ways)
-        sys.stderr.write("<!-- Get relations %s -->\n" % time.time())
+
+        timeE = time.time()
+        sys.stderr.write("<!-- Get relations %s -->\n" % (timeE - timeD))
 
         doc = {'bounds': {'minlat': bbox[0][0],
                           'minlon': bbox[0][1],
@@ -275,13 +293,15 @@ def bareApi(request):
                            (r'^api$', bareApi))
 
 if __name__ == '__main__':
-    """
     import time, sys
-    bbox = [[46.784,-92.3746],[46.8197,-92.3159]]
+    #bbox = [[46.784,-92.3746],[46.8197,-92.3159]]
+    bbox = [[39.2674,-75.5644],[39.2839,-75.5349]]
     api = OsmApi()
     data = api.getBbox(bbox)
 
     outputter = OsmXmlOutput()
-    outputter.write(data)
-    sys.stderr.write("<!-- XML output %s -->\n" % time.time())
-    """
+    start = time.time()
+    outfile = open(sys.argv[1], 'w')
+    outfile.write(outputter.toXml(data))
+    outfile.close()
+    sys.stderr.write("<!-- XML output %s -->\n" % (time.time() - start))