解决高德数据爬取中address字段不存在导致程序运行报错的问题

liujiao111 · Jun 9, 2020 · e114be1 · e114be1
1 parent 082354a
commit e114be1
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 39 deletions.
diff --git a/gaode/poi-city/app.py b/gaode/poi-city/app.py
@@ -18,8 +18,6 @@
 2019.10.10:
     1. 数据导出支持CSV以及XLS两种格式;
     2. CSV格式数据会生成.shp文件，可以直接在ARCGIS中使用
-
-
 '''
 
 #################################################需要修改###########################################################
@@ -28,10 +26,10 @@
 amap_web_key = '高德密钥'
 
 # TODO 2.分类关键字,最好对照<<高德地图POI分类关键字以及编码.xlsx>>来填写对应编码，多个用逗号隔开
-keyword = ['190307']
+keyword = ['分类编码/关键字']
 
 # TODO 3.城市，多个用逗号隔开
-city = ['衡阳市']
+city = ['城市名']
 
 # TODO 4.输出数据坐标系,1为高德GCJ20坐标系，2WGS84坐标系，3百度BD09坐标系
 coord = 2
@@ -81,13 +79,13 @@ def write_to_excel(poilist, cityname, classfield):
     sheet.write(0, 7, 'type')
 
     for i in range(len(poilist)):
-        location = poilist[i]['location']
-        name = poilist[i]['name']
-        address = poilist[i]['address']
-        pname = poilist[i]['pname']
-        cityname = poilist[i]['cityname']
-        business_area = poilist[i]['business_area']
-        type = poilist[i]['type']
+        location = poilist[i].get('location')
+        name = poilist[i].get('name')
+        address = poilist[i].get('address')
+        pname = poilist[i].get('pname')
+        cityname = poilist[i].get('cityname')
+        business_area = poilist[i].get('business_area')
+        type = poilist[i].get('type')
         lng = str(location).split(",")[0]
         lat = str(location).split(",")[1]
 
@@ -120,13 +118,15 @@ def write_to_csv(poilist, cityname, classfield):
     lons, lats, names, addresss, pnames, citynames, business_areas, types = [], [], [], [], [], [], [], []
 
     for i in range(len(poilist)):
-        location = poilist[i]['location']
-        name = poilist[i]['name']
-        address = poilist[i]['address']
-        pname = poilist[i]['pname']
-        cityname = poilist[i]['cityname']
-        business_area = poilist[i]['business_area']
-        type = poilist[i]['type']
+        print('===================')
+        print(poilist[i])
+        location = poilist[i].get('location')
+        name = poilist[i].get('name')
+        address = poilist[i].get('address')
+        pname = poilist[i].get('pname')
+        cityname = poilist[i].get('cityname')
+        business_area = poilist[i].get('business_area')
+        type = poilist[i].get('type')
         lng = str(location).split(",")[0]
         lat = str(location).split(",")[1]
 

diff --git a/gaode/poi-pology/app.py b/gaode/poi-pology/app.py
@@ -31,7 +31,7 @@
 typs = ['大学']  # ['企业', '公园', '广场', '风景名胜', '小学']
 
 ## TODO 4. 高德开放平台密钥
-gaode_key = ['292616426c4d5d268ab2a064359f7f43', '23b84bf3ce4c93c7175720d7b828530a']
+gaode_key = ['高德密钥1', '高德密钥2']
 
 # TODO 5.输出数据坐标系,1为高德GCJ20坐标系，2WGS84坐标系，3百度BD09坐标系
 coord = 2
@@ -96,17 +96,17 @@ def write_to_csv(poilist, citycode, classfield, coord):
         return None, None
 
     for i in range(len(poilist)):
-        location = poilist[i]['location']
-        name = poilist[i]['name']
-        address = poilist[i]['address']
-        pname = poilist[i]['pname']
-        cityname = poilist[i]['cityname']
-        business_area = poilist[i]['business_area']
-        type = poilist[i]['type']
-        typecode = poilist[i]['typecode']
+        location = poilist[i].get('location')
+        name = poilist[i].get('name')
+        address = poilist[i].get('address')
+        pname = poilist[i].get('pname')
+        cityname = poilist[i].get('cityname')
+        business_area = poilist[i].get('business_area')
+        type = poilist[i].get('type')
+        typecode = poilist[i].get('typecode')
         lng = str(location).split(",")[0]
         lat = str(location).split(",")[1]
-        id = poilist[i]['id']
+        id = poilist[i].get('id')
 
         if (coord == 2):
             result = gcj02_to_wgs84(float(lng), float(lat))

diff --git a/gaode/poi-province/app.py b/gaode/poi-province/app.py
@@ -131,7 +131,6 @@ def getpoi_page(cityname, keywords, page):
     if data['status'] == '0':  # 请求成功，但是返回数据失败
         if data['infocode'] == '10001':
             print('无效的密钥！！！！！！！！！！！！！，重新切换密钥进行爬取')
-            # TODO 重新更换密钥
             buffer_keys.remove(buffer_keys[0])
             return getpoi_page(cityname, keywords, page)
         if data['infocode'] == '10003':
@@ -214,19 +213,19 @@ def write_to_csv(poilist, provincename, classfield):
         , type_1s, type_2s, type_3s, type_4s, ids = [], [], [], [], [], [], [], [], [], [], [], [], [], [], []
 
     for i in range(len(poilist)):
-        location = poilist[i]['location']
-        name = poilist[i]['name']
-        address = poilist[i]['address']
-        pname = poilist[i]['pname']
+        location = poilist[i].get('location')
+        name = poilist[i].get('name')
+        address = poilist[i].get('address')
+        pname = poilist[i].get('pname')
         #provincename = poilist[i]['provincename']
-        business_area = poilist[i]['business_area']
-        cityname = poilist[i]['cityname']
-        adname = poilist[i]['adname']
-        type = poilist[i]['type']
-        typecode = poilist[i]['typecode']
+        business_area = poilist[i].get('business_area')
+        cityname = poilist[i].get('cityname')
+        adname = poilist[i].get('adname')
+        type = poilist[i].get('type')
+        typecode = poilist[i].get('typecode')
         lng = str(location).split(",")[0]
         lat = str(location).split(",")[1]
-        id = poilist[i]['id']
+        id = poilist[i].get('id')
         type = str(type)
         type_1 = ''
         type_2 = ''