高德爬虫

爬虫"/>

高德爬虫

from urllib.parse import quote
from urllib import request
import json
import xlwt#TODO 替换为上面申请的密钥
amap_web_key = ''
poi_search_url = ""
poi_boundary_url = ""
#from transCoordinateSystem import gcj02_to_wgs84#TODO cityname为需要爬取的POI所属的城市名，nanning_areas为城市下面的所有区，classes为多个分类名集合. (中文名或者代码都可以，代码详见高德地图的POI分类编码表)
cityname = '北京'
nanning_areas = ["东城区","西城区","朝阳区""丰台区","石景山区","海淀区","顺义区","通州区","大兴区","房山区","门头沟区","昌平区","平谷区","密云区","怀柔区","延庆区"]
classes = ['烧烤','铁锅炖','火锅']# 根据城市名称和分类关键字获取poi数据
def getpois(cityname, keywords):i = 1poilist = []while True:  # 使用while循环不断分页获取数据result = getpoi_page(cityname, keywords, i)print(result)result = json.loads(result)  # 将字符串转换为jsonif result['count'] == '0':breakhand(poilist, result)i = i + 1return poilist# 数据写入excel
def write_to_excel(poilist, cityname, classfield):# 一个Workbook对象，这就相当于创建了一个Excel文件book = xlwt.Workbook(encoding='utf-8', style_compression=0)sheet = book.add_sheet(classfield, cell_overwrite_ok=True)# 第一行(列标题)sheet.write(0, 0, 'x')sheet.write(0, 1, 'y')sheet.write(0, 2, 'count')sheet.write(0, 3, 'name')sheet.write(0, 4, 'tel')sheet.write(0, 5, 'adname')sheet.write(0, 6, 'address')for i in range(len(poilist)):location = poilist[i]['location']name = poilist[i]['name']lng = str(location).split(",")[0]lat = str(location).split(",")[1]tel = poilist[i]['tel']adname = poilist[i]['adname']address = poilist[i].get('address','空')'''result = gcj02_to_wgs84(float(lng), float(lat))lng = result[0]lat = result[1]'''# 每一行写入sheet.write(i + 1, 0, lng)sheet.write(i + 1, 1, lat)sheet.write(i + 1, 2, 1)sheet.write(i + 1, 3, name)sheet.write(i + 1, 4, tel)sheet.write(i + 1, 5, adname)sheet.write(i + 1, 6, address)# 最后，将以上操作保存到指定的Excel文件中book.save(r'' + cityname + "_" + classfield + '.xls')# 将返回的poi数据装入集合返回
def hand(poilist, result):# result = json.loads(result)  # 将字符串转换为jsonpois = result['pois']for i in range(len(pois)):poilist.append(pois[i])# 单页获取pois
def getpoi_page(cityname, keywords, page):req_url = poi_search_url + "?key=" + amap_web_key + '&extensions=all&keywords=' + quote(keywords) + '&city=' + quote(cityname) + '&citylimit=true' + '&offset=25' + '&page=' + str(page) + '&output=json'data = ''with request.urlopen(req_url) as f:data = f.read()data = data.decode('utf-8')return datafor clas in classes:classes_all_pois = []for area in nanning_areas:pois_area = getpois(area, clas)print('当前城区：' + str(area) + ', 分类：' + str(clas) + ", 总的有" + str(len(pois_area)) + "条数据")classes_all_pois.extend(pois_area)print("所有城区的数据汇总，总数为：" + str(len(classes_all_pois)))write_to_excel(classes_all_pois, cityname, clas)print('================分类：'  + str(clas) + "写入成功")