python爬虫抓包爬取NBA中文网2020

爬虫抓包爬取NBA中文网2020"/>

python爬虫抓包爬取NBA中文网2020

# 因为NBA中文网球员数据是通过json异步加载的，所以需要抓包实现
# json链接可以打开F12的Console查看
import time
import requests
import json
import csv# 获取json文件里面的内容并提取信息
def get_json_content(url):# r_content=requests.get(url).content.decode('utf-8')r_content = requests.get(url).text# 将json文件里的内容转化为json格式，结构化方便提取数据r_json = json.loads(r_content)for i in range(0, 50):# 用一个字典保存球员数据所在的赛季data = {}player = r_json.get('payload')['players'][i]data['球员姓名'] = player['playerProfile']['displayName']data['球队'] = player['teamProfile']['displayAbbr']data['场均得分'] = player['statAverage']['pointsPg']data['上场时间'] = player['statAverage']['minsPg']data['篮板'] = player['statAverage']['rebsPg']data['助攻'] = player['statAverage']['assistsPg']data['盖帽'] = player['statAverage']['blocksPg']data['防守'] = player['statAverage']['defRebsPg']data['效率'] = player['statAverage']['efficiency']data['命中率'] = player['statAverage']['fgpct']print('第' + str(i + 1) + '个球员数据信息已经爬取完成')print(data)# 引用全局变量文件路径global pathwith open(path, 'a') as f:# 得到一个CSV写入对象writer = csv.writer(f, dialect=my_dialect)# 向CSV文件中写入一行数据writer.writerow((data['球员姓名'], data['球队'], data['场均得分'], data['上场时间'],data['篮板'], data['助攻'], data['盖帽'], data['防守'], data['效率'],data['命中率']))if __name__ == '__main__':# 数据保存路径path = 'E://NBA_data.csv'# 自定义一个CSV文件内容分隔形式class my_dialect(csv.Dialect):lineterminator = '\n'delimiter = ','quotechar = '"'quoting = csv.QUOTE_MINIMALwith open(path, 'w') as f:writer = csv.writer(f, dialect=my_dialect)writer.writerow(('球员姓名', '球队', '场均得分', '上场时间','篮板', '助攻', '盖帽', '防守', '效率', '命中率'))# 球员数据的json包链接url = '.json'get_json_content(url)

• 注意目前默认导出的文件路径是在E盘
• 需要其它相关数据可以直接在浏览器打开url的json链接，自行查找并对代码进行修改