python3 爬虫 爬取华为应用市场 APP应用评论（一）爬取全部评论——学霸君

python3 爬虫 爬取华为应用市场 APP应用评论（一）爬取全部评论——学霸君

import requests
import re
import requests
from bs4 import  BeautifulSoup as bs
import json
import csv
import re

def getHTMLText(url):
    header = {
        'referer': 'https://appgallery.huawei/',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36 SE 2.X MetaSr 1.0',
        'Connection': 'keep-alive'
    }
    try:
        r = requests.get(url, timeout=30,headers=header)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.content
    except:
        return ""

def main():
    depth = 5
    infoList = []
    for i in range(depth):
        try:
            url = 'https://web-drcn.hispace.dbankcloud/uowap/index?method=internal.usermenList3&serviceType=20&reqPageNum='+str(i)+'&maxResults=25&appid=C10207269&version=10.0.0&zone=&locale=zh_CN'
            html = getHTMLText(url).decode("utf-8")
            result=json.loads(html)    
            for i in range(len(result['list'])):
                #print(result['list'][i]['accountName'])
                comment = result['list'][i]['commentInfo']
                #print(comment)
                infoList.append(comment)
            #print(result)
            #parsePage(infoList, html)
            
        except:
            continue
    #printGoodsList(infoList)
    print(infoList)
    # 把评论数据保存到文件中
    with open('xuebajunComment.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(infoList, ensure_ascii=False) + '\n')

main()

代码说明：
对华为应用市场中 学霸君 APP的用户评论进行爬取
网址如下
https://web-drcn.hispace.dbankcloud/uowap/index?method=internal.usermenList3&serviceType=20&reqPageNum=1&maxResults=25&appid=C10207269&version=10.0.0&zone=&locale=zh_CN