和xun基金排名爬取

基金排名爬取"/>

和xun基金排名爬取

大约到200多页的时候，出现了反爬

from selenium import webdriver
from selenium.webdrivermon.by import By
from selenium.webdrivermon.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time
from bs4 import BeautifulSoup
import pymongo
import datetimeclass heshunSpider(object):client =pymongo.MongoClient(host="127.0.0.1", port=27017)db = client.hexun.hxdef __init__(self):self.driver = webdriver.Firefox()def __del__(self):self.driver.close()def run(self):self.driver.maximize_window()self.driver.get(".aspx")time.sleep(1)page = 0while page<399:time.sleep(6)html = self.driver.page_sourcesoup = BeautifulSoup(html,'lxml')time.sleep(1)tr_list = soup.select('.tab01>table>tbody>tr')for tr in tr_list:item = {}item['xuhao'] = tr.find_all("td")[1].stringitem['code'] = tr.find_all("td")[2].find("a").stringitem['name'] = tr.find_all("td")[3].find("a").stringitem['date'] = tr.find_all("td")[4].get_text()item['onem'] = tr.find_all("td")[5].stringitem['threem'] = tr.find_all("td")[6].stringitem['sixm'] = tr.find_all("td")[7].stringitem['oney'] = tr.find_all("td")[8].stringitem['twoy'] = tr.find_all("td")[9].stringitem['threey'] = tr.find_all("td")[10].stringitem['todayy'] = tr.find_all("td")[11].stringitem['create'] = tr.find_all("td")[12].stringitem['rate'] = tr.find_all("td")[13].stringself.db.insert_one(item)# print(item)self.driver.execute_script("window.scrollTo(0,document.body.scrollHeight);")page = page+1self.driver.find_element_by_class_name('next').click()if __name__ == "__main__":spider = heshunSpider()spider.run()

存入mongodb

注：我是个大菜鸟，恳求大神指正，让我进步！