GrofChen的博客

import threading,urllib,jsonfrom queue import Queue from bs4 import BeautifulSoup as bsCRAWL_EXIT = Falseclass ThreadCrawl(threading.Thread): def __init__(self,threadName,pageQueue,dataQueue):...

from bs4 import BeautifulSoup as bsimport requests,json,timefile = open("duanzi.json", "a")def crawl_html(num): headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20...

from bs4 import BeautifulSoup as bsimport requests,json,timefile = open("duanzi.json", "a")def crawl_html(num): headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20...

有道翻译import urllib.requestimport urllib.parseurl='http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule' # 有道翻译错误代码50时，去掉translate_o?的_oheaders={"User-Agent":"Mozilla/5.0 (Windo...

这里使用requests库爬取网页要比urllib库方便用finditer查询import requestsimport reurl='https://www.flyai.com/' # 带爬取的网页html=requests.get(url).text # text为转化为str数据pat='(\w+@\w+.com)|(\d{11})' # 正则模式res=re.findite...

from bs4 import BeautifulSoup as bsimport urllibimport reclass Spider(object): def __init__(self): print('init') self.begin_page=1 self.end_page=4 self.base_url=...

from bs4 import BeautifulSoup as bsimport urllibclass Spider(object): def __init__(self): self.base_url='https://www.biquge.com.cn' def load_page(self,url): headers={"User-Ag...