xymandy的博客

from urllib.request import urlopenfrom urllib.parse import urlparsefrom bs4 import BeautifulSoupimport reimport datetimeimport randompages = set()random.seed(datetime.datetime.now())#获取页面所有内链的列表def ge...

import requestsimport reimport bs4from bs4 import BeautifulSoup#提交URL获得html页面def getHTMLText(url): try: r = requests.get(url,timeout = 30) r.raise_for_status() r.encoding = r.apparent_encoding ret...

import requestsimport redef getHTMLText(url): try: r = requests.get(url,timeout=30) r.raise_for_status() r.encoding = r.apparent_encoding return r.text except: return ""def parsePage(ilt,html): t...

在cmd里面输入在ide里面修改books.py文件# -*- coding: utf-8 -*-import scrapyclass BooksSpider(scrapy.Spider): #爬虫唯一标识 name = 'books' #入口点 start_urls = ['http://books.toscrape.com/'] #解析页面，1.提取页面中的数据，2、提取页面中的链接   #页...