day002爬虫-下厨房-lxml-treads多线程-redis多进程多线程分布式--scrapy

curl https://www.xiachufang.com/ | grep -oP '(?<=src=\")http://i2\.chuimg\.com/\w+\.jpg' | more

curl -s https://www.xiachufang.com/ | grep -oP '(?<=src=\")http://i2\.chuimg\.com/\w+\.jpg' | xargs -i curl {
   } -O

1. requests_bs4.py

import os
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import requests

r = requests.get('https://www.xiachufang.com/')
soup = BeautifulSoup(r.text)
# print(soup)

img_list = []
for img in soup.select('img'):
    if img.has_attr('data-src'):
        img_list.append(img.attrs['data-src'])
    else:
        img_list.append(img.attrs['src'])

# 初始化下载文件目录
image_dir = os.path.join(os.curdir, 'images')
# if not os.path.isdir(image_dir):
#     os.mkdir(image_dir)

for img in img_list:
    o = urlparse(img)
    filename = o.path[1:].split('@')[0]
    filepath = os.path.join(image_dir, filename)
    if not os.path.isdir(os.path.dirname(filepath)):
        os.mkdir(os.path.dirname(filepath))
    url = '%s://%s/%s' %(o.scheme, o.netloc, filename)
    print(url)
    resp = requests.get(img)
    with open(filepath, 'wb') as f:
        for chunk in resp.iter_content(1024):
            f.write(chunk)

2.qianmu.py

import requests
import lxml.etree.ElemenTree as ET

# print(str(ET.tostring(dom代码块, encoding='utf8'), 'utf-8'))

start_url = ''


def fetch(url):
    """请求并下载网页"""
    r = requests.get(url)
    if r.status_code != 200:
        r.raise_for_status()
    return r.text.replace('\t', '')


def parse_university(url):
    """处理大学详情页面"""
    selector_ = ET.HTML(fetch(url))
    data_ = {
   }
    data_['name'] = selector_.xpath('//div[@id="wikiContent"]/h1/text()')[0]
    table = selector_.xpath(&