bs4使用
1.拿到页面源代码
2.使用bs4进行解析,拿到数据
首先基本框架
import requests
from bs4 import BeautifulSoup
import csv
url = "http://www.xinfadi.com.cn/marketanalysis/0/list/1.shtml"
resp = requests.get(url)
f = open("菜价.csv", mode="w",encoding="utf-8")
cswriter = csv.writer(f)
然后解析数据
把页面源代码交给BeautifulSoup进行处理,生成bs对象
page = BeautifulSoup(resp.text, 'html.parser')
其中‘html.parser’是指定解析器
找东西是用如下两种代码之一
page.find("table", class_="hq_table")
table = page.find("table", attrs={"class": "hq_table"})
再拿所有数据行的数据,html中标签为<tr>
所以我们采取如下方法:
trs = table.findAll("tr")[1:]
循环拿数据并写入文件:
for tr in trs:
tds = tr.find_all("td") # 拿到每一行中的所有td
name = tds[0].text # 表示拿到被标签标记的内容
low = tds[1].text
avg = tds[2].text
high = tds[3].text
gui = tds[4].text
kind = tds[5].text
date = tds[6].text
cswriter.writerow([name,low,avg,high,gui,kind,date])
这样就爬取到了数据
爬取优美图库
基本框架同上
这里主要是拿到每一块的子网页源码和图片的下载链接
拿到主页面的源代码,然后提取到子页面的链接地址href
通过href拿到子页面的内容,从子页面中找到图片的下载地址
child_page_resp = requests.get(href)
child_page_resp.encoding = 'utf-8'
child_page = BeautifulSoup(child_page_resp.text, 'html.parser')
p = child_page.find("p", align="center")
img = p.find("img")
src = img.get("src")
下载图片:
img_resp = requests.get(src)
img_name = src.split("/")[-1]
with open(img_name,mode="wb") as f:
f.write(img_resp.content)
xpath解析
库
from lxml import etree
# result = tree.xpath("/book") # /表示层级关系,第一个/是根节点
# result = tree.xpath("/book/name/text()")
# tree.xpath("/book") 从文件中提取括号里对应内容
result = tree.xpath("/book/author/nick/text()") # text() 是拿文本
result1 = tree.xpath("/book/author//nick/text()") # //表示后代
result2 = tree.xpath("/book/author/*/nick/text()") # * 表示任意的节点,同配符
Xpath解析实例:
爬取猪八戒网
首先框架:
import requests
from lxml import etree
url="https://ganzhou.zbj.com/search/f/?type=new&kw=saas"
resp = requests.get(url)
#解析
html = etree.HTML(resp.text)
#首先拿到每一个服务商的div
divs = html.xpath("/html/body/div[6]/div/div/div[2]/div[6]/div[1]/div")
for i in divs: # 每一个服务商信息
price = i.xpath('./div/div/a[1]/div[2]/div[1]/span[1]/text()')
title = "saas".join(i.xpath('./div/div/a[1]/div[2]/div[2]/p/text()'))
com_name = i.xpath('./div/div/a[2]/div[1]/p/text()')
location = i.xpath('./div/div/a[2]/div[1]/div/span/text()')