爬虫所有实际的例子

1.

import requests
import re
def getHtml(url):
try:
page=requests.get(url)
page.raise_for_status()
page.encode=page.apparent_encoding
return page.text
except:
return ‘产生异常’

def getImage(html):
global x
base=‘https://www.50zw.com/files/’
matcher=re.findall(r’article/image.*?jpg’,html)
for i in matcher:
print(base+i)
response=requests.get(base+i)
with open(‘image/%s.jpg’%x,‘wb’) as f:
f.write(response.content)
x=x+1

x=0
for i in [1,2,3]:
url=‘https://m.50zw.com/wapsort/1_’+str(i)+’.html’
html=getHtml(url)
getImage(html)

2

import requests
import re
from bs4 import BeautifulSoup
def getHtml(url):
try:
page=requests.get(url)
page.raise_for_status()
page.encoding=page.apparent_encoding
return page.text
except:
print(“error”)

def getImage(html):
soup=BeautifulSoup(html,“html.parser”)
tags=soup.find_all(name=“img”)
print(len(tags))
for tag in tags:
print(tag[‘src’])
response=requests.get(tag[‘src’])
with open("") as f:
12313

url=“https://m.50zw.com/wapsort/1_1.html”
html=getHtml(url)
getImage(html)

3

import requests
import re
import pandas as pd
from bs4 import BeautifulSoup
def getHtml(url):
try:
page=requests.get(url)
page.raise_for_status()
page.encoding=page.apparent_encoding
return page.text
except:
print(“error”)

def getInformation(html):
soup=BeautifulSoup(html,“html.parser”)
tags=soup.select(‘div.article’)
names=[]
writers=[]
counts=[]

for tag in tags:
    name=tag.select('a')[0].string
    writer=tag.select('span.mr15')[0].string
    count=tag.select('span.count')[0].string
    describe=tag.select('span')[-1].string
    names.append(name)
    writers.append(writer)
    counts.append(count)
    #s=name + '  ' + writer + '  ' + count + '\n'
p=pd.DataFrame({"name":names,"writer":writers,"counts":counts})
print(p)
p.to_csv("123.csv")

url=“https://m.50zw.com/wapsort/1_1.html”
html=getHtml(url)
getInformation(html)

4