python采集文章_Python爬虫，python设计原文章采集源码

#设计原采集

import requests

import re

from lxml import etree

import os

os.makedirs(f'shejy/',exist_ok=True)

#获取页码

def get_page(category):

url = f"http://jy.sccnn.com/category-{category}_1.html"

html = requests.get(url).text

# print(html)

pagesze=r'1(.+?)

'

pages=re.findall(pagesze,html,re.S)

pageze=r'>›.+?››'

page=re.findall(pageze,pages[0],re.S)

page=page[0]

print(page)

return page

#获取链接列表

def get_urls(category,i):

url=f"http://jy.sccnn.com/category-{category}_{i}.html"

try:

html=requests.get(url).text

#print(html)

urlsze=r'auth1">.+?

urls=re.findall(urlsze,html,re.S)

print(len(urls))

print(urls)

return urls

except:

print(f"{url}-----访问失败！")

with open('shejy/spider.txt','a+',encoding='utf-8') as f:

f.write(f"{url}-----访问失败！\n")

pass

#获取正文

def get_con(url):

try:

html = requests.get(url).text

#print(html)

con = etree.HTML(html)

# 获取标题

h = con.xpath('//div[@class="singletitle"]/h2/text()')

h = h[0]

h = re.sub(r'[\|\/\<\>\:\*\?\\\"]', "_", h) # 剔除不合法字符

print(h)

os.makedirs(f'shejy/{h}/', exist_ok=True)

# 获取源码

con_textze = r'

(.+?)

'

con_text = re.findall(con_textze, html, re.S)

con_text = con_text[0]

print(con_text)

# 获取文本

texts = etree.HTML(con_text)

text = texts.xpath('string(.)')

print(text)

# 获取图片

imgs = texts.xpath('//img')

# print(len(imgs))

for img in imgs:

img_url = img.attrib['src']

img_name = img_url[-12:]

print(img_name)

try:

r = requests.get(img_url)

with open(f'shejy/{h}/{img_name}', 'wb') as f:

f.write(r.content)

print(f"保存{img_name}图片成功了！")

except:

print(f"{img_url}-----访问失败！")

with open('shejy/spider.txt', 'a+', encoding='utf-8') as f:

f.write(f"{img_url}-----访问失败！\n")

pass

dates = '%s%s%s%s%s' % (h, '\n', con_text, '\n', text)

with open(f'shejy/{h}/{h}.txt', 'w+', encoding='utf-8') as f:

f.write(dates)

print(f"保存{h}.txt成功了！")

except:

print(f"{url}-----访问失败！")

with open('shejy/spider.txt','a+',encoding='utf-8') as f:

f.write(f"{url}-----访问失败！\n")

if __name__ == '__main__':

category=input("请输入要采集的分类：")

num=get_page(category)

num=int(num)

urlss=[]

for i in range(1,num+1):

urls=get_urls(category,i)

urlss.extend(urls)

print(urlss)

for url in urlss:

print(url)

get_con(url)