python爬取标题_Python抓取push cool页面的标题代码,python,爬取,推酷

import re

import requests

from lxml import etree

from bs4 import BeautifulSoup

import csv

def get_one_page(url):

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36'

}

response = requests.get(url, headers=headers)

if response.status_code == 200:

return response.text

return None

def main():

file = open('tuiku.txt', 'a', encoding='utf-8')

with open('tuiku.csv', 'w', newline='') as csvfile:

writer = csv.writer(csvfile)

writer.writerow(['title'])

for i in range(5):

url = 'https://www.tuicool.com/ah/0/' + str(i) + '?lang=1'

html = get_one_page(url)

soup = BeautifulSoup(html, "lxml")

for a in soup.findAll(name="a", attrs={"href": re.compile("^/articles")}):

print(a.string)

writer.writerow([a.string])

file.write(a.string)

file.write('\n')

file.close()

csvfile.close()

main()