import re
import requests
from lxml import etree
from bs4 import BeautifulSoup
import csv
def get_one_page(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36'
}
response = requests.get(url, headers=headers)
if response.status_code == 200:
return response.text
return None
def main():
file = open('tuiku.txt', 'a', encoding='utf-8')
with open('tuiku.csv', 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(['title'])
for i in range(5):
url = 'https://www.tuicool.com/ah/0/' + str(i) + '?lang=1'
html = get_one_page(url)
soup = BeautifulSoup(html, "lxml")
for a in soup.findAll(name="a", attrs={"href": re.compile("^/articles")}):
print(a.string)
writer.writerow([a.string])
file.write(a.string)
file.write('\n')
file.close()
csvfile.close()
main()