#-*- coding = utf-8 -*-
#@Time : 2020/7/23 15:09
#@Author :
#@File : douban_group.py
#@software : PyCharm
#爬取豆瓣组数
#待解决的问题:标签下小组不全!!
import csv
import importlib
import urllib.request
import urllib.error
import re
from random import sample
from urllib.parse import quote
import string
import regex
from bs4 import BeautifulSoup
import sqlite3
import xlwt
import pandas as pd
#确定一个主函数
def main():
baseurl = "https://www.douban.com/"
datalist = getDATA(baseurl)
#print(datalist)
pd.DataFrame(datalist).to_csv("豆瓣小组.csv")
#savepath = "豆瓣组员数超过一万的小组.xls"
#saveDATA(datalist)
findtag = regex.compile(r'
(.*)')findlink = regex.compile('
')soup = BeautifulSoup(html, "html.parser")
for cate in soup.find_all('div', class_="cate group-cate"):#筛选到tag和link
#link = regex.findall(findlink, cate, overlapped=True)
link = "https://www.douban.com/group/explore?"
tag = regex.findall(findtag, cate, overlapped=True)
')findpagenum = re.compile(r'1')
url1 = link + "tag=" + quote(str(tag[i]))
soup_temp = BeautifulSoup(html_temp, "html.parser")
for paginator in soup_temp.find_all('div',class_="paginator"):
pagenum = re.findall(findpagenum, paginator)[0]
for j in range(0, int(pagenum)):
url2 = link + "start=" + str(j*20) + "&tag=" + quote(str(tag[i]))
soup_temp1 = BeautifulSoup(html_temp1, "html.parser")
for result in soup_temp1.find_all('div', class_="result"):
name =re.findall(findname, result)[0]
sum = re.findall(findsum, result)[0]
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
'Opera/9.25 (Windows NT 5.1; U; en)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
"User-Agent": sample(my_headers, 1)[0]}
req = urllib.request.Request(url=url, headers=head)
response = urllib.request.urlopen(req)
html = response.read().decode("utf-8")
except urllib.error.URLError as e:
# pf = pd.DataFrame(list(datalist))
# pf.rename(columns=columns_map, inplace=True)
# file_path = pd.ExcelWriter('doubangroup.xlsx')
# pf.fillna(' ', inplace=True)
# pf.to_excel(file_path, encoding='utf-8', index=False)