# 爬取丁香园的数据并存储进数据库
import requests
from time import sleep
from sqlalchemy.orm import sessionmaker
from dxy.model import User, Subject, Comment, db_engine
from sqlalchemy.exc import IntegrityError
session = sessionmaker(bind=db_engine)
db = session()
def save(data):
if type(data) == list:
for i in data:
try:
db.add(i)
db.commit()
except IntegrityError:
db.flush()
else:
db.add(data)
db.commit()
def crawler(url):
headers = {
'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36'),
}
cookies = {}
temp_cookies_string = ("DXY_USER_GROUP=49; __auc=f5db5ffc1693f4415a8b9b324af; _ga=GA1.2.406327704.1551544823; __"
"utmz=1.1551655682.5.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr="
"(not%20provided); __utmz=3004402.1551676197.1.1.utmcsr=(direct)|utmccn=(direct)|"
"utmcmd=(none); _gid=GA1.2.13027547.1551889001; JUTE_BBS_DATA=15fded56752024cd95f26a0e8df"
"09dabc0b65dec4e30437ed04c6b1520f500a3a8e02c6161f0a5bd48d0f3ef0959dd38b9276650ec31f4f0c1"
"59e419c1b97cd34c3a0891d95f2a3926ef6fb7c3b40b4a551ebbb281325a043e4082b5e123d2287015bdcf2e"
"4925add012fdb048e846953a845df43b4b505c; __utma=1.406327704.1551544823.1551889130.1551914"
"990.8; JSESSIONID=6D78A1886433974BF211D4EA7FFBDA91; __utma=3004402.406327704.1551544823."
"1551914699.1551963394.6; __utmc=3004402; JUTE_SESSION_ID=f4417a7d-8d0b-417d-8f62-70b7018"
"78879; JUTE_TOKEN=b72f68e1-a1d1-45e8-8f7f-974abcad9dc9; JUTE_SESSION=04c9d3a941888796762"
"cc09e4d6b56b7a4047b1d26ec72a4bbb0433ff00144852b0e659c4741b4b7151463e8a91fdd12db83bc5ecde"
"5622b66b04b11d64be607c44fe976b21f8170")
for i in temp_cookies_string.split(';'):
key_value = i.strip().split('=')
cookies[key_value[0]] = key_value[1]
response = requests.get(url, headers=headers, cookies=cookies).json()
max_page = response['pageBean']['total']
subject = Subject(title=response['subject'], user_id=response['items'][0]['userId'])
save(subject)
# 获取 comment 数据与 user 数据
for i in range(1, max_page+1):
response = requests.get(url.format(i), headers=headers, cookies=cookies).json()
item_user = [item['user'] for item in response['items']]
# 获取评论的用户数据
user = [User(user_id=item['userId'], avatar=item['avatar'], nickname=item['nickname']) for item in item_user]
# 获取评论的详情
body = [Comment(user_id=item['userId'], content=item['body'], subject_id=subject.id) for item in response['items']]
save(user)
save(body)
sleep(2)
def main(url):
crawler(url)
if __name__ == '__main__':
url = ('http://3g.dxy.cn/bbs/bbsapi/mobile?'
's=view_topic&checkUserAction=1&with'
'Good=1&order=0&size=20&id=509959&page={}')
main(url)