南昌大学数据库实训大作业_实训大作业7

# 爬取丁香园的数据并存储进数据库

import requests

from time import sleep

from sqlalchemy.orm import sessionmaker

from dxy.model import User, Subject, Comment, db_engine

from sqlalchemy.exc import IntegrityError

session = sessionmaker(bind=db_engine)

db = session()

def save(data):

if type(data) == list:

for i in data:

try:

db.add(i)

db.commit()

except IntegrityError:

db.flush()

else:

db.add(data)

db.commit()

def crawler(url):

headers = {

'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '

'(KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36'),

}

cookies = {}

temp_cookies_string = ("DXY_USER_GROUP=49; __auc=f5db5ffc1693f4415a8b9b324af; _ga=GA1.2.406327704.1551544823; __"

"utmz=1.1551655682.5.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr="

"(not%20provided); __utmz=3004402.1551676197.1.1.utmcsr=(direct)|utmccn=(direct)|"

"utmcmd=(none); _gid=GA1.2.13027547.1551889001; JUTE_BBS_DATA=15fded56752024cd95f26a0e8df"

"09dabc0b65dec4e30437ed04c6b1520f500a3a8e02c6161f0a5bd48d0f3ef0959dd38b9276650ec31f4f0c1"

"59e419c1b97cd34c3a0891d95f2a3926ef6fb7c3b40b4a551ebbb281325a043e4082b5e123d2287015bdcf2e"

"4925add012fdb048e846953a845df43b4b505c; __utma=1.406327704.1551544823.1551889130.1551914"

"990.8; JSESSIONID=6D78A1886433974BF211D4EA7FFBDA91; __utma=3004402.406327704.1551544823."

"1551914699.1551963394.6; __utmc=3004402; JUTE_SESSION_ID=f4417a7d-8d0b-417d-8f62-70b7018"

"78879; JUTE_TOKEN=b72f68e1-a1d1-45e8-8f7f-974abcad9dc9; JUTE_SESSION=04c9d3a941888796762"

"cc09e4d6b56b7a4047b1d26ec72a4bbb0433ff00144852b0e659c4741b4b7151463e8a91fdd12db83bc5ecde"

"5622b66b04b11d64be607c44fe976b21f8170")

for i in temp_cookies_string.split(';'):

key_value = i.strip().split('=')

cookies[key_value[0]] = key_value[1]

response = requests.get(url, headers=headers, cookies=cookies).json()

max_page = response['pageBean']['total']

subject = Subject(title=response['subject'], user_id=response['items'][0]['userId'])

save(subject)

# 获取 comment 数据与 user 数据

for i in range(1, max_page+1):

response = requests.get(url.format(i), headers=headers, cookies=cookies).json()

item_user = [item['user'] for item in response['items']]

# 获取评论的用户数据

user = [User(user_id=item['userId'], avatar=item['avatar'], nickname=item['nickname']) for item in item_user]

# 获取评论的详情

body = [Comment(user_id=item['userId'], content=item['body'], subject_id=subject.id) for item in response['items']]

save(user)

save(body)

sleep(2)

def main(url):

crawler(url)

if __name__ == '__main__':

url = ('http://3g.dxy.cn/bbs/bbsapi/mobile?'

's=view_topic&checkUserAction=1&with'

'Good=1&order=0&size=20&id=509959&page={}')

main(url)