python cookie池_scrapy 设置cookie池

#!/usr/bin/python#coding=utf-8#__author__=‘dahu‘#data=2017-#

importrequestsimporttimefrom pymongo importMongoClientimportcookielibimporturllib2from bson.objectid importObjectId

url= ‘https://www.so.com‘

#url = ‘https://cn.bing.com/translator‘

client = MongoClient(‘localhost‘, 27017)

db= client[‘save_cookie‘]

collection= db[‘san60cookie‘]defget_header():

header={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8","Accept-Encoding": "gzip, deflate, br","Accept-Language": "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4","Cache-Control": "max-age=0","Connection": "keep-alive","Host": "www.so.com","Upgrade-Insecure-Requests": "1","User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",

}returnheaderdefget_cookie_lib():

cookie=cookielib.CookieJar()

handler=urllib2.HTTPCookieProcessor(cookie)

opener=urllib2.build_opener(handler)

response=opener.open(url)#for item in cookie:

#print "%s : %s" % (item.name, item.value)

cookie_dict ={}for cook incookie:

cookie_dict[cook.name]=cook.valuereturncookie_dictdefsave_cookie_into_mongodb(cookie):print ‘insert‘insert_data={}

insert_data[‘cookie‘] =cookie

insert_data[‘insert_time‘] = time.strftime(‘%Y-%m-%d %H:%M:%S‘)

insert_data[‘request_url‘]=url

insert_data[‘insert_timestamp‘] =time.time()

collection.insert(insert_data)defdelete_timeout_cookie(request_url):

time_out= 300

for data in collection.find({‘request_url‘:request_url}):if (time.time() - data.get(‘insert_timestamp‘)) >time_out:print ‘delete: %s‘ % data.get(‘_id‘)

collection.delete_one({‘_id‘: ObjectId(data.get(‘_id‘))})

#这里有疑问的话可以参考http://api.mongodb.com/python/current/tutorial.html#querying-by-objectiddefget_cookie_from_mongodb():

cookies= [data.get(‘cookie‘) for data incollection.find()]returncookiesif __name__ == ‘__main__‘:

num=0while 1:if num == 2:print ‘deleting‘delete_timeout_cookie(url)

num=0else:

cookie=get_cookie_lib()

save_cookie_into_mongodb(cookie)

num+= 1time.sleep(5)