Python爬虫自动登陆

    背景：下载WISA2017会议论文集。

    工具：python

import requests
from requests.auth import HTTPBasicAuth
from bs4 import BeautifulSoup

page_config = {'username':'XXX',
               'password':'XXX'}
url = "http://conferences.computer.org/wisa/2017/"
auth = HTTPBasicAuth('XXX','XXX')
page = requests.get(url,auth=(page_config['username'],page_config['password']))
soup = BeautifulSoup(page.content, 'html.parser')#获取html页面

for i in soup.find_all('a'):
    name.append(i.string)
    link.append(url + i['href'])
def getFile(paper_url, file_name):

    u = requests.get(url=paper_url, auth=auth)    #下载url
    print u
    #f = open(file_name, 'wb')
    with open(r'paper/'+file_name+'.pdf', "wb") as pdf:
        for chunk in u.iter_content(chunk_size=10240):
            if chunk:
                pdf.write(chunk)
    print ("Sucessful to download" + " " + file_name)

    关键字：beautifulsoup；css选择器；auth验证；文件操作（bug相关----pdf文件名不能包含英文问号）；