python模拟用户登录爬取阳光采购平台数据

阳光采购平台每月初会把当月的价格挂到平台上，现模拟用户登录平台，将需要的数据保存到csv文件和数据库，并且发送给指定人员。
开发环境搭建：

网上教程一大堆，不赘述了。安装好后需要安装一些必须的库，如下：

bs4(页面html解析)

csv(用于保存csv文件)

smtplib(用于发送邮件)

mysql.connector(用于连接数据库)

写了一个gl.py,用于保存全局变量：

'''
遇到不懂的问题？Python学习交流群：821460695满足你的需求，资料都已经上传群文件，可以自行下载！
'''
import time

timeStr = time.strftime('%Y%m%d', time.localtime(time.time()))
monthStr = time.strftime('%m', time.localtime(time.time()))
yearStr = time.strftime('%Y', time.localtime(time.time()))
LOG_FILE = "log/" + timeStr + '.log'
csvFileName = "csv/" + timeStr + ".csv"
fileName = timeStr + ".csv"
fmt = '%(asctime)s - %(filename)s:%(lineno)s  - %(message)s'
loginUrl = "http://yourpath/Login.aspx"
productUrl = 'http://yourpath/aaa.aspx'
username = 'aaaa'
password = "aaa"
preCodeurl = "yourpath"
host="yourip"
user="aaa"
passwd="aaa"
db="mysql"
charset="utf8"
postData={
            '__VIEWSTATE':'',
            '__EVENTTARGET':'',
            '__EVENTARGUMENT':'',
            'btnLogin':"登录",
            'txtUserId':'aaaa',
            'txtUserPwd':'aaa',
            'txtCode':'',
            'hfip':'yourip'
            }
tdd={
'__VIEWSTATE':'',
'__EVENTTARGET':'ctl00$ContentPlaceHolder1$AspNetPager1',
'ctl00$ContentPlaceHolder1$AspNetPager1_input':'1',
'ctl00$ContentPlaceHolder1$AspNetPager1_pagesize':'50',
'ctl00$ContentPlaceHolder1$txtYear':'',
'ctl00$ContentPlaceHolder1$txtMonth':'',
'__EVENTARGUMENT':'',
}
vs={
'__VIEWSTATE':''
}

主代码中设置日志，csv，数据库连接，cookie：

    formatter = logging.Formatter(gl.fmt)
    handler.setFormatter(formatter)
    logger = logging.getLogger('tst')
    logger.addHandler(handler)
    logger.setLevel(logging.DEBUG)
    csvFile = codecs.open(gl.csvFileName, 'w+', 'utf_8_sig')
    writer = csv.writer(csvFile)
    conn = mysql.connector.connect(host=gl.host, user=gl.user, passwd=gl.passwd, db=gl.db, charset=gl.charset)
    cursor = conn.cursor()

    cookiejar = cookielib.MozillaCookieJar()
    cookieSupport = urllib2.HTTPCookieProcessor(cookiejar)
    httpsHandLer = urllib2.HTTPSHandler(debuglevel=0)
    opener = urllib2.build_opener(cookieSupport, httpsHandLer)
    opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')]
    urllib2.install_opener(opener)

登录方法：

首先是识别验证码，转为数字。然后用(密码+用户名+验证)提交到登录方法，可能会失败，因为识别验证码有时候识别的不正确。如果登录失败，那么重新获取验证码，再次识别，再次登录，直到登录成功。

def get_logined_Data(opener,logger,views):
    print "get_logined_Data"
    indexCount = 1
    retData = None
    while indexCount <= 15:
        print "begin login ", str(indexCount), " time"
        logger.info("begin login " + str(indexCount) + " time")
        vrifycodeUrl = gl.preCodeurl + str(random.random())
        text = get_image(vrifycodeUrl)#封装一个方法，传入验证码URL，返回识别出的数字
        postData = gl.postData
        postData["txtCode"] = text
        postData["__VIEWSTATE"]=views


        data = urllib.urlencode(postData)
        try:
            headers22 = {
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
                'Accept-Encoding': 'gzip, deflate, br',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Connection': 'keep-alive',
                'Content-Type': 'application/x-www-form-urlencoded',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'
            }
            request = urllib2.Request(gl.loginUrl, data, headers22)
            opener.open(request)
        except Exception as e:
            print "catch Exception when login"
            print e

        request = urllib2.Request(gl.productUrl)
        response = opener.open(request)
        dataPage = response.read().decode('utf-8')

        bsObj = BeautifulSoup(dataPage,'html.parser')
        tabcontent = bsObj.find(id="tabcontent") #登录成功后，页面才有tabcontent这个元素，所以更具这个来判断是否登录成功
        if (tabcontent is not None):
            print "login succesfully"
            logger.info("login succesfully")
            retData = bsObj
            break
        else:
            print "enter failed,try again"
            logger.info("enter failed,try again")
            time.sleep(3)
            indexCount += 1
    return retData

分析代码发现，每次请求获取数据都需要带上’__VIEWSTATE’这个参数，这个参数是存放在页面，所以需要把‘__VIEWSTATE’提出出来，用于访问下一页的时候带到参数里面去。

验证码解析：

通过验证码的url地址，将验证码保存到本地，因为验证码是彩色的，所有需要先把验证码置灰，然后再调用图像识别转为数字。这个验证码为4位数字，但是调用图像识别的时候，可能会转成字母，所有手动将字母转为数字，转换后识别率还能接受。

def get_image(codeurl):
    print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) + " begin get code num")
    index = 1
    while index<=15:
        file = urllib2.urlopen(codeurl).read()
        im = cStringIO.StringIO(file)
        img = Image.open(im)
        imgName = "vrifycode/" + gl.timeStr + "_" + str(index) + ".png"
        print 'begin get vrifycode'
        text = convert_image(img, imgName)
        print "vrifycode", index, ":", text
        # logger.info('vrifycode' + str(index) + ":" + text)

        if (len(text) != 4 or text.isdigit() == False):  # 如果验证码不是4位那么肯定是错误的。
            print 'vrifycode:', index, ' is wrong'
            index += 1
            time.sleep(2)
            continue
        return text

#将图片转为数字
def convert_image(image,impName):
    print "enter convert_image"
    image = image.convert('L')  # 灰度
    image2 = Image.new('L', image.size, 255)
    for x in range(image.size[0]):
        for y in range(image.size[1]):
            pix = image.getpixel((x, y))
            if pix < 90:  # 灰度低于120 设置为 0
                image2.putpixel((x, y), 0)
    print "begin save"
    image2.save(impName)  # 将灰度图存储下来看效果
    print "begin convert"
    text = pytesseract.image_to_string(image2)
    print "end convert"
    snum = ""
    for j in text:#进行简单转换
        if (j == 'Z'):
            snum += "2"
        elif (j == 'T'):
            snum += "7"
        elif (j == 'b'):
            snum += "5"
        elif (j == 's'):
            snum += "8"
        elif (j == 'S'):
            snum += "8"
        elif (j == 'O'):
            snum += "0"
        elif (j == 'o'):
            snum += "0"
        else:
            snum += j
    return snum

数据转换：

将html数据转换为数组，供保存csv文件和数据库时使用

def paras_data(nameList,logger):
    data = []
    mainlist = nameList
    rows = mainlist.findAll("tr", {"class": {"row", "alter"}})
    try:
        if (len(rows) != 0):
            for name in rows:
                tds = name.findAll("td")
                if tds == None:
                    print "get tds is null"
                    logger.info("get tds is null")
                else:
                    item = []
                    for index in range(len(tds)):
                        s_span = (tds[index]).find("span")
                        if (s_span is not None):
                            tmp = s_span["title"]
                        else:
                            tmp = (tds[index]).get_text()
                        # tmp=(tds[index]).get_text()
                        item.append(tmp.encode('utf-8'))  # gb2312  utf-8
                    item.append(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))#本条数据获取时间
                    data.append(tuple(item))

    except Exception as e:
        print "catch exception when save csv", e
        logger.info("catch exception when save csv" + e.message)
    return data

保存csv文件：

def save_to_csv(data ,writer):
    for d in data:
        if d is not None:
            writer.writerow(d)

保存数据库：

def save_to_mysql(data,conn,cursor):
    try:
        cursor.executemany(
            "INSERT INTO `aaa`(aaa,bbb) VALUES (%s,%s)",
            data)
        conn.commit()

    except Exception as e:
        print "catch exception when save to mysql",e
    else:
        pass

保存指定页数据：

'''
遇到不懂的问题？Python学习交流群：821460695满足你的需求，资料都已经上传群文件，可以自行下载！
'''
def get_appointed_page(snum,opener,vs,logger):
    tdd = get_tdd()
    tdd["__VIEWSTATE"] = vs['__VIEWSTATE']
    tdd["__EVENTARGUMENT"] = snum
    tdd=urllib.urlencode(tdd)
    # print "tdd",tdd
    op = opener.open(gl.productUrl, tdd)
    if (op.getcode() != 200):
        print("the" + snum + " page ,state not 200,try connect again")
        return None
    data = op.read().decode('utf-8', 'ignore')
    # print "data",data
    bsObj = BeautifulSoup(data,"lxml")
    nameList = bsObj.find("table", {"class": "mainlist"})
    # print "nameList",nameList
    if len(nameList) == 0:
        return None
    viewState = bsObj.find(id="__VIEWSTATE")
    if viewState is None:
        logger.info("the other page,no viewstate,try connect again")
        print("the other page,no viewstate,try connect again")
        return None
    vs['__VIEWSTATE'] = viewState["value"]
    return nameList

Main方法：

while flag == True and logintime <50:
            try:
                print "global login the ", str(logintime), " times"
                logger.info("global login the " + str(logintime) + " times")
                bsObj = get_logined_Data(opener, logger,views)
                if bsObj is None:
                    print "try login 15 times,but failed,exit"
                    logger.info("try login 15 times,but failed,exit")
                    exit()
                else:
                    print "global login the ", str(logintime), " times succesfully!"
                    logger.info("global login the " + str(logintime) + " times succesfully!")
                    viewState_Source = bsObj.find(id="__VIEWSTATE")
                    if totalNum == -1:
                        totalNum = get_totalNum(bsObj)
                        print "totalNum:",str(totalNum)
                        logger.info("totalnum:"+str(totalNum))
                    vs = gl.vs
                    if viewState_Source != None:
                        vs['__VIEWSTATE'] = viewState_Source["value"]

                    # 获取指定snum页的数据
                    # while snum<=totalNum:
                    while snum<=totalNum:
                        print "begin get the ",str(snum)," page"
                        logger.info("begin get the "+str(snum)+" page")
                        nameList = get_appointed_page(snum, opener, vs, logger)
                        if nameList is None:
                            print "get the nameList failed,connect agian"
                            logger.info("get the nameList failed,connect agian")
                            raise Exception
                        else:
                            print "get the ", str(snum), " successfully"
                            logger.info("get the " + str(snum) + " successfully")

                  
                            mydata = paras_data(nameList,logger)
                            #保存CSV文件
                            save_to_csv(mydata, snum, writer)
                            #保存到数据库
                            save_to_mysql(mydata, conn, cursor)

                            snum+=1
                            time.sleep(3)

                flag = False
            except Exception as e:
                logintime+=1
                print "catch exception",e
                logger.error("catch exception"+e.message)

定时任务设置：

cd /var/spool/cron/

crontab –e#编辑定时任务

输入：1 1 1 * * /yourpath/normal_script.sh>>/yourpath/cronlog.log  2>&1

（上面定时任务的意思是每月1号1点1分执行文件normal_script.sh，日志存放在cronlog.log）

目录结构：