python采集网页到后台_Python Post and Get 登陆web后台系统并抓取页面

#coding=utf8

#! /usr/bin/env python

import httplib

import re

import socket

import urllib

timeout = 60

socket.setdefaulttimeout(timeout)

def getTable():

f = open('kvpage.html')

page = f.readlines()

f.close()

pattern = re.compile(r'.*

(.*?).*')

for line in page:

#print line

m = pattern.match(line.strip())

if m is not None:

return m.group(1)

return None

def extractKvEvents(content):

#init result

table = []

#init pattern

patternTR = re.compile(r"

(.*?)")

patternTD = re.compile(r'

(.*?)')

#search all the rows

allrows = patternTR.findall(content)

if allrows is not None:

for row in allrows:

#print row

cols = patternTD.findall(row)

if cols is not None:

table.append(cols)

return table

def outputToExcel(table):

for row in table:

print row

def loginWiki():

httpClient = None

try:

params = urllib.urlencode({'os_username': 'xxxx@xxx.com',

'os_password': 'xxxx',

'login': 'Log In'})

headers = {"Content-type": "application/x-www-form-urlencoded"

, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"}

httpClient = httplib.HTTPConnection("xxx.com", 8080, timeout=30)

httpClient.request("POST", "/login.action", params, headers)

response = httpClient.getresponse()

# print response.status

# print response.reason

# print response.read()

# print response.getheaders()

print response.getheader('Set-Cookie')

cookieFile = open('cookie.txt', 'w')

cookieFile.write(response.getheader('Set-Cookie'))

cookieFile.close()

except Exception, e:

print e

finally:

if httpClient:

httpClient.close()

def catchPage():

httpClient = None

try:

#read cookie

f = open('cookie.txt')

cookie = f.read().strip()

print cookie

f.close()

#init headers

headers = {"Content-type": "application/x-www-form-urlencoded",

"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",

'Cookie': cookie}

#send request

httpClient = httplib.HTTPConnection('xxx.com', 8080, timeout=30)

httpClient.request('GET', '/xxxPath', headers=headers)

#response是HTTPResponse对象

response = httpClient.getresponse()

print response.status

print response.reason

htmlPage = open('kvpage.html', 'w')

htmlPage.write(response.read())

htmlPage.close()

except Exception, e:

print e

finally:

if httpClient:

httpClient.close()

if __name__ == '__main__':

loginWiki()

catchPage()

tablecontent = getTable()

table = extractKvEvents(tablecontent)

outputToExcel(table)