python爬虫代码实例源码_PYTHON爬虫示例

【实例简介】

【实例截图】

【核心代码】

#encoding:utf-8

#!/usr/bin/env python

from urllib import urlretrieve

from time import ctime, sleep

from random import randint

from operator import add, mul

from functools import partial

def firstNonBlank(lines):

for eachLine in lines:

if not eachLine.strip():

continue

else:

#sleep(1)

#print eachLine

return eachLine

def firstLast(webpage):

print 'webpage= ', webpage

f = open(webpage)

lines = f.readlines()

#print 'lines=',lines

f.close()

print firstNonBlank(lines),lines.reverse()

print firstNonBlank(lines),

def download(url='http://www.baidu.com/', process=firstLast):

try:

retval = urlretrieve(url)[0]

print 'retval= ',retval

except IOError:

retval = None

if retval: # do some processing

process(retval)

def newfoo(arg1, arg2, *nkw, **kw):

'''display regular args and all variable args'''

print 'arg1 is:', arg1

print 'arg2 is:', arg2

for eachNKW in nkw:

print 'additional non-keyword arg:', eachNKW

for eachKW in kw.keys():

print "additional keyword arg '%s': %s" % \

(eachKW, kw[eachKW])

def testit(func, *nkwargs, **kwargs):

try:

retval = func(*nkwargs, **kwargs)

result = (True, retval)

except Exception, diag:

result = (False, str(diag))

return result

def test():

funcs = (int, long, float)

vals = (1234, 12.34, '1234', '12.34')

for eachFunc in funcs:

print '-' * 20

for eachVal in vals:

retval = testit(eachFunc, eachVal)

if retval[0]:

print '%s(%s) =' % \

(eachFunc.__name__, `eachVal`), retval[1]

else:

print '%s(%s) = FAILED:' % \

(eachFunc.__name__, `eachVal`), retval[1]

'''jishu'''

def odd(n):

return n % 2

def counter(start_at=0):

count = [start_at]

def incr():

count[0] = 1

return count[0]

return incr

if __name__ == '__main__':

allNums = []

for eachNum in range(9):

radNum= randint(1, 99)

allNums.append(radNum)

print '[%02d]=[%02d]' % (eachNum,radNum)

print [n for n in allNums if n%2]

print filter(odd, allNums)

map1 = map((lambda x: x 2), [0, 1, 2, 3, 4, 5])

map2 = map(lambda x, y: x y, [1,3,5], [2,4,6])

print map1

print map2

print reduce((lambda x,y: x y), range(5))

baseTwo = partial(int, base=2)

baseTwo.__doc__ = 'Convert base 2 string to an int.'

baseTwo('10010')

print 'use Closures'

count = counter(5)

print count()

print count()

print count()

print 'end use Closures'

test()

newfoo('wolf', 3, 'projects',1,2,3, freud=90, gamble=96)

aTuple = (6, 7, 8)

aDict = {'z': 9}

newfoo(1, 2, 3, x=4, y=5, *aTuple, **aDict)

download()