python之自定义爬虫脚本

原创 2015年11月18日 23:35:00

## spider.py


import urllib2,httplib
import os,sys,re,time,random



#[OK]
def removeBR(site):
    result = re.search(r'\n',site)
    if result:
        return site.replace('\n','')
    return site
#[OK]
def getRequestList(fname):
    file = open(fname,"r")
    sites = file.readlines()
    list = []
    for s in sites:
        list.append(removeBR(s))
    return list
#[OK]
def inOldSet(url,set):
    if url in set:
        return True
    return False
#[]

#[OK]
def isEndWithJavascript(site):
    ret = re.search("/javascript:",site)
    if ret:
        return True
    return False

#[OK]
def ignoreJavascript(site):
    if isEndWithJavascript(site):
        s1,s2 = site.rsplit("/javascript:",1)
        return s1
    return site
#[50%_OK]
def isFile(url):
    res = re.search("(.html|.htm|.xml|.txt|.css|.js|.avi|.flv|.jpg|.gif|.bmp|.png|.xhtml|.dat|.doc|.xls|.php|.jsp|.asp)$",url)
    if res:
        return True
    return False
#[OK] http://aa.com/index.txt -> http://aa.com
def getBaseUrl(url):
    if isFile(url):
        s1,s2 = url.rsplit("/",1)
        return s1
    if url[len(url) - 1:] == "/":
        return url[:len(url) - 1]
    else:
        return url
#[OK]
def addIndexForSite(site):
    if isFile(site):
        return site
    bs = getBaseUrl(site)
    return "%s/index.html" % (bs)
#[OK]
def isError(site):
    try:
        getCntList(site)
        return False
    except:
        return True
#[OK]
def isPHPIndex(site):
    site = addIndexForSite(site)
    if isError(site):
        return True
    else:
        return False
#[OK]
def pathToFile(site):
    if isFile(site):
        return site
    bs = getBaseUrl(site)
    ext = "index.html"
    if isPHPIndex(site):
        ext = "index.php"
    return "%s/%s" % (bs,ext)
#[OK] ./a/c.txt  /a/b.txt a/v.txt -> a/a.txt
def getRelPath(sub):
    if sub[:1] == ".":
        if sub[1:2] == ".":
            return sub
        if sub[1:2] == "/":
            return sub[2:]
        return sub
    return sub
#[OK]
def getDomain(url):
    if isFullPath(url):
        s1,s2 = url.split("http://",1)
        if s2.find(r'/') != -1:
            s3,s4 = s2.split(r'/',1)
            return s3
        else:
            return s2
#[OK]
def toFullPath(sub,parent):
    if isFullPath(sub):
        return sub
    if sub == "/":
        return "http://%s" % (getDomain(parent))
    if sub[:1] == "/":
        baseUrl = getDomain(parent)
    else:
        baseUrl = getBaseUrl(parent)
    relPath = getRelPath(sub)
    return "http://%s/%s" % (baseUrl,relPath)
#[OK]
def formatUrl(url):
    url = ignoreJavascript(url)
    #if not isFile(url):
    #    url = pathToFile(url)
    return url
#[OK]
def isFullPath(url):
    if re.match("\s*http://",url):
        return True
    return False
#[OK]
def isPHPFile(site):
    ret = re.search("\.php\?",site)
    if ret:
        return True
    ret = re.search("\.php$",site)
    if ret:
        return True
    return False
#[OK?]
def isValidHTML(url):
    if isPHPFile(url) or not isFullPath(url):
        return False
    return True
def getCntList(url):
    list = []
    try:
        cnt = urllib2.urlopen(url,timeout=240).read()
        cnt = cnt.replace('\n','')
        cnt = cnt.replace('<a ','\n<a ')
        cnt = cnt.replace('<img ','\n<img ')
        sites = cnt.split('\n')
        for s in sites:
            if s:
                #print removeBR(s) + "\n\n"
                list.append(removeBR(s))
    except:
        print "load error~2"
    return list
#[OK]
def getAnchorHref(str):
    return re.finditer(r'<a\s.*href="([^<>\s\"]+)"',str)
def getImgSrc(str):
    return re.finditer(r'<img\s.*src="([^<>\s\"]+)"',str)
#[OK]
def isImage(url):
    res = re.search("(.jpg|.JPG|.gif|.GIF|.bmp|.BMP|.png|.PNG|.jpeg|.JPEG)$",url)
    if res:
        return True
    else:
        return False
#[OK]
def isHTML(url):
    res = re.search("(.html|.htm|.asp|.aspx)$",url)
    if res:
        return True
    else:
        return False
gcnt = 0
#[]
def parseHTML(url):
    global gcnt
    gcnt = gcnt + 1
    print "parsing[%d]-->%s" % (gcnt,url)
    hSet = []
    iSet = []
    res = []
    list = getCntList(url)
    for s in list:
        anchor = getAnchorHref(s)
        for a in anchor:
            print "anchor[sub]->%s" % (a.group(1))
            print "fullPath[sub]->%s" % (formatUrl(toFullPath(a.group(1),url)))
            res.append(formatUrl(toFullPath(a.group(1),url)))
        img = getImgSrc(s)
        for im in img:
            print "image[sub]->%s" % (im.group(1))
            res.append(formatUrl(toFullPath(im.group(1),url)))
    for r in res:
        if isImage(r):
            iSet.append(r)
        if isHTML(r):
            hSet.append(r)
    return iSet,hSet
#[OK]
def updateUrlSet(urlSet,htmlSet,oldSet):
    for h in htmlSet:
        if h in oldSet:
            continue
        if h in urlSet:
            continue
        urlSet.append(h)
#[OK]
def getExt(url):
    s1,s2 = url.rsplit(".",1)
    return s2
#[OK]
def getRandomName():
    return "%d_%d" % (time.time(),random.randint(0,99999))
#[OK]
def getPath(url):
    name = getRandomName()
    ext = getExt(url)
    return "%s.%s" % (name,ext)
#[OK]
def getUrl(url):
    try:
        return urllib2.urlopen(url,timeout=240).read()
    except:
        print "load error~"
        return None
#[OK]
def saveToRoot(str,path):
    newPath = os.path.join(os.getcwd(),"download")
    newPath = os.path.join(newPath,path)
    fp = open(newPath,"wb")
    fp.write(str)
    fp.close()
#[OK]
def saveImage(imgSet):
    for site in imgSet:
        print "Get-->%s" % (site)
        page = getUrl(site)
        if page == None:
            continue
        path = getPath(site)
        print "Save-->%s" % (path)
        saveToRoot(page,path)
#[Test]
def main(fname):
    urlSet = getRequestList(fname)
    oldSet = []
    for url in urlSet:
        if inOldSet(url,oldSet):
            del(url)
            continue
        newUrl = formatUrl(url)
        #print "newUrl::" + newUrl
        #newUrl = ignoreJavascript(url)
        if isValidHTML(newUrl):
            imgSet,htmlSet = parseHTML(newUrl)
            saveImage(imgSet)
            oldSet.append(url)
            oldSet.append(newUrl)
            updateUrlSet(urlSet,htmlSet,oldSet)
        else:
            del(url)

def test(set):
    for t in set:
        print formatUrl(removeBR(t))

main('list.txt')


## list.txt

http://www.gaoxiaola.com/p/gif/index_4.html


@@用法说明

spider.py list.txt

试图从网页中抓取图片

版权声明:本文为博主原创文章,未经博主允许不得转载。

相关文章推荐

[Python]linux自定义Python脚本命令

在window下写好的程序配置到Linux上,要实现任意目录下的命令调用。 由于初学Linux,这里从文件传输等最基本的方法入手,记录配置的过程中遇到的各种问题。

python爬虫——校园网自动重连脚本

一、背景最近学校校园网不知道是什么情况,总出现掉线的情况。每次掉线都需要我手动打开web浏览器重新进行账号密码输入,重新进行登录。系统的问题我没办法解决,但是可以写一个简单的python脚本用于自动登...

yum源python爬虫脚本

python 网站爬虫 下载在线盗墓笔记小说到本地的脚本

python 网络爬虫 下载在线小说 ..................最近闲着没事想看小说,找到一个全是南派三叔的小说的网站,决定都下载下来看看,于是动手,在很多QQ群里高手的帮助下(本人正则表达...

【选课脚本】用Python网页爬虫来进行选(qiang)课 (更新至v1.0.7)

每当选课的时候,都如同打仗一般 都有自己想要的课,但是名额就那么一点 于是各显神通,有人用js,有人用chrome的console 人生苦短,我用Python
  • okcd00
  • okcd00
  • 2017-06-01 08:53
  • 1250

【Python脚本】-爬虫得到CSDN博客的文章访问量和评论量

对于CSDN博客,我们比较关注的就是文章的访问量和评论量。但是当文章多了之后,我们想看每篇文章的访问量变得很费劲。通过爬虫,我们可以把每篇博客的基本信息都能得到。之后,可以再进行进一步的统计分析。脚本...
内容举报
返回顶部
收藏助手
不良信息举报
您举报文章:深度学习:神经网络中的前向传播和反向传播算法推导
举报原因:
原因补充:

(最多只允许输入30个字)