python爬虫第一课,制作搜索引擎

from BeautifulSoup import *
from urlparse import urljoin

ignaorewords=set(['the','of','to','and','a','in','is','it'])

我们的搜索引擎基于关键词, 所以将连词,冠词忽略

下面的代码是爬虫, 将网页的文本数据存储到我们的sqlite中, 大家看不懂也没有关系, 知道这些函数是干什么的就行了

from sqlite3 import dbapi2 as sqlite
import urllib2
class crawler:
    def __init__(self,dbname):
        self.con=sqlite.connect(dbname)
        #连接并建立数据库, dbname 随意, 'xxx.db'就可以
    def __del__(self):
        self.con.close()
    def dbcommit(self):
        self.con.commit()
    
    def getentryid(self,table,field,value,createnew=True):
        cur=self.con.execute(
            "select rowid from %s where %s='%s'" %(table,field,value))
        res=cur.fetchone()
        if res==None:
            cur=self.con.execute(
                "insert into %s (%s) values ('%s')" % (table,field,value))
            return cur.lastrowid
        else:
            return res[0]
    
    
    def addtoindex(self,url,soup):
        if self.isindexed(url): return
        print 'Indexing',url
        
        #Get words
        text=self.gettextonly(soup)
        words=self.separatewords(text)
        
        #Get URL id
        urlid=self.getentryid('urllist','url',url)
        
        # Link word to url
        for i in range(len(words)):
            word=words[i]
            if word in ignaorewords: continue
            wordid=self.getentryid('wordlist','word',word)
            self.con.execute("insert into wordlocation(urlid,wordid,location) \
            values(%d,%d,%d)" % (urlid,wordid,i))
            
            
    
    def gettextonly(self,soup):
        v=soup.string
        if v==None:
            c=soup.contents
            resulttext=''
            for t in c: