用python写自己的信息_第一次用python 写的简单爬虫 记录在自己的博客

#python.py

from bs4 import BeautifulSoup

import urllib.request

from MySqlite import MySqlite

global g_intid

g_intid=0

def GetBlogTileAndName(url):

res = urllib.request.urlopen(url)

html = res.read()

res.close()

str(html, 'utf-8')

soup=BeautifulSoup(html)

divs=soup.find_all(attrs={"class":"postTitle"})

for divname in divs:

print("title:=",divname.a.string,"href:=",divname.a["href"])

global g_intid

g_intid+=1

x=MySqlite()

x.InsertDate(g_intid,divname.a["href"],divname.a.string)

def GetBlogPage(url):

res = urllib.request.urlopen(url)

html = res.read()

res.close()

str(html, 'utf-8')

soup=BeautifulSoup(html)

divPager=soup.find(attrs={"class":"pager"})

print(divPager.string)

for i in range(1,8) :

url=r"http://www.cnblogs.com/FCoding/default.html?page="+str(i)

GetBlogTileAndName(url)

#MySqlite.py

class MySqlite(object):

"""description of class"""

def __init__(self, *args):

return super().__init__(*args)

def callstr(self,str):

print(str)

def InsertDate(self,id,url,title):

conn = sqlite3.connect(r"d:\123.db")

c=conn.cursor()

#try:

# c.execute('create table blog (ID intergeer,url text,title text , PRIMARY KEY(ID))')

#except ValueError:

# print("error My")

strExe="insert into blog values ({0}, \"{1}\",\"{2}\")".format(id,url,title)

print(id)

#c.execute('insert into blog values (last_insert_rowid(),url,title)')

c.execute(strExe)

conn.commit()

c.close()

conn.close()

def GetDate(self):

import sqlite3

conn = sqlite3.connect(r"d:\123.db")

c=conn.cursor()

res=c.execute("select count(*) from blog")

res=c.fetchone()

print(res[0])

data=c.execute("select * from blog")

for item in data:

for ite in item:

print(ite)

conn.commit()

c.close()

conn.close()

简述一下功能：

通过urllib 下载网页 使用BeautifulSoup 解析

调用find_all(attrs={"class":"postTitle"})

找到HTML 中所有class=posttitle 的tag

然后遍历 取出title 和href 保存到数据库中

此程序 无容错。新手无笑！