爬取企业名录然后存储到数据库中。
#-*- coding:utf-8 -*-
import requests
import MySQLdb
import re
from bs4 import BeautifulSoup
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36"
}
cookies = {'ASPSESSIONIDQCSRSBRS':'FBEPJPLCHEEMEHNLHFKCBCGB',
'Hm_lvt_819e30d55b0d1cf6f2c4563aa3c36208':'1483118719',
'Hm_lpvt_819e30d55b0d1cf6f2c4563aa3c36208':'1483120442'}
db = MySQLdb.connect("localhost","root",'liao1234','liao',charset="utf8")
cursor = db.cursor()
'''
sql ="""create table company(name char(100) not null,
type char(50),
addr char(150),
leader char(30),
date char(30))"""
cursor.execute(sql)
'''
#获取各地区链接
r = requests.get("http://hangzhou.11467.com/",headers=headers,cookies=cookies)
html = r.text
soup = BeautifulSoup(html)
for tag in soup.find('div',class_='box sidesubcat t0').find_all('a'):
print tag.attrs['href']
base_url = "http://hangzhou.11467.com"+tag.attrs['href']+'pn'
for i in range(1,10):
url = base_url + str(i)
r1 = requests.get(url,headers=headers,cookies=cookies)
html1 = r1.text
soup1 = BeautifulSoup(html1)
for tag1 in soup1.find('ul',id='slist').find_all('li'):
ss = []
for cc in tag1.find_all('a'):
print cc.string
ss.append(cc.string)
for aa in tag1.find_all('dd'):
print aa.string
ss.append(aa.string)
if len(ss) == 3:
ss.append("none")
ss.append("none")
elif len(ss) == 4:
ss.append("none")
else:
pass
print len(ss)
if len(ss)==0:
continue
sql = "insert into company(name,type,addr,leader,date) values('%s','%s','%s','%s','%s')"%(ss[0],ss[1],ss[2],ss[3],ss[4])
cursor.execute(sql)
print ss[0]
db.close()