Python爬虫：基于Scrapy的爬取失踪人口数据小爬虫

Scrapy简介：

Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。 其可以应用在数据挖掘，信息处理或存储历史数据等一系列的程序中。其最初是为了页面抓取 (更确切来说, 网络抓取 )所设计的， 也可以应用在获取API所返回的数据(例如 Amazon Associates Web Services ) 或者通用的网络爬虫。Scrapy用途广泛，可以用于数据挖掘、监测和自动化测试。

仓库地址：

https://github.com/XiaoZhong233/missingPerson_spider

---

说明：

1、数据来源：宝贝回家网

2、数据后处理中加入了经纬度坐标，方便在空间上定位，原始数据是没有这个信息的

 

技术点：

1. scrapy爬虫框架
2. mysql连接池技术
3. 数据清洗（正则表达式）
4. 百度API地理编码

Spider的工作流程，就不具体说了，他是一个很强大的爬虫框架。

 

运行结果：

 

爬虫主要代码：

# -*- coding: utf-8 -*-

#更改搜索路径，但没有用哦
import random
import sys
import os

from PersonItem import PersonItem

curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)


import scrapy
import json
import re
from urllib.request import urlopen, quote
import datetime
import time

#E:\Desktop\python\zhiyuan_test\zhiyuan_test

#
# PATH = os.environ
# for key in PATH:
#     print(key, PATH[key])

#print(rootPath)
#百度坐标地理编码
def getBlnglat(address):
    url = 'http://api.map.baidu.com/geocoder/v2/'
    output = 'json'
    ak = 'O8fpFPd6pw5VYK1uGsdMSAtL5MQRjAjs'
    add = quote(address)  #为防止乱码，先用quote进行编码
    uri = url + '?' + 'address=' + add + '&output=' + output + '&ak=' + ak
    req = urlopen(uri)
    res = req.read().decode()
    temp = json.loads(res)
    # print(temp.keys())
    # print(temp.values())
    # print(type(temp['result']))
    if temp['status'] == 0:
        lng = temp['result']['location']['lng']  # 纬度
        lat = (temp['result']['location']['lat'])  # 经度
        # print(lng, lat)
        return str(lng) + "," + str(lat)
    else:
        # print(address+"没找到")
        return "NULL"








#爬取失踪人口信息
class MissingpersonSpider(scrapy.Spider):
    name = 'missingPerson'
    allowed_domains = ['baobeihuijia.com']
    #https://bbs.baobeihuijia.com/thread-402350-1-1.html
    start_urls = []


    #读取本地数据
    # path = r".\missingUrl.json"
    # with open(path, 'rb') as f:
    #     data = json.load(f)
    #     #print(type(data))
    # len(data)
    # for index in range(6, 20):
    #     #str = re.match('.*', url)
    #     start_urls.append(data[index]["detailUrl"])
    # print(start_urls)

    #读取url数据库
    import MySQLdb
    #打开数据库连接
    db = MySQLdb.connect("localhost", "root", "root", "webgisdb", charset='utf8')
    cursor = db.cursor()
    sql = "select * from url"
    try:
        cursor.execute(sql)
        results = cursor.fetchall()
        for row in results:
            start_urls.append(row[0])
            print(row[0])
    except:
        print
        "Error: unable to fecth data"
    db.close()

    def string_toDatetime(st):
        print("2.把字符串转成datetime: ", datetime.datetime.strptime(st, "%Y-%m-%d"))

    def parse(self, response):

        mycsv = ['NULL', 'NULL', 'NULL', 'NULL', 'NULL',"NULL", 'NULL', 'NULL','NULL']  # 初始化提取信息列表
        # 爬取图片url
        imgurl=""
        img = response.xpath('//body//div[@class="t_fsz"]//ul/li//ignore_js_op/img/@file').extract();
        if len(img)>0:
            #print(img, "图片")
            #这里拉了多张图片
            for s in img:
                #str = img[0]
                rs=s.split("forum")
                tem = "https://youtu.baobeihuijia.com/forum" + rs[1]
                if imgurl=="":
                    imgurl=tem
                else:
                    from macpath import join
                    imgurl=join(",".join((imgurl,tem)))

            #print(rs[1])
        else:
            imgurl="http://www.svmuu.com/img/4.0/backstage/no-data.jpg"
            pass
        #print(imgurl)


        #爬取文字信息
        for li in response.xpath('//body//div[@class="t_fsz"]//ul/li'):

            # 伪装
            from scrapy.conf import settings
            origin = "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/"
            ran = random.randint(10, 100000)
            origin += str(ran)
            settings['BOT_NAME'] = origin

            info = li.xpath('font/text()').extract()
            #print(info)

            if(info!=None):
                for item in info:
                    if "失踪日期" in item[0:10] or "失踪日期" in item[0:10]:
                        index = item.find("：")
                        missingdate=item[index + 1:]


                        #正则化匹配
                        date_reg_exp = re.compile('\d{4}[-/年]\d{1,2}[-/月]\d{2}\s*')
                        matches_list = date_reg_exp.findall(missingdate)
                        #for match in matches_list:
                        match = None
                        if len(matches_list)==0:
                            #print("正则表达式1无匹配"+missingdate+"，尝试2")
                            date_reg_exp1= re.compile('\d{4}[-/.,年]\d{1,2}[-/.,月]\s*')
                            matches_list = date_reg_exp1.findall(missingdate)
                            if len(matches_list)==0:
                                #print("正则表达式2无匹配"+missingdate+"，尝试3")
                                date_reg_exp2 = re.compile('\d{4}[年]\s*')
                                matches_list = date_reg_exp2.findall(missingdate)
                                if len(matches_list)==0:
                                    #print(missingdate+"无匹配")
                                    match = None
                        else:
                            #print(matches_list[0])
                            match= matches_list[0]
                        times = None
                        #解析日期

                        if match!=None:
                            try:
                                times = time.strptime(match, "%Y年%m月%d")
                            except ValueError:
                                try:
                                    times = time.strptime(match, "%Y-%m-%d")
                                except ValueError:
                                    try:
                                        times = time.strptime(match, "%Y/%m/%d")
                                    except ValueError:
                                        try:
                                            times = time.strptime(match, "%Y年%m月%d日 ")
                                        except ValueError:
                                            try:
                                                times = time.strptime(match, "%Y-%m-%d ")
                                            except ValueError:
                                                try:
                                                    times = time.strptime(match, "%Y/%m/%d ")
                                                except ValueError:
                                                    #print("错误的日期格式或日期"+match)
                                                    times=None

                        mycsv[8] = times
                        #print(missingdate)
                        pass
                    if "姓    名" in item[0:6]:
                        index = item.find("：")
                        missingName= item[index + 1:]
                        mycsv[0]= missingName
                    if "性    别" in item:
                        index = item.find("：")
                        sex=item[index + 1:]
                        mycsv[1] = sex
                    if "出生日期" in item:
                        index = item.find("：")
                        birth = item[index + 1:]
                        #正则化匹配
                        date_reg_exp = re.compile('\d{4}[-/年]\d{1,2}[-/月]\d{2}\s*')
                        matches_list = date_reg_exp.findall(birth)
                        #for match in matches_list:
                        match = None
                        if len(matches_list) == 0:
                            #print("正则表达式1无匹配"+birth+"，尝试2")
                            date_reg_exp1 = re.compile('\d{4}[-/年]\d{1,2}[-/月]\s*')
                            matches_list = date_reg_exp1.findall(birth)
                            if len(matches_list) == 0:
                                #print("正则表达式2无匹配"+birth+"，尝试3")
                                date_reg_exp2 = re.compile('\d{4}[年]\s*')
                                matches_list = date_reg_exp2.findall(birth)
                                if len(matches_list) == 0:
                                    #print(birth+"无匹配")
                                    match = None
                        else:
                            #print(matches_list[0])
                            match= matches_list[0]
                        times = None
                        #解析日期

                        if match!=None:
                            try:
                                times = time.strptime(match, "%Y年%m月%d")
                            except ValueError:
                                try:
                                    times = time.strptime(match, "%Y-%m-%d")
                                except ValueError:
                                    try:
                                        times = time.strptime(match, "%Y/%m/%d")
                                    except ValueError:
                                        try:
                                            times = time.strptime(match, "%Y年%m月%d日 ")
                                        except ValueError:
                                            try:
                                                times = time.strptime(match, "%Y-%m-%d ")
                                            except ValueError:
                                                try:
                                                    times = time.strptime(match, "%Y/%m/%d" )
                                                except ValueError:
                                                    #print("错误是日期格式或日期")
                                                    times=None


                        #print(times)

                        #print(birth)
                        mycsv[2] = times
                    if "失踪时身高" in item:
                        index = item.find("：")
                        tall = item[index + 1:]
                        print("string:" + tall)
                        date_reg_exp = re.compile('\d{1,4}')
                        matches_list = date_reg_exp.findall(tall)
                        #print(matches_list)
                        matchTall = None
                        if len(matches_list) == 0:
                            print("身高正则匹配失败 string:"+ tall)
                        else:
                            matchTall=matches_list[0]
                            print("正则化字符串： "+matchTall)
                        mycsv[3] = matchTall
                    if "失踪地点" in item:
                        index = item.find("：")
                        adress = item[index + 1:]
                        mycsv[4] = adress
                        mycsv[5] = getBlnglat(adress)
                    if"失踪者特征描述" in item:
                        index = item.find("：")
                        describe = item[index + 1:]
                        mycsv[6] = describe
                    pattern = re.compile(r'^\d+')
                    detail = pattern.match(item)
                    if detail:
                        tem = mycsv[7]
                        if tem == "NULL":
                            tem = ""
                        s = tem+item
                        mycsv[7] = s

        #and mycsv[1] != "NULL" and mycsv[2] != "NULL"
        #整合信息
        if mycsv[0]!="NULL"  and mycsv[4]!="NULL" and mycsv[5]!="NULL" :
            print(mycsv)
            item = PersonItem()  # 实例化item类
            item['name'] = mycsv[0]
            item['sex'] = mycsv[1]
            item['birth'] = mycsv[2]
            item['tall'] = mycsv[3]
            item['adress'] = mycsv[4]
            item['cor'] = mycsv[5]
            item['describe'] = mycsv[6]
            item['other'] = mycsv[7]
            item['imgsrc'] = imgurl
            item['missingdate'] = mycsv[8]
            yield item