小爬虫学到的一点东西

这几天学了一下怎么写爬虫爬百度的页面，借同学的代码看了看，学到了点东西，记一下。

我写的爬虫很简单，就是get传入搜索内容，然后爬取response里的网址就行了。需要注意的是：GET传参时需要带上cookie，不然不会返回搜索结果。

然后用正则表达式匹配这里的html代码就行。注意，这里的正则表达式需要用re.findall()，re.search()只会返回匹配到的第一串字符。且re.findall()没有.group()的用法。

有的爬取到的页面有&nbsp的字符，需要进行优化，用如下代码来实现：

x = re.findall('(.*?)/*? ', i)#i是刚刚爬取的url
        if x:
            with open(savepath, 'a+') as file:
                print (x[0])
                file.writelines(x[0]+"\n")
        else:
            print ("no")
            if len(i) < 200 & len(re.findall('[\u4e00-\u9fa5]', i)) == 0:
                print(i)

有些结果不是一个url，而是一段经过HTML或者css装饰过的，就干脆删除了。

爬虫源码如下：

# -*- coding: utf-8 -*-
import requests
import re

savepath = ""
key = ""#输入要搜索的内容
def craw(url_input):
    url_test = url_input
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/73.0.3683.103 Safari/537.36',
    'Cookie':'Cookie: BAIDUID=95138E5B7E16A449906E72DC2630BEB0:FG=1; BIDUPSID=5DD767C133912FC046EB0DABF56F2D53; PSTM=1572007988; BDUSS=FPTzdFZjVuTHN0aEJQdDBpeHFDMVdhU1hEVG1NV0FpRmVsZkYtOG5veTJvTlpkSVFBQUFBJCQAAAAAAAAAAAEAAADmEUfTVGhpc19pc19ZXwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALYTr122E69dTE; H_WISE_SIDS=141749_142080_141256_142209_142062_142115_135847_141003_138596_138078_142170_133995_138878_137978_141200_140173_131246_137745_138165_107320_138883_140259_141837_140631_140202_136863_138585_141651_140988_140114_140327_140578_133847_140066_134046_131423_140368_137815_140966_137703_136753_110085_141942_127969_140593_139885_138425_138942_141190_142160_141925; BD_UPN=13314752; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; ORIGIN=2; ISSW=1; ISSW=1; BDSFRCVID=zFFOJeCAa7tU7vouiu5VUUWq5mKK0gOTH6Hh_CKKSacFeLIVJG1BEG0Pqf8g0Ku-O295ogKKQgOTHR_F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=JJ4O_C-5tCvDqTrP-trf5DCShUFs-JDJB2Q-XPoO3KO1sJcP5J85DfkA3JJh3jjiWbRT_MbgylRp8P3y0bb2DUA1y4vpWttO22TxoUJ23I31O-jDqfv65xtebPRiJPb9Qg-qahQ7tt5W8ncFbT7l5hKpbt-q0x-jLTnhVn0MBCK0MCt9jj8bDjPVKgTa54cbb4o2WbCQWlnd8pcN2b5oQTtnQf6a-R0etmOP-x3D0DovOIJMjpOUWfAkXpJvQnJjt2JxaqRCBJ8Wjp5jDh3MQ5-nyhQlex7gbe7y0hvctn3cShPCyUjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t2XjQhjG0Ot60ftR3aQ5rtKRTffjrnhPF3KUumXP6-hnjy3b77KRctb4-5hJRa3qJtL6DUyN3Tth3RymJ42-39LPO2hpRjyxv4X4JbMPoxJpOJ3nv7aPjIHlR6flOvbURvD--g3-AqBM5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIE_CI5JDIMbPcdMtOHhICV-frb-C62aKDsKKoYBhcqEpjpjnOljnKe5frtB5oPQGvD_R6cWKJJ8UbSjx7sBUtUynbK5M5-Bg5paq0h0q5nhMJlb67JDMP0qtPHq6Qy523ibR3vQpnC_pQ3DRoWXPIqbN7P-p5Z5mAqKl0MLPbtbb0xXj_0jjcWjGuJqbT32CnKW5rtKRTffjrnhPF32xKzXP6-3MoKJ5nP_-QObRThMfQa3qJajtuUyN3MWh37Jeb3MhOcWMOHHpQNXMr4yxbLL4oxJpOD5JbMopvaHlAKbD5vbURv0t-g3-7NyM5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIE3-oJqC_WhCDm3j; H_PS_PSSID=; ZD_ENTRY=baidu; BDRCVFR[VXHUG3ZuJnT]=mk3SLVN4HKm; delPer=0; BD_CK_SAM=1; PSINO=6; H_PS_645EC=9623chClYIcdzOwa8seCfXVZl6c3Y38NoxnGuZR6jEupFxBaHwLbMpmvbYtPrCFChz8wgzEKERfh'}

    repose = requests.get(url=url_test, headers=headers)
    return repose.text

def bd_spider(key_word, page):
    """
    key_word:关键字
    page:页数
    """
    url = 'https://www.baidu.com/s?wd={}&pn={}'.format(key_word, page)
    text = craw(url)
    result = re.findall('class="c-showurl" style="text-decoration:none;">(.*?)</a>', text)
    for i in result:
        x = re.findall('(.*?)/*?&nbsp;', i)
        if x:
            with open(savepath, 'a+') as file:
                print (x[0])
                file.writelines(x[0]+"\n")
        else:
            if len(i) < 200 & len(re.findall('[\u4e00-\u9fa5]', i)) == 0:
                print(i)

for i in range(0, 10000, 10):
    bd_spider(key, i)

这个爬虫有时候爬网址怕不全，如下图，因为baidu的显示不全，这我也没办法了。。

而且由于爬虫有可能会爬到相同网站的不同页面，于是我就再写了一个效率比较低的脚本来去重，整体思路很简单，就是打开去重前的文件，进行查找，如果去重后的文件里没有就写入，有就跳过。

但是我的代码是遍历文件，比较低效，只好以后慢慢优化了。

代码如下：

import re

before_del_path = "G:/Python_Practice/safy/1.txt"
after_del_path = "G:/Python_Practice/safy/2.txt"
url=""

with open(before_del_path,"r") as f:
    before_do = f.readlines()
    for i in before_do:
        flag = True
        if(re.search("http://(.*)",i)):
            url = re.search("http://(.*)",i).group(1)#判断是否为http开头
        elif(re.search("https://(.*)",i)):
            url = re.search("https://(.*)",i).group(1)#判断是否为https开头
        else:
            url = i
        url = url.split("/")切割url
        file = open(after_del_path,"r")
        result = file.readlines()
        for j in result:
            if (bool(re.search(url[0],j))):
                file.close()
                flag = False
                break
        if(flag):
            file.close()
            do = open(after_del_path, "a+")
            do.write(url[0])
            do.write("\n")
            do.close()