Step05：爬虫小项目，爬取最新电影迅雷下载地址

1.简述

由于电影天堂的广告实在令人不厌其烦，但其视频资源却的确有可取之处。因此，趁着学习爬虫技术的这段时间，简单实现了一个完整的小项目。
（完整代码——链接）

2.技术准备

IDE:Pycharm，python3.6.5，使用requests+re从电影天堂爬取最新电影资源的下载地址。使用tkinter设计简单的界面，中间还涉及使用了多线程技术，python对于多线程有threading库支持，简化了许多工作。

3.项目步骤

进入Pycharm建立project，实现以下目录结构：

\ThunderAndSpider\message_spider\spider_config.py

headers={
    'Cookie':'37cs_user=37cs63629906334; XLA_CI=3e976860bea5549a9a73e10df8153fcd; 37cs_pidx=2; 37cs_show=253%2C75; cscpvrich5041_fidx=3',
    'Host':'www.dytt8.net',
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding':'gzip, deflate',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6726.400 QQBrowser/10.2.2265.400',
}

other_headers={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6726.400 QQBrowser/10.2.2265.400',
    'Referer':'http://www.dytt8.net/',
    'Host':'www.dytt8.net',
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
}

url_dytt = "http://www.dytt8.net"
re_strIndex = r'>最新电影下载</a>]<a\shref=\'(.*?)\'>(.*?)</a><br'
re_strLink = r'<td\sstyle="WORD-WRAP.*?<a\shref=".*?">(.*?)</a></td>'

\ThunderAndSpider\message_spider\dytt_spider.py（简单实现的爬虫类）

import requests
import re

from message_spider.spider_config import *
from requests.exceptions import RequestException

class dytt_spider:
    def __init__(self):
        self.url = url_dytt

    def get_html(self, url, headers):
        try:
            response = requests.get(url, headers=headers)
            # Python HTTP库requests中文页面乱码解决方案
            response.encoding = response.apparent_encoding
            if response.status_code == 200:
                return response.text
            return None
        except RequestException:
            return None

    def _get_re_findall_items(self, html, re_str):
        pattern = re.compile(re_str, re.S)
        items = re.findall(pattern, html)
        return items

    def get_index(self, html, re_str):
        items = self._get_re_findall_items(html, re_str)
        for item in items:
            yield {
                "name": item[1],
                "url": url_dytt + item[0]
            }

    def get_thunder_link(self, html, re_str):
        items = self._get_re_findall_items(html, re_str)
        for item in items:
            yield {
                "thunder": item
            }

    def get_all_thunderlink(self):
        index = self.get_html(self.url, headers)
        for item in self.get_index(index, re_strIndex):
            html = self.get_html(item['url'], other_headers)
            if html:
                for x in self.get_thunder_link(html, re_strLink):
                    yield {
                        "影片名：": item["name"],
                        "磁力链接：": x['thunder'],
                    }

if __name__=="__main__":
    dytt = dytt_spider(url_dytt)
    for mess in dytt.get_all_thunderlink():
        print(mess)

\ThunderAndSpider\thunder\thunder_config.py

#==注意，此处要修改为迅雷所在目录的完整路径==
thunder_path = 'E:\Thunder.exe'
#==注意，此处修改为迅雷下载文件存放目录==
save_path= 'G:\\thunder_download\\'

\ThunderAndSpider\thunder\dytt_thunder.py

import os, time
#import threading
from thunder.thunder_config import *

class my_thunder:
    def __init__(self, url):
        self.url = url
        self.filename = os.path.split(self.url)[1]
        self.args = r'"{thunder_path}" {url}'.format(thunder_path=thunder_path, url=url)

    def start_target(self):
        print("准备下载---{name}".format(name=self.filename))
        os.system(self.args)
        #new_thread = threading.Thread(target=os.system, args=(self.args,))
        #new_thread.start()

    def check_start(self):
        check_file=self.filename+".xltd"
        return os.path.exists(os.path.join(save_path, check_file))

    def check_end(self):
        return os.path.exists(os.path.join(save_path, self.filename))
    '''
    def download(self):
        self.start_target()
        print("正在下载{name}".format(name=self.filename))
        if self.check_start():
            while True:
                time.sleep(60)
                if self.check_end():
                    print("下载完成")
                    return True
        else:
            print("下载失败")
            return False
    '''

\ThunderAndSpider\win_gui\main_gui.py（此处设计界面）

from tkinter import *

class MainGUI:
    def __init__(self):
        self.root=Tk()
        self.root.title("电影下载")
        self.root.geometry("700x500")
        self.root.resizable(False, False)
        self._set_gui()

    def open_gui(self):
        self.root.mainloop()

    def _set_gui(self):
        Label(self.root, text="资源来源：").grid(row=0, column=0)
        self.entry_01 = Entry(self.root)
        self.entry_01.grid(row=0,column=1,sticky=W)
        Label(self.root, text="资源种子：").grid(row=1, column=0)
        self.text_01 = Text(self.root)
        self.text_01.grid(row=1,column=1, sticky=W)
        Label(self.root, text="当前电影：").grid(row=2, column=0)
        self.entry_02 = Entry(self.root, width=300)
        self.entry_02.grid(row=2,column=1,sticky=W)
        self.frm = Frame(self.root)
        self.frm.grid(row=3, column=1, sticky=W)
        self.btn_01 = Button(self.frm, text="上一部")
        self.btn_01.grid(row=0, column=1)
        self.btn_02 = Button(self.frm, text="下一部")
        self.btn_02.grid(row=0, column=2)
        self.btn_03 = Button(self.frm, text="下载当前部")
        self.btn_03.grid(row=0, column=3)

\ThunderAndSpider\main.py

from win_gui.main_gui import *
from thunder.dytt_thunder import *
from message_spider.dytt_spider import *
from message_spider.spider_config import url_dytt

import threading

urls = []
link_message = ""
urls_index = 0

def get_urls_and_linkmessage(spider):
    global urls
    global link_message
    for x in spider.get_all_thunderlink():
        link_message += x["影片名："]
        link_message += "\n"
        link_message += x["磁力链接："]
        link_message += "\n\n"
        urls.append(x['磁力链接：'])

def change_entry_a(entry):
    global urls_index
    if urls_index == 0:
        urls_index = len(urls) - 1
    else:
        urls_index = urls_index - 1
    entry.delete(0, END)
    entry.insert(20, urls[urls_index])

def change_entry_b(entry):
    global urls_index
    if urls_index ==len(urls)-1:
        urls_index = 0
    else:
        urls_index = urls_index + 1
    entry.delete(0,END)
    entry.insert(20,urls[urls_index])

def download_current():
    thunder = my_thunder(urls[urls_index])
    #new_thread = threading.Thread(target=thunder.download)
    new_thread = threading.Thread(target=thunder.start_target)
    new_thread.start()
    #thunder.download()

def mainGUI_config(mainWin):
    # INSERT索引表示在光标处插入,END索引号表示在最后插入
    mainWin.entry_01.insert(END, url_dytt)
    mainWin.text_01.insert(1.0, link_message)
    mainWin.entry_02.insert(20, urls[0])
    mainWin.btn_01.config(command=lambda: change_entry_a(mainWin.entry_02))
    mainWin.btn_02.config(command=lambda: change_entry_b(mainWin.entry_02))
    mainWin.btn_03.config(command=download_current)

def main():
    print("++++主程序启动++++")
    mainWin = MainGUI()
    spider = dytt_spider()

    get_urls_and_linkmessage(spider)
    mainGUI_config(mainWin)
    mainWin.open_gui()

if __name__ =="__main__":
    main()

4.项目成果

目标站点：

项目主界面