最近在学习python爬虫,就尝试着写了一个脚本去抓取某电影网站的ftp下载地址,下面是代码:
from multiprocessing import Process,Manager
import requests
from bs4 import BeautifulSoup
import random
import re
import time
def write_data_proc(dest_url_no,dest_url_over,dest_ftp_path,dest_file):
time.sleep(1)
num = random.randint(1,100)
print('子进程开始执行--%d'%num)
dest_url_file = dest_file["dest_url_file"]
dest_ftp_file = dest_file["dest_ftp_file"]
dest_url_handle = open(dest_url_file,"a+",encoding="utf-8")
dest_ftp_handle = open(dest_ftp_file,"a+",encoding="utf-8")
while len(dest_url_no) > 0:
dest_url = dest_url_no.pop(0)
try:
reponse = requests.get(dest_url,timeout=1.0)
except (TimeoutError,ConnectionError,requests.exceptions.ReadTimeout,requests.exceptions.ConnectionError,requests.exceptions.InvalidURL):
continue
reponse.encoding = 'gb2312'
soup = BeautifulSoup(reponse.text,"html.parser")
for tag in soup.find_all(re.compile(r"^a")):
if "href" in tag.attrs:
if tag.attrs["href"][0:3] == "ftp":
ftp_path = tag.attrs["href"]
if ftp_path not in dest_ftp_path:
dest_ftp_path.append(ftp_path)
dest_ftp_handle.write(ftp_path+"\n")
else:
url_path = tag.attrs["href"]
if url_path[0:4] == "http" or url_path[0:4] == "#" or url_path[0:4] =="/":
continue
if url_path[0:10] == "index.html" or url_path[0:10] == "javascript":
continue
if url_path[0:8] == "/support" or url_path[0:8] == "/webPlay" :
continue
if (url_path.find("game") != -1) or (url_path.find("magnet:?xt") != -1) or (url_path.find("thunder") != -1) or(url_path.find("ed2k://") != -1):
continue
url_path = "http://....."+url_path
if (url_path not in dest_url_over) and (url_path not in dest_url_no):
dest_url_no.append(url_path)
dest_url_handle.write(url_path+"\n")
dest_url_over.append(dest_url)
print("子进程执行完毕--%d"%num)
if __name__=='__main__':
print("主进程开始执行")
manager = Manager()
dest_url_no = manager.list()
dest_url = "http://"
dest_url_no.append(dest_url)
dest_url_over = manager.list()
dest_ftp_path = manager.list()
dest_file = manager.dict()
dest_file = {"dest_url_file":"./movic_url_bak.txt","dest_ftp_file":"./movic_ftp_bak.txt"}
process_list = [];
for i in range(10):
p = Process(target=write_data_proc, args=(dest_url_no,dest_url_over,dest_ftp_path,dest_file))
p.start()
process_list.append(p)
for item in process_list:
item.join()
print('执行完毕')复制代码