【Chat Is Cheap Show Me Your Code——Python Crawler for B***】

Chat Is Cheap Show Me Your Code

import os
import re
import json
import requests
import time
import random
import subprocess
import argparse
from tqdm import tqdm
from urllib.parse import quote
import pickle
import hashlib
import threading
from fake_useragent import UserAgent
import platform

class BilibiliDownloader:
    def __init__(self, save_path="downloads", proxy=None, max_retries=5, request_delay=2.5):
        self.session = requests.Session()
        
        # 动态生成更真实的请求头
        self.session.headers.update({
            "User-Agent": UserAgent().random,
            "Referer": "https://www.bilibili.com/",
            "Origin": "https://www.bilibili.com",
            "Accept": "application/json, text/plain, */*",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Accept-Encoding": "gzip, deflate, br",
            "Connection": "keep-alive",
            "DNT": "1"  # Do Not Track
        })
        
        # 动态生成Cookie
        self.session.cookies.update({
            "buvid3": self.generate_random_buvid(),
            "buvid4": self.generate_random_buvid(36),
            "CURRENT_FNVAL": "4048",
            "b_lsid": self.generate_random_lsid(),
            "_uuid": self.generate_random_uuid()
        })
        
        # 代理设置
        if proxy:
            self.session.proxies = {
                "http": proxy,
                "https": proxy
            }
        
        self.save_path = save_path
        os.makedirs(save_path, exist_ok=True)
        
        # 反爬虫参数
        self.max_retries = max_retries
        self.request_delay = request_delay
        self.download_state = {}
        self.state_file = os.path.join(save_path, "download_state.pkl")
        
        # 加载之前的下载状态
        self.load_download_state()
    
    def generate_random_buvid(self, length=32):
        """生成随机的B站buvid值"""
        characters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
        return ''.join(random.choices(characters, k=length))
    
    def generate_random_lsid(self):
        """生成随机的LSID值"""
        timestamp = int(time.time() * 1000)
        random_str = ''.join(random.choices("0123456789ABCDEF", k=16))
        return f"{timestamp}_{random_str}"
    
    def generate_random_uuid(self):
        """生成随机的UUID格式字符串"""
        return f"{''.join(random.choices('0123456789abcdef', k=8))}-" \
               f"{''.join(random.choices('0123456789abcdef', k=4))}-" \
               f"{''.join(random.choices('0123456789abcdef', k=4))}-" \
               f"{''.join(random.choices('0123456789abcdef', k=4))}-" \
               f"{''.join(random.choices('0123456789abcdef', k=12))}"
    
    def load_download_state(self):
        """加载下载状态"""
        try:
            if os.path.exists(self.state_file):
                with open(self.state_file, 'rb') as f:
                    self.download_state = pickle.load(f)
        except:
            self.download_state = {}
    
    def save_download_state(self):
        """保存下载状态"""
        try:
            with open(self.state_file, 'wb') as f:
                pickle.dump(self.download_state, f)
        except:
            pass
    
    def safe_request(self, url, method='get', params=None, data=None, headers=None, retry_count=0):
        """安全的请求函数,带重试和延迟"""
        # 随机延迟避免请求过快
        delay = random.uniform(self.request_delay, self.request_delay * 2)
        time.sleep(delay)
        
        # 更新User-Agent
        self.session.headers["User-Agent"] = UserAgent().random
        
        try:
            if method.lower() == 'get':
                response = self.session.get(url, params=params, headers=headers, timeout=20)
            else:
                response = self.session.post(url, data=data, headers=headers, timeout=20)
            
            # 修正状态码检查:允许200和206状态码
            if response.status_code in (403, 418):
                raise PermissionError("访问被拒绝,可能触发了反爬虫机制")
            if response.status_code == 429:
                raise ConnectionError("请求过于频繁,请稍后再试")
            if response.status_code not in (200, 206):
                raise ConnectionError(f"请求失败,状态码: {response.status_code}")
            
            return response
        
        except Exception as e:
            if retry_count < self.max_retries:
                print(f"请求失败: {str(e)},第 {retry_count+1}/{self.max_retries} 次重试...")
                # 生成新的Cookie
                self.session.cookies.update({
                    "buvid3": self.generate_random_buvid(),
                    "b_lsid": self.generate_random_lsid()
                })
                return self.safe_request(url, method, params, data, headers, retry_count+1)
            else:
                raise
    
    def search_bilibili(self, keyword, search_type="video", page=1, page_size=20):
        """搜索B站内容"""
        # 构建搜索URL
        if search_type == "video":
            search_url = f"https://api.bilibili.com/x/web-interface/search/type?search_type=video&keyword={quote(keyword)}&page={page}&page_size={page_size}"
        elif search_type == "audio":
            search_url = f"https://api.bilibili.com/audio/music-service-c/web/song/search?keyword={quote(keyword)}&page={page}&page_size={page_size}"
        else:
            raise ValueError("不支持的搜索类型")
        
        try:
            # 添加签名参数避免简单重复请求
            params = {
                "t": int(time.time()),
                "r": random.randint(1000, 9999)
            }
            response = self.safe_request(search_url, params=params)
            data = response.json()
            
            if data.get("code", -1) != 0:
                error_msg = data.get("message", "未知错误")
                print(f"搜索失败: {error_msg}")
                return []
            
            results = []
            if search_type == "video":
                for item in data["data"]["result"]:
                    # 清理标题中的HTML标签
                    title = re.sub(r'<[^>]+>', '', item["title"])
                    results.append({
                        "type": "video",
                        "title": title,
                        "bvid": item["bvid"],
                        "author": item["author"],
                        "duration": item["duration"],
                        "url": f"https://www.bilibili.com/video/{item['bvid']}"
                    })
            elif search_type == "audio":
                for item in data["data"]["data"]:
                    results.append({
                        "type": "audio",
                        "title": item["title"],
                        "auid": item["id"],
                        "author": item["author"],
                        "duration": item["duration"],
                        "url": f"https://www.bilibili.com/audio/au{item['id']}"
                    })
            
            return results
        
        except Exception as e:
            print(f"搜索过程中出错: {str(e)}")
            return []
    
    def interactive_search(self, keyword, search_type="video"):
        """交互式搜索和选择"""
        print(f"\n正在搜索'{keyword}'...")
        results = self.search_bilibili(keyword, search_type)
        
        if not results:
            print("未找到相关内容")
            return []
        
        print("\n搜索结果:")
        for i, item in enumerate(results):
            print(f"{i+1}. [{item['type']}] {item['title']} - {item['author']} ({item['duration']}秒)")
        
        print("\n选择下载选项:")
        print("a. 下载全部")
        print("s. 选择部分下载")
        print("c. 取消")
        
        choice = input("请输入选择: ").strip().lower()
        
        selected = []
        if choice == 'a':
            selected = results
        elif choice == 's':
            selections = input("请输入要下载的项目编号(多个用逗号分隔): ").strip()
            try:
                indices = [int(idx.strip()) - 1 for idx in selections.split(',')]
                for idx in indices:
                    if 0 <= idx < len(results):
                        selected.append(results[idx])
            except:
                print("无效的选择")
        elif choice == 'c':
            print("操作已取消")
        else:
            print("无效的选择")
        
        return selected
    
    def get_video_info(self, bvid):
        """获取视频信息"""
        api_url = f"https://api.bilibili.com/x/web-interface/view?bvid={bvid}"
        
        # 添加签名参数
        params = {
            "t": int(time.time()),
            "r": random.randint(1000, 9999)
        }
        response = self.safe_request(api_url, params=params)
        data = response.json()
        
        if data["code"] != 0:
            error_msg = data.get("message", "未知错误")
            if data["code"] == -404:
                raise ValueError("视频不存在或已被删除")
            raise ValueError(f"获取视频信息失败: {error_msg}")
        
        return data["data"]
    
    def get_audio_info(self, auid):
        """获取音频信息"""
        api_url = f"https://www.bilibili.com/audio/music-service-c/web/song/info?sid={auid}"
        
        # 添加签名参数
        params = {
            "t": int(time.time()),
            "r": random.randint(1000, 9999)
        }
        response = self.safe_request(api_url, params=params)
        data = response.json()
        
        if data["code"] != 0:
            error_msg = data.get("msg", "未知错误")
            if data["code"] == -404:
                raise ValueError("音频不存在或已被删除")
            raise ValueError(f"获取音频信息失败: {error_msg}")
        
        return data["data"]
    
    def get_video_download_url(self, bvid, cid, quality=80):
        """获取视频下载URL"""
        api_url = "https://api.bilibili.com/x/player/playurl"
        
        # 添加签名参数
        params = {
            "bvid": bvid,
            "cid": cid,
            "qn": quality,
            "fnval": "4048",
            "t": int(time.time()),
            "r": random.randint(1000, 9999)
        }
        
        response = self.safe_request(api_url, params=params)
        data = response.json()
        
        if data["code"] != 0:
            error_msg = data.get("message", "未知错误")
            if data["code"] == -404:
                raise PermissionError("此视频可能需要登录或大会员权限")
            raise ValueError(f"获取下载URL失败: {error_msg}")
        
        # 提取最佳质量的视频和音频URL
        video_url = audio_url = None
        
        # 检查是否有dash格式
        if "dash" in data["data"]:
            # 提取视频流
            video_streams = data["data"]["dash"]["video"]
            # 按质量排序
            video_streams.sort(key=lambda x: x["id"], reverse=True)
            # 选择第一个(最高质量)
            video_url = video_streams[0]["baseUrl"]
            
            # 提取音频流
            audio_streams = data["data"]["dash"]["audio"]
            if audio_streams:
                audio_url = audio_streams[0]["baseUrl"]
        # 检查是否有durl格式
        elif "durl" in data["data"]:
            # 使用FLV格式
            durls = data["data"]["durl"]
            if durls:
                video_url = durls[0]["url"]
        
        if not video_url:
            # 尝试备用方法
            accept_quality = data["data"].get("accept_quality", [])
            if accept_quality:
                # 尝试使用最高可用质量
                backup_quality = max(accept_quality)
                return self.get_video_download_url(bvid, cid, backup_quality)
            else:
                raise ValueError("无法解析下载URL")
        
        return video_url, audio_url
    
    def download_file(self, url, filename, desc="下载中", content_type="video"):
        """下载文件并显示进度条,支持断点续传"""
        # 创建文件下载状态key
        state_key = hashlib.md5(f"{url}_{filename}".encode()).hexdigest()
        
        # 检查是否已有部分下载
        downloaded_size = 0
        mode = 'wb'
        
        if state_key in self.download_state and os.path.exists(filename):
            downloaded_size = self.download_state[state_key]
            mode = 'ab'
            print(f"检测到未完成的下载,继续下载 ({downloaded_size} 字节已下载)")
        
        try:
            # 添加range请求头支持断点续传
            headers = {}
            if downloaded_size > 0:
                headers["Range"] = f"bytes={downloaded_size}-"
            
            response = self.safe_request(url, headers=headers)
            
            # 处理206响应(断点续传)
            if response.status_code == 206:
                # 解析Content-Range获取总大小
                content_range = response.headers.get('Content-Range', '')
                if '/' in content_range:
                    total_size = int(content_range.split('/')[1])
                else:
                    total_size = downloaded_size + int(response.headers.get('content-length', 0))
            elif response.status_code == 200:
                total_size = int(response.headers.get('content-length', 0))
            else:
                raise ValueError(f"意外的状态码: {response.status_code}")
            
            with open(filename, mode) as f:
                with tqdm(total=total_size, unit='B', unit_scale=True, 
                         desc=desc, initial=downloaded_size) as pbar:
                    for chunk in response.iter_content(chunk_size=1024*1024):  # 1MB chunks
                        if chunk:
                            f.write(chunk)
                            pbar.update(len(chunk))
                            
                            # 更新下载状态
                            downloaded_size += len(chunk)
                            self.download_state[state_key] = downloaded_size
                            # 每5MB保存一次状态
                            if downloaded_size % (5 * 1024 * 1024) == 0:
                                self.save_download_state()
            
            # 下载完成后移除状态
            if state_key in self.download_state:
                del self.download_state[state_key]
                self.save_download_state()
            
            return True
        except Exception as e:
            print(f"下载过程中出错: {str(e)}")
            # 保存当前下载状态
            self.save_download_state()
            return False
    
    def merge_video_audio(self, video_path, audio_path, output_path):
        """使用FFmpeg合并视频和音频"""
        try:
            # 检查文件是否存在
            if not os.path.exists(video_path) or not os.path.exists(audio_path):
                raise FileNotFoundError("临时文件缺失")
            
            # 尝试自动查找ffmpeg
            ffmpeg_path = self.find_ffmpeg()
            if not ffmpeg_path:
                raise FileNotFoundError("未找到FFmpeg,请安装并添加到系统PATH")
            
            # 根据系统设置不同的命令
            if platform.system() == "Windows":
                # Windows下隐藏FFmpeg窗口
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                startupinfo.wShowWindow = 0
            else:
                startupinfo = None
            
            # 使用FFmpeg合并
            subprocess.run(
                [
                    ffmpeg_path,
                    "-i", video_path,
                    "-i", audio_path,
                    "-c", "copy",
                    "-map", "0:v:0",
                    "-map", "1:a:0",
                    "-loglevel", "error",
                    "-y",
                    output_path
                ],
                check=True,
                startupinfo=startupinfo,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL
            )
            
            # 删除临时文件
            os.remove(video_path)
            if os.path.exists(audio_path):
                os.remove(audio_path)
            return True
        except (subprocess.CalledProcessError, FileNotFoundError) as e:
            print(f"合并失败: {str(e)}")
            return False
        except Exception as e:
            print(f"合并过程中出错: {str(e)}")
            return False
    
    def find_ffmpeg(self):
        """尝试在系统路径中查找 FFmpeg"""
        try:
            # 检查是否在PATH中
            if platform.system() == "Windows":
                subprocess.run(["ffmpeg", "-version"], 
                              stdout=subprocess.DEVNULL, 
                              stderr=subprocess.DEVNULL,
                              creationflags=subprocess.CREATE_NO_WINDOW,
                              check=True)
            else:
                subprocess.run(["ffmpeg", "-version"], 
                              stdout=subprocess.DEVNULL, 
                              stderr=subprocess.DEVNULL,
                              check=True)
            return "ffmpeg"
        except:
            # 尝试常见安装路径
            paths = [
                r"C:\ffmpeg\bin\ffmpeg.exe",
                r"D:\ffmpeg\bin\ffmpeg.exe",
                r"~\ffmpeg\bin\ffmpeg.exe",
                "/usr/bin/ffmpeg",
                "/usr/local/bin/ffmpeg",
                r"C:\Program Files\ffmpeg\bin\ffmpeg.exe"
            ]
            for path in paths:
                expanded_path = os.path.expanduser(path)
                if os.path.exists(expanded_path):
                    return expanded_path
            return None
    
    def download_video(self, url, quality=80):
        """下载单个视频"""
        print(f"\n{'='*50}")
        print(f"处理视频: {url}")
        
        # 从URL提取BV号
        match = re.search(r"BV[0-9A-Za-z]{10}", url)
        if not match:
            print("无效的B站视频URL")
            return False
        bvid = match.group(0)
        
        try:
            # 获取视频信息
            video_info = self.get_video_info(bvid)
            title = re.sub(r'[\\/*?:"<>|]', "", video_info["title"])  # 移除非法文件名字符
            cid = video_info["cid"]
            
            print(f"开始下载视频: {title} (CID: {cid})")
            
            # 获取下载URL
            video_url, audio_url = self.get_video_download_url(bvid, cid, quality)
            
            if not video_url:
                print("无法获取视频下载链接")
                return False
            
            print(f"视频流URL: {video_url[:60]}...")
            if audio_url:
                print(f"音频流URL: {audio_url[:60]}...")
            
            # 创建保存目录
            safe_title = title[:100]  # 防止文件名过长
            safe_title = re.sub(r'<[^>]+>', '', safe_title)  # 移除HTML标签
            video_dir = os.path.join(self.save_path, "videos", safe_title)
            os.makedirs(video_dir, exist_ok=True)
            
            # 下载视频
            video_ext = '.mp4' if audio_url else '.flv'
            video_temp = os.path.join(video_dir, "video_temp" + video_ext)
            
            if not self.download_file(video_url, video_temp, "下载视频流", "video"):
                print("视频下载失败")
                return False
                
            # 下载音频(如果存在)
            audio_temp = None
            if audio_url:
                audio_temp = os.path.join(video_dir, "audio_temp.m4s")
                if not self.download_file(audio_url, audio_temp, "下载音频流", "audio"):
                    print("音频下载失败,将尝试使用单文件模式")
                    audio_temp = None
            
            # 确定最终输出路径
            final_ext = '.mp4' if audio_temp else video_ext
            final_path = os.path.join(video_dir, f"{safe_title}{final_ext}")
            
            # 合并或重命名
            if audio_temp:
                print("合并视频和音频...")
                if self.merge_video_audio(video_temp, audio_temp, final_path):
                    print(f"视频下载完成: {final_path}")
                    return True
                else:
                    # 合并失败,保留视频文件
                    os.rename(video_temp, final_path)
                    print(f"合并失败,已保存视频文件: {final_path}")
                    return True
            else:
                # 没有音频,直接重命名视频文件
                os.rename(video_temp, final_path)
                print(f"视频下载完成: {final_path}")
                return True
                
        except Exception as e:
            print(f"下载失败: {str(e)}")
            return False
    
    def download_audio(self, url):
        """下载单个音频"""
        print(f"\n{'='*50}")
        print(f"处理音频: {url}")
        
        # 从URL提取AU号
        match = re.search(r"au(\d+)", url)
        if not match:
            print("无效的B站音频URL")
            return False
        auid = match.group(1)
        
        try:
            # 获取音频信息
            audio_info = self.get_audio_info(auid)
            title = re.sub(r'[\\/*?:"<>|]', "", audio_info["title"])
            
            print(f"开始下载音频: {title}")
            
            # 获取下载URL
            audio_url = audio_info.get("play_url") or audio_info.get("cdns")[0]
            if not audio_url:
                # 尝试备用方法
                if "cdns" in audio_info and audio_info["cdns"]:
                    audio_url = audio_info["cdns"][0]
                else:
                    raise ValueError("无法获取音频下载链接")
            
            print(f"音频URL: {audio_url[:60]}...")
            
            # 创建保存目录
            audio_dir = os.path.join(self.save_path, "audios")
            os.makedirs(audio_dir, exist_ok=True)
            
            # 确定文件扩展名
            file_ext = '.mp3'
            if '.flac' in audio_url:
                file_ext = '.flac'
            elif '.m4a' in audio_url:
                file_ext = '.m4a'
            
            # 下载音频
            safe_title = title[:100]  # 防止文件名过长
            audio_path = os.path.join(audio_dir, f"{safe_title}{file_ext}")
            
            if self.download_file(audio_url, audio_path, "下载音频", "audio"):
                print(f"音频下载完成: {audio_path}")
                return True
            else:
                print("音频下载失败")
                return False
            
        except Exception as e:
            print(f"下载失败: {str(e)}")
            return False
    
    def download_content(self, item, quality=80):
        """下载单个内容项"""
        if item["type"] == "video":
            return self.download_video(item["url"], quality)
        elif item["type"] == "audio":
            return self.download_audio(item["url"])
        return False
    
    def batch_download(self, items, quality=80):
        """批量下载内容"""
        success_count = 0
        total = len(items)
        
        for i, item in enumerate(items):
            print(f"\n{'='*50}")
            print(f"处理项目 {i+1}/{total}: [{item['type']}] {item['title']}")
            
            try:
                if self.download_content(item, quality):
                    success_count += 1
            except Exception as e:
                print(f"处理过程中发生错误: {str(e)}")
            
            # 随机延迟避免请求过快
            delay = random.uniform(4, 10)
            print(f"等待 {delay:.1f} 秒继续...")
            time.sleep(delay)
        
        print(f"\n{'='*50}")
        print(f"批量下载完成: 成功 {success_count}/{total} 个项目")
        return success_count

def main():
    parser = argparse.ArgumentParser(description="B站视频/音乐搜索与下载工具", 
                                    formatter_class=argparse.RawTextHelpFormatter)
    subparsers = parser.add_subparsers(dest='command', help='命令')
    
    # 搜索命令
    search_parser = subparsers.add_parser('search', help='搜索内容')
    search_parser.add_argument("keyword", help="搜索关键词")
    search_parser.add_argument("--type", choices=["video", "audio"], default="video", 
                        help="搜索内容类型: video(视频) 或 audio(音频)")
    search_parser.add_argument("--output", default="downloads", help="保存路径")
    search_parser.add_argument("--proxy", help="代理服务器地址 (例如: http://127.0.0.1:8080)")
    
    # 下载命令
    download_parser = subparsers.add_parser('download', help='下载内容')
    download_parser.add_argument("urls", nargs="+", help="B站视频或音频URL列表")
    download_parser.add_argument("--type", choices=["video", "audio"], default="video", 
                        help="下载内容类型: video(视频) 或 audio(音频)")
    download_parser.add_argument("--quality", type=int, default=80, 
                        help="视频质量 (80: 高清, 64: 流畅, 32: 标清, 16: 低清)")
    download_parser.add_argument("--output", default="downloads", help="保存路径")
    download_parser.add_argument("--proxy", help="代理服务器地址 (例如: http://127.0.0.1:8080)")
    
    # 如果没有参数,显示帮助
    if len(sys.argv) == 1:
        parser.print_help(sys.stderr)
        sys.exit(1)
    
    args = parser.parse_args()
    
    downloader = BilibiliDownloader(
        save_path=args.output, 
        proxy=args.proxy,
        request_delay=2.5,
        max_retries=5
    )
    
    if args.command == "search":
        selected_items = downloader.interactive_search(args.keyword, args.type)
        if selected_items:
            downloader.batch_download(selected_items)
    elif args.command == "download":
        items = []
        for url in args.urls:
            if "bilibili.com/video/" in url:
                items.append({
                    "type": "video",
                    "title": "直接下载",
                    "url": url
                })
            elif "bilibili.com/audio/" in url:
                items.append({
                    "type": "audio",
                    "title": "直接下载",
                    "url": url
                })
        
        if items:
            downloader.batch_download(items, args.quality)
        else:
            print("未提供有效的下载URL")
    
    # 程序退出前保存下载状态
    downloader.save_download_state()

if __name__ == "__main__":
    import sys
    main()



### 使用说明

1. **安装依赖**:
```bash
pip install requests tqdm fake-useragent
  1. 安装FFmpeg(必需):
  • Windows:下载并添加到PATH FFmpeg官网
  • macOS:brew install ffmpeg
  • Linux:sudo apt install ffmpeg
  1. 使用示例
# 搜索并下载视频
python bilibili_downloader.py search "Rick Astley" --type video

# 下载特定视频
python bilibili_downloader.py download "https://www.bilibili.com/video/BV1GJ411x7h7"

# 使用代理下载
python bilibili_downloader.py download "https://www.bilibili.com/video/BV1GJ411x7h7" --proxy "http://127.0.0.1:8080"
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值