php多进程采集百度,php实现多进程下载百度网盘文件

大家知道百度网盘下载对于非会员是有下载限速的, 最大速度基本上维持在 100kB/s以内,要下个电影啥的,那就有得等了. 之前还可以把网盘里的文件链接解析出来放到Uget或迅雷之类的下载工具里去多线程下载, 但是现在百度的文件服务器对ua做了校验, 没找到好用的可编辑http header的下载工具, 于是自己动手写一个了.基本上可以充分利用已有的带宽

download_demo.gif

PS: 请自行安装swoole拓展和Guzzle http包.

直接上代码吧:

include 'vendor/autoload.php';

// $service = new Service('http://peterq.cn/movie/api/video_redirect?fid=543468589252145', __DIR__);

$service = new Service('http://peterq.cn/movie/api/video_redirect?fid=364402848596280', __DIR__);

$service->start();

use GuzzleHttp\Client;

class Service

{

/**

* @var Client;

*/

protected $client;

protected $worker_pool; // 下载进程池

protected $available_worker_queue; // 可用的进程队列

protected $worker_number = 16; // 定义需要开多少个进程, 文件较小时, 并不一定全部用得上, 取决于你的分片大小

protected $started = false; // 是否已经开始下载

protected $url; // 下载链接

protected $length; // 文件大小

protected $dir; // 保存目录

protected $filename; // 文件绝对路径

protected $downloaded = 0; // 已下载字节数

protected $speedArr = []; // 用来计算下载速度的数组

protected $distributed = 0; // 对于要下载的文件, 已经分配到哪个位置了

public function __construct($url, $dir)

{

$this->url = $url;

$this->dir = realpath($dir);

}

public function start()

{

if ($this->started) return;

$this->available_worker_queue = new SplQueue();

$this->started = true;

// 创建客户端

$this->client = new Client([

'headers' => [

"Accept" => "application/json, text/javascript, text/html, */*; q=0.01",

"Accept-Encoding" => "gzip, deflate, sdch",

"Accept-Language" => "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2",

"Referer" => "http://pan.baidu.com/disk/home",

"X-Requested-With" => "XMLHttpRequest",

"User-Agent" => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36",

"Connection" => "keep-alive",

],

]);

// 设置进程名称

swoole_set_process_name('download-master');

echo 'master pid:' . posix_getpid() . PHP_EOL;

// 创建多个下载进程

for ($i = 0; $i < $this->worker_number; $i++) {

$process = $this->createProcess($i);

$this->worker_pool[$i] = $process;

// 通信通道加入事件轮训, 进行异步通信

swoole_event_add($process->pipe, function ($pipe) use ($process) {

$data = $process->read();

$data = unserialize($data);

$this->handleChildMessage($process, $data['type'], $data['data']);

});

$process->start();

$this->available_worker_queue->enqueue($process);

}

// 子进程退出回收

swoole_process::signal(SIGCHLD, function ($sig) {

static $exited = 0;

// 必须为false，非阻塞模式

while ($ret = swoole_process::wait(false)) {

echo "child process exited, PID={$ret['pid']}\n";

$exited++;

if ($exited == count($this->worker_pool)) exit();

}

});

$this->initDownload();

}

// 初始化下载

protected function initDownload()

{

$resp = $this->client->request('GET', $this->url, [

'stream' => true,

'read_timeout' => 10,

]);

// 处理重定向

while (in_array($resp->getStatusCode(), [301, 302])) {

$this->url = $resp->getBody()->read(1024);

dump('redirect: ' . $this->url);

$resp = $this->client->request('GET', $this->url, [

'stream' => true,

'read_timeout' => 10,

]);

}

if (!$resp->getHeader('Content-Disposition')) {

dump('not a file download url');

}

$this->length = intval($resp->getHeader('Content-Length')[0]);

$fname = $resp->getHeader('Content-Disposition')[0];

$fname = substr($fname, strpos($fname, 'filename=') + strlen('filename='));

$fname = urldecode($fname);

$this->filename = $this->dir . '/' . $fname;

dump([

'文件' => $this->filename,

'大小' => round($this->length / 1024 / 1024, 2) . 'MB'

]);

file_put_contents($this->filename, '');

$this->download();

}

// 启动下载

protected function download()

{

while (

$this->distributed < $this->length

&& $this->available_worker_queue->count()

&& $process = $this->available_worker_queue->dequeue()) {

$this->distributeSegment($process);

}

}

// 分配下一块区间给一个进程

protected function distributeSegment($process)

{

// 分成 1 MB 一个段去下载

$size = 1 * 1024 * 1024;

$process->write(serialize([

'type' => 'new-segment',

'data' => [

'url' => $this->url,

'file' => $this->filename,

'start' => $this->distributed,

'length' => min($size, $this->length - $this->distributed),

]

]));

$this->distributed += $size;

}

// 进程间通信处理

protected function handleChildMessage($process, $type, $data)

{

method_exists($this, 'on' . ucfirst($type)) and $this->{'on' . ucfirst($type)}($process, $data);

}

// 当下载进程下载一小块时, 通过此回调通知master进程

protected function onRange(swoole_process $process, $data)

{

$this->downloaded += $data;

static $lastClearTime = 0;

$time = time();

$this->speedArr[$time] = $this->speedArr[$time] ?? 0;

$this->speedArr[$time] += $data;

// 取过去 5 秒作为平均速度 作为速度显示, 粗略计算, 并不准确

if ($time > $lastClearTime) {

$lastClearTime = $time;

foreach ($this->speedArr as $t => $size) {

if ($t < $time - 5) unset($this->speedArr[$t]);

}

}

$speed = array_sum($this->speedArr) / count($this->speedArr);

$percent = $this->downloaded / $this->length * 100;

$percent = round($percent, 2);

$size = humanSize($this->downloaded);

$speed = humanSize($speed);

echo "\r\033[2K" . "已下载: $size, $percent%; 当前速度：　$speed/s";

}

// 当分配给下载进程的下载任务完成时执行的回调

protected function onTaskFinished($process, $data)

{

if ($this->distributed < $this->length)

$this->distributeSegment($process);

else {

$this->available_worker_queue->enqueue($process);

if ($this->available_worker_queue->count() == count($this->worker_pool)) {

dump('文件下载完成');

foreach ($this->worker_pool as $worker) {

$worker->write(serialize([

'type' => 'exit', 'data' => ''

]));

}

}

}

}

// 创建下载进程

protected function createProcess($index = null)

{

$process = new swoole_process(function (swoole_process $process) use ($index) {

swoole_set_process_name('download worker' . $index);

echo sprintf('worker:%s, pid:%s', $index, posix_getpid()) . PHP_EOL;

$downloader = null;

// 通信通道加入事件轮训, 进行异步通信

swoole_event_add($process->pipe, function ($pipe) use ($process, &$downloader) {

$data = $process->read();

$data = unserialize($data);

$type = $data['type'];

$data = $data['data'];

// 这里会阻塞掉, 后续改进

if ($type == 'new-segment') {

$downloader = new Downloader($process, $this->client, $data['url'], $data['file'], $data['start'], $data['length']);

$downloader->download();

$process->write(serialize([

'type' => 'taskFinished',

'data' => ''

]));

$downloader = null;

return;

}

if ($type == 'exit') exit(0);

});

}, false, 2);

return $process;

}

}

// 下载器类

class Downloader

{

protected $client; // guzzle实例

protected $process; // 当前进程实例

protected $file; // 文件名

protected $url;

protected $start; // 开始位置

protected $length; // 下载长度

protected $offset; // 已经下到哪一个位置了

public function __construct(swoole_process $process, Client $client, $url, $file, $start, $length)

{

$this->process = $process;

$this->client = $client;

$this->url = $url;

$this->file = $file;

$this->start = $start;

$this->length = $length;

}

public function download()

{

$this->offset = $this->start;

$res = fopen($this->file, 'rb+');

fseek($res, $this->start, SEEK_SET);

$resp = $this->client->request('GET', $this->url, [

'stream' => true,

'headers' => [

'Range' => 'bytes=' . $this->start . '-' . ($this->start + $this->length)

]

]);

$loaded = 0;

while (!$resp->getBody()->eof()) {

// 5 kb 的下载

$size = 1024 * 5;

$data = $resp->getBody()->read($size);

$loaded += strlen($data);

fwrite($res, $data);

$this->process->write(serialize([

'type' => 'range',

'data' => strlen($data)

]));

if ($loaded >= $this->length) break; // eof 貌似不起作用，　手动退出

}

fclose($res);

dump($this->length / 1024 / 1024 . 'MB下载完成');

}

}

// 把文件大小从字节转换为合适的单位

function humanSize($size) {

$units = ['B', 'KB', 'MB', 'GB'];

foreach ($units as $unit) {

if ($size > 1024)

$size /= 1024;

else break;

}

return round($size, 2) . $unit;

}