场景:有一批数据没有推到中间件,需要我这里搞一个脚本,大约有600万数据。我开始比较蠢的方法是将文件拆分,然后一条一条读。一条一条请求,速度很慢,如果因为某些因素中断程序,虽然有日志,但是还是要将已经跑过的数据去除,很麻烦
后来领导说放redis,然后用hyperf协程搞。
为啥这样搞,redis保证数据不丢。开协程增加并发
废话不多说,上代码
/**
* composer test -- --filter=testCmshzSignHandle
*/
public function testCmshzSignHandle()
{
try {
$logFiles = '/opt/www/test/Cases/text.txt';
if (! file_exists($logFiles)) {
throw new \Exception('日志文件不存在');
}
$file = fopen($logFiles, 'r');
$redisKey = 'handle-redis';
$i = 0;
while (! feof($file)) {
$i++;
$lineData = fgets($file);
if (intval($lineData)==0) {
continue;
}
$requestData[] = trim($lineData, '"');
if (count($requestData) > 500) {
Redis::push($redisKey, ...$requestData);
$requestData = [];
}
}
$result = Redis::llen($redisKey);
$this->assertTrue($result);
} catch (\Throwable $e) {
var_dump($e->getMessage());
$this->assertTrue(false, $e->getMessage());
}
}
/**
* composer test -- --filter=testCsGo
*/
public function testCsGo()
{
$key = 'handle-redis';
$request = new RequestServices();
$parallel = new Coroutine\Concurrent(10);
while (true) {
$result = Redis::pop($key, 10);
if (empty($result)) {
break;
}
$parallel->create(function () use($result,$request, $key) {
$params = [
'suborderid' => $result,
];
//请求数据添加数据签名
$params = CmsEncryptServices::getInstance()->dataSign($params);
$res = $request->post($url, $params);
Log::info($key, [$res]);
usleep(100);
if ($res['code'] != 0) {
Redis::push($key, $result);
}
});
}
}