nodejs + mongo + redis + oss追加写入CSV文件

朱绍佳

已于 2024-04-12 10:21:30 修改

阅读量658

点赞数 7

文章标签： redis javascript node.js mongodb

于 2024-04-12 10:19:24 首次发布

本文链接：https://blog.csdn.net/qq_32531777/article/details/137671137

版权

文章描述了一个Node.js控制器中的函数，用于处理不同类型的文件导出（如用户列表、交易记录等），并利用阿里云OSS进行文件上传。它检查OSS配置，处理数据导出、总数检查、文件创建和写入，以及OSS上传流程。

摘要由CSDN通过智能技术生成

const fs = require(‘fs’);
const path = require(‘path’);
const ENUM = require(‘…/…/extend/enum’);
const OSS = require(‘ali-oss’);
const page = 1;
const limit = 1000;
let OSS_CONFIG = think.config(‘OSS_CONFIG’)
OSS_CONFIG[‘timeout’] = 86400

module.exports = class extends think.Controller {
async writeSvc(data){
const object_id = data._id
if (think.isEmpty(OSS_CONFIG) || !object_id) {
await this.editDataSvc(object_id, {status: 4, fail_reason: ${object_id}未找到oss配置})
return
}
let params = data.where || {}
params.limit = limit;
params.page = page;
// 构建查询参数
const channel = data.channel;
const menu_button = data.menu_button || []
// 下载类型
let downloadType = data.downloadType;

let result = {};
// 构建开始 ---------------------------------------------------------------------
// 去除参数中为空字符串的条件
params = Object.fromEntries(
  Object.entries(params).filter(([key, value]) => value !== "")
);
// 根据下载类型调用相应的导出方法
switch (downloadType) {
    case "userList":
        result = await think.model("common/derive").userListExport(params, channel, menu_button);
        break;
    case "ExchangeList":
        result = await think.model("common/derive").GetWithdrawListExport(params, channel);
        break;
    case "payRecord":
        result = await think.model("common/derive").GetPayRecordExport(params, channel);
        break;
    case "DrawInfoRecord":
        result = await think.model("common/derive").getDrawPageRecordExport(params, channel);
        break;
    case "gameData":
        result = await think.model("common/derive").getGameStaticsDataExport(params, channel);
        break;
    case "ActiveDetails":
        result = await think.model("common/derive").ActiveDetails(params, channel);
        break;
    default:
        console.log("无效的下载类型");
}
// 构建结束 ---------------------------------------------------------------------
// 此次下载总数量
const total = result.count

console.log(`导出类型：${downloadType},导出数量：${total}`);

// 数量检查
if (total >= 1000000 || think.isEmpty(total)) {
  await this.editDataSvc(object_id, {status: 4, fail_reason: '导出数据超过限制或无数据'})
  console.log('导出数据超过限制');
  return;
}

// 记录总次数
await this.editDataSvc(object_id, {export_quantity: total})

const numOfIterations = Math.ceil(total / limit); // 计算需要循环导出的次数
console.log(`应循环总次数为 ${numOfIterations}`);

// 定义文件夹名称
const folderName = 'downloadCenter';
// 确定文件夹路径
const folderPath = path.join(think.ROOT_PATH, folderName);

// 如果文件夹不存在，则创建文件夹
if (!fs.existsSync(folderPath)) {
  fs.mkdirSync(folderPath);
}

// 定义CSV文件名称
const csvFileName = path.join(folderPath, `${Date.now()}_${think.uuid()}.csv`);

// 如果文件不存在，则写入BOM和表头
if (!fs.existsSync(csvFileName)) {
  const bom = Buffer.from("EFBBBF", "hex").toString("utf-8");
  const headers = Object.values(data.ToCnName); // 获取表头的中文列名
  fs.writeFileSync(csvFileName, bom + headers.join(',') + '\n');
}

// 支付方式枚举
const pay_payment = await think.mongoPay('conf_pay_type').field('_id, remarks').select();
// 跳转位置
const jumpList = await think.mongoConf('conf_jump_enums').field('value, name').select();
//获取支付平台
const paymentPlatform = await think.mongoPay('conf_channel').select();

// 进度
let downloadedBytes = 0;
// 循环导出数据
for (let i = 0; i < numOfIterations; i++) {
  console.log(data.file_name + '循环次数·', i + 1);
  // 下载页码
  params.page = i + 1;
  // 下载数据
  let list = {};
  switch (downloadType) {
    case "userList":
        let userList = await think.model("common/derive").userListExport(params, channel, menu_button);
        list = await ENUM.usersDataReplace(userList.data)
        break;
    case "ExchangeList":
        let withdrawList = await think.model("common/derive").GetWithdrawListExport(params, channel);
        list = await ENUM.getExchangeData(withdrawList.data, params, paymentPlatform)
        break;
    case "payRecord":
        let payRecordList = await think.model("common/derive").GetPayRecordExport(params, channel);
        list = await ENUM.getPayRecordData(payRecordList.data, params, pay_payment, jumpList, paymentPlatform)
        break;
    case "DrawInfoRecord":
        let DrawInfoRecordList = await think.model("common/derive").getDrawPageRecordExport(params, channel);
        list = await ENUM.getDrawInfoRecordData(DrawInfoRecordList.data)
        break;
    case "gameData":
        let gameDataList = await think.model("common/derive").getGameStaticsDataExport(params, channel);
        list = await ENUM.getGameData(gameDataList.data,  params.game_enum)
        break;
    default:
        console.log("无效的下载类型");
  }
  // 追加写入数据
  for (const record of list) {
    const rowValues = Object.keys(data.ToCnName).map(key => {
      let value = record[key];
      if (typeof value !== 'string') {
        value = String(value); // 将非字符串类型的值转换为字符串
      }
      // 如果数据中包含逗号，则用双引号括起来
      return value.includes(',') ? `"${value}"` : value;
    });
    fs.appendFileSync(csvFileName, rowValues.join(',') + '\n');
  }
  downloadedBytes += list.length;
  const progress = Math.floor((downloadedBytes / total) * 100);
  await this.editDataSvc(object_id, {progress: progress})
}
// 上传oss
// 定义CSV文件路径  
const csvFilePath = `${folderName}/${think.formatDate(Date.now(), 'yyyy-MM-dd')}/${data.file_name}.csv`;
let url = await this.upOssloadFile(object_id, csvFilePath, csvFileName)
if (url) {
  // 删除文件
  fs.unlink(csvFileName, (err) => {
    if (err) {
      console.error('Error deleting file:', err);
      return;
    }
    console.log('File deleted successfully');
  });
  await this.editDataSvc(object_id, {status: 2, download_url: url})
} else {
  await this.editDataSvc(object_id, {status: 4, fail_reason: 'oss上传失败'})
}
console.log('CSV 文件写入完成');

}

// 上传文件
async upOssloadFile(object_id, csvFilePath, csvFileName) {
const ossClient = new OSS(OSS_CONFIG);
try {
await this.editDataSvc(object_id, {status: 5})
const result = await ossClient.put(csvFilePath, csvFileName);
let url = await this.extractPathAfterDomain(result[‘url’])
return url
} catch (err) {
console.error(‘Upload error:’, err);
}
}

async extractPathAfterDomain(url) {
// 匹配域名后的路径
var pathAfterDomain = url.match(/(?:https?😕/)?(?:[^{@\n]+@)?(?:www.)?[}😕\n?]+(/[^?#]+)/img);
if (pathAfterDomain && pathAfterDomain.length > 0) {
// 返回匹配的路径
return pathAfterDomain[0].replace(/^.*//[/]+/, ‘’);
} else {
return null;
}
}

// 更新下载进度及其他信息
async editDataSvc(id, data) {
await think.mongoRecord(‘record_download_csv_list’).where({_id: id}).update({$set: data})
}
}