[pig框架实战] 手撕视频管理发布平台[06] - 数据批量导入功能

准备工作

手机同屏（方便直播）

同屏软件千万家，适合程序猿的，必然是开源产品，我们选择weditor

• 创建python3.86的虚拟环境（非必须）
  • 我这里使用pycharm生成的
  • 目录为J:\_ALL\python_env\Py386_scrapy\
• 安装weditor
  • cd J:\_ALL\python_env\Py386_scrapy\Scripts
  • 升级pip python.exe -m pip install --upgrade pip
  • pip install weditor
• 连接手机，打开调试模式等
• 双击运行 weditor.exe

springboot配置单元测试

参考：springboot(16)Spring Boot使用单元测试

数据批量导入功能

单元测试

    @Test
    public void importData() {
        // 读取本地文件（json）
        String fileName = "J:\\_ALL\\CODE\\gitee\\constellations\\Capricorn\\WeChat\\爬虫\\tiktok\\data\\2021-12-10--6307840-gmail.json";
        String s = readJsonFile(fileName);

        // 文件解析入库
        JSONObject jobj = JSON.parseObject(s);
        Integer statusCode = jobj.getInteger("statusCode");
        System.out.println("statusCode = " + statusCode);

        List<Myvideos> myvideosList = new ArrayList<Myvideos>();
        JSONArray itemList = jobj.getJSONArray("itemList");//构建JSONArray数组
        for (int i = 0; i < itemList.size(); i++){
            JSONObject item = (JSONObject)itemList.get(i);
            String _id = (String)item.get("id");
            System.out.println("_id = " + _id);
//            String desc = (String)item.get("desc");
//            System.out.println("desc = " + desc);

            QueryWrapper<Myvideos> queryWrapper = new QueryWrapper<>();
            Map<String,Object> map = new HashMap<>();
            map.put("j_id",_id);
            queryWrapper.allEq(map, false);
            List<Myvideos> video = myvideosService.list(queryWrapper);
            // 判断数据库中是否包含_id对应的数据
            if (video.size() >= 1) {
                System.out.println("【已经存在该视频】 ： " + _id);
                continue;
            }
            
            // 将json对象中的数据，保存在Myvideos对象中
            Myvideos myvideos_new = new Myvideos();
            myvideos_new.setJId(_id);
            myvideos_new.setJDesc((String)item.get("desc"));
            myvideos_new.setJCreatetime((Integer) item.get("createTime"));

            // 解析video元素
            // ……
            // ……
            // ……
            
            myvideosList.add(myvideos_new);
            System.out.println("【视频入库】 ： " + _id);
        }
        
        myvideosService.saveBatch(myvideosList);
    }

集成到web端

1. 服务器代码编写

MyvideosController

	@ApiOperation(value = "新增视频管理", notes = "新增视频管理")
	@SysLog("新增视频管理" )
	@PostMapping("/saveAll")
	public R saveAll(@RequestParam("file") MultipartFile files,
					 String myVideoClass, String myTagsCn) {
		boolean ret = myvideosService.saveAll(files, myVideoClass, myTagsCn);
		return R.ok(ret);
	}

MyvideosServiceImpl

@Service
public class MyvideosServiceImpl extends ServiceImpl<MyvideosMapper, Myvideos> implements MyvideosService {

	//读取json文件
	public static String readJsonFile(String fileName) {
		String jsonStr = "";
		try {
			File jsonFile = new File(fileName);
			FileReader fileReader = new FileReader(jsonFile);
			Reader reader = new InputStreamReader(new FileInputStream(jsonFile),"utf-8");
			int ch = 0;
			StringBuffer sb = new StringBuffer();
			while ((ch = reader.read()) != -1) {
				sb.append((char) ch);
			}
			fileReader.close();
			reader.close();
			jsonStr = sb.toString();
			return jsonStr;
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
	}

	public static String readJsonFile(MultipartFile files) {
		String jsonStr = "";
		try {
			//起手转成字符流
			InputStream is = files.getInputStream();
			InputStreamReader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
			int ch = 0;
			StringBuffer sb = new StringBuffer();
			while ((ch = reader.read()) != -1) {
				sb.append((char) ch);
			}
			reader.close();
			jsonStr = sb.toString();
			return jsonStr;
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
	}

	public static Integer convertBoolean2Interger(Object obj) {
		return (Boolean)obj?1:0;
	}

	public boolean saveAll(MultipartFile files, String myVideoClass, String myTagsCn) {
		// 读取本地文件（json）
		// String fileName = "J:\\_ALL\\CODE\\gitee\\constellations\\Capricorn\\WeChat\\爬虫\\tiktok\\data\\2021-12-10--6307840-gmail.json";
//		String fileName = "J:\\_ALL\\CODE\\gitee\\constellations\\Capricorn\\WeChat\\爬虫\\tiktok\\data\\2021-12-10--kinghz-gmail.json";
//		String s = readJsonFile(fileName);
		String s = readJsonFile(files);

		// 文件解析入库
		JSONObject jobj = JSON.parseObject(s);
		Integer statusCode = jobj.getInteger("statusCode");
		System.out.println("statusCode = " + statusCode);

		List<Myvideos> myvideosList = new ArrayList<Myvideos>();
		JSONArray itemList = jobj.getJSONArray("itemList");//构建JSONArray数组
		for (int i = 0; i < itemList.size(); i++){
			JSONObject item = (JSONObject)itemList.get(i);
			String _id = (String)item.get("id");
			System.out.println("_id = " + _id);
//            String desc = (String)item.get("desc");
//            System.out.println("desc = " + desc);

			QueryWrapper<Myvideos> queryWrapper = new QueryWrapper<>();
			Map<String,Object> map = new HashMap<>();
			map.put("j_id",_id);
			queryWrapper.allEq(map, false);
			List<Myvideos> video = this.list(queryWrapper);
			// 判断数据库中是否包含_id对应的数据
			if (video.size() >= 1) {
				System.out.println("【已经存在该视频】 ： " + _id);
				continue;
			}

			// 将json对象中的数据，保存在Myvideos对象中
			Myvideos myvideos_new = new Myvideos();
			// 设置参数
			myvideos_new.setMyVideoClass(myVideoClass);
			myvideos_new.setMyTagsCn(myTagsCn);
			// 设置基础数据
			myvideos_new.setJId(_id);
			myvideos_new.setJDesc((String)item.get("desc"));
			myvideos_new.setJCreatetime((Integer) item.get("createTime"));

            // 解析video元素
            // ……
            // ……
            // ……

			// 其他自定义的字段
			myvideos_new.setCreateBy("MyvideosServiceImpl");

			myvideosList.add(myvideos_new);
			System.out.println("【视频入库】 ： " + _id);
		}

		return this.saveBatch(myvideosList);
	}
}

2. pig-ui页面搭建和接口编写

                <el-upload ref="uploadAwesomeVideoInfo" :file-list="uploadAwesomeVideoInfofileList"
                  :action="uploadAwesomeVideoInfoAction" :auto-upload="true"
                  :headers="myHeaders"
                  :multiple="false"
                  :data="uploadAwesomeVideoInfoParam"
                  :before-upload="uploadAwesomeVideoInfoBeforeUpload">
                  <el-button size="small" type="primary" icon="el-icon-upload">视频信息入库</el-button>
                </el-upload>

参数说明：

• headers： 表示请求头，需要添加Authorization！！！
• data： 入库的默认参数，添加了视频类别和标签。

3. 解决错误： 424 Failed Dependency

写完界面后，会发现调用接口，出现424，服务器返回{"code":1,"msg":"token expire","data":"Full authentication is required to access this resource"}

参考资料

• 本文章后台代码 https://gitee.com/constellations/pig
• 本文章前端代码 https://gitee.com/constellations/pig-ui
• 记录在使用 OAuth2 遇到的巨坑 https://www.shangyang.me/2017/06/01/spring-cloud-oauth2-zuul-potholes/
• el-upload 添加请求头authorization https://www.jianshu.com/p/2e9e5bfe0ffd