fileSystem.copyFromLocalFile和 IOUtils.copyBytes区别

转自：https://blog.csdn.net/a822631129/article/details/45914899
之前上传下载hdfs文件都是使用的IOUtils.copyBytes(…)，不过也看见过别的方式FileSystem.copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst)等通过FileSystem操作文件的所以就追踪了一下FileSystem.copyFromLocalFile的执行过程。

<span style="font-size:18px;"> public void copyFromLocalFile(boolean delSrc, boolean overwrite, 
                                Path src, Path dst)
    throws IOException {
    Configuration conf = getConf();
    FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
  }</span>

//调用了 FileUtil.copy(),进到这个copy()里：

<span style="font-size:18px;"> public static boolean copy(FileSystem srcFS, Path src, FileSystem dstFS, Path dst, boolean deleteSource,
                             boolean overwrite, Configuration conf) throws IOException {
             FileStatus fileStatus = srcFS.getFileStatus(src);
             return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
       }</span>

//这个copy把第二个参数的Path src改为了 FileStatus fileStatus，再进到这个copy()：

 public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst,
                             boolean deleteSource,boolean overwrite, Configuration conf) throws IOException {
    Path src = srcStatus.getPath();
    dst = checkDest(src.getName(), dstFS, dst, overwrite);
    if (srcStatus.isDirectory()) {
      checkDependencies(srcFS, src, dstFS, dst); //校验目标路径是否是源路径或者子路径，如果是就返回错误信息
      if (!dstFS.mkdirs(dst)) {
        return false;
      }
      FileStatus contents[] = srcFS.listStatus(src);
      for (int i = 0; i < contents.length; i++) {
        copy(srcFS, contents[i], dstFS,
             new Path(dst, contents[i].getPath().getName()),
             deleteSource, overwrite, conf); //这个copy是遍历源文件目录下的文件传到hdfs
      }
    } else {
      InputStream in=null;
      OutputStream out = null;
      try {
        in = srcFS.open(src);
      <span style="color:#cc0000;">  out = dstFS.create(dst, overwrite); //创建hdfs的路径输出流，用于后面的流的读写</span>
        IOUtils.copyBytes(in, out, conf, true);
      } catch (IOException e) {
        IOUtils.closeStream(out);
        IOUtils.closeStream(in);
        throw e;
      }
    }
    if (deleteSource) {
      return srcFS.delete(src, true);
    } else {
      return true;
    } 
  }</span>

//这个copy最后还是调用了 IOUtils.copyBytes(in, out, conf, true) ，再进到copyBytes(in, out, conf, true)里：

<span style="font-size:18px;">public static void copyBytes(InputStream in, OutputStream out, int buffSize, boolean close) 
    throws IOException {
      copyBytes(in, out, buffSize);
      if(close) {
        out.close();
        out = null;
        in.close();
        in = null;
      }
  }</span>

//这个是调用了 copyBytes(in, out, buffSize)，继续追进去：

<span style="font-size:18px;"> public static void copyBytes(InputStream in, OutputStream out, int buffSize) 
    throws IOException {
    PrintStream ps = out instanceof PrintStream ? (PrintStream)out : null;
    byte buf[] = new byte[buffSize];
    int bytesRead = in.read(buf);
    while (bytesRead >= 0) {
      out.write(buf, 0, bytesRead);
      if ((ps != null) && ps.checkError()) {
        throw new IOException("Unable to write to output stream.");
      }
      bytesRead = in.read(buf);
    }
  }</span>

//到这结束了，这个就把本件以流的形式写到了hdfs文件对应的存储地方了。不过有必要看一下是怎么把hdfs path转为输出流的即上面的深红色标注的 out = dstFS.create(dst, overwrite);