单线程下载文件
如果想多线程下载文件，得先学会单线程下载文件
流程：
1. 想要下载网络文件得要用HttpURLConnection进行连接
2. 获取资源后，通过InputStream 进行读取，通过fileOutputStream 进行写入文件
3. 在new一个线程进行检测写入文件的进度
4. 下载完成后务必让二个流关闭，尽量写在finally块里
代码如下
public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException {
//是否删除分片的目录
boolean delete = true;
//获取桌面页面
File desktopDir = FileSystemView.getFileSystemView().getHomeDirectory();
String pathurl = "https://v3-default.ixigua.com/d1d5a61f4873557d59cef9cfa4fd091f/628c90b4/video/tos/cn/tos-cn-v-6f4170/1ac798d04da74eb3b1062b0a5ef5420d/";
//获取后缀名
String suffix;
if (pathurl.contains("?")) {
String substring = pathurl.substring(0, pathurl.lastIndexOf('?'));
suffix = substring.substring(substring.lastIndexOf('.') + 1, substring.length());
} else {
suffix = pathurl.substring(pathurl.lastIndexOf('.') + 1, pathurl.length());
}
//指定文件的名字
String name = "";
//如果url没有后缀，请自己指定
suffix = "mp4";
System.out.println("后缀名:" + suffix);
URL url = new URL(pathurl);
//从连接获取文件
long start = System.currentTimeMillis();
String data = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String path = desktopDir.getAbsolutePath() + "/yj" + data;
//单线程网络文件下载
spilt4(url, desktopDir.getAbsolutePath() + "/" + name + data + "." + suffix);
System.out.println("下载文件名:" + name + data + "." + suffix);
System.out.println("耗时时间:" + (System.currentTimeMillis() - start));
}
volatile static double sumLen = 0;
public static void spilt4(URL url, String to) throws IOException {
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
//设置允许接收消息
urlConnection.setDoInput(true);
urlConnection.connect();
int fileSize = urlConnection.getContentLength();//大小;
System.out.println("文件总共大小：" + fileSize + "字节");
InputStream inputStream = urlConnection.getInputStream();
FileOutputStream fileOutputStream = null;
//进度
new Thread(() -> {
while (sumLen < fileSize) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("显示进度:---------" + String.format("%.2f", (sumLen / fileSize) * 100) + "%-----------");
}
}).start();
try {
fileOutputStream = new FileOutputStream(to);
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = inputStream.read(bytes)) != -1) {
sumLen += len;
fileOutputStream.write(bytes, 0, len);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
inputStream.close();
if (null != fileOutputStream) {
fileOutputStream.flush();
fileOutputStream.close();
}
}
System.out.println("成功");
}
多线程下载文件
学会单线程下载文件后，来进行多线程，多线程就是很多线程进行写入文件。
流程：
1. 获取网络资源，一样进行连接，进行流读取和写入
2. 使用多线程尽量线程池，并不是线程越多越越快，我这边线程控制在电脑核数*2-1个线程
3. 原来是一个线程在写文件，现在多线程，所有在写入的文件进行操作。我们每个线程只写属于自己的哪一点空间的数据。写完后，然后把所有线程写入的文件，进行合并，就可以得到最终的文件。
4. 防止文件生成得太多，如果下载的文件过大，写入生成的文件随着跟着变大
代码如下
public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException {
//是否删除分片的目录
boolean delete = true;
//获取桌面页面
File desktopDir = FileSystemView.getFileSystemView().getHomeDirectory();
String pathurl = "http://127.0.0.1/upload/aaa.mp4";
//获取后缀名
String suffix;
if (pathurl.contains("?")) {
String substring = pathurl.substring(0, pathurl.lastIndexOf('?'));
suffix = substring.substring(substring.lastIndexOf('.') + 1, substring.length());
} else {
suffix = pathurl.substring(pathurl.lastIndexOf('.') + 1, pathurl.length());
}
//指定文件的名字
String name = "";
//如果url没有后缀，请自己指定
suffix = "mp4";
System.out.println("后缀名:" + suffix);
URL url = new URL(pathurl);
//从连接获取文件
long start = System.currentTimeMillis();
String data = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String path = desktopDir.getAbsolutePath() + "/yj" + data;
//多线程网络文件下载
spilt3(url, path);
merge(path, desktopDir.getAbsolutePath() + "/" + name + data + "." + suffix, delete);
//本地文件传输
System.out.println("下载文件名:" + name + data + "." + suffix);
System.out.println("耗时时间:" + (System.currentTimeMillis() - start));
}
public static void spilt3(URL url, String to) throws IOException {
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
//设置允许接收消息
urlConnection.setDoInput(true);
urlConnection.connect();
File file = new File(to);
if (!file.exists()) {
file.mkdirs();
}
int fileSize = urlConnection.getContentLength();//大小;
System.out.println("文件总共大小：" + fileSize + "字节");
int size = 20;//默认1m
long mb100 = 1024 * 1024 * 100; //100mb
long gb1 = 1024 * 1024 * 1024; //1gb
long gb10 = 10L * 1024 * 1024 * 1024; //1gb
if (mb100 <= fileSize && gb1 > fileSize) {
size = 10;
} else if (gb1 <= fileSize && gb10 > fileSize) {
size = 100;
} else if (gb10 <= fileSize){
size = 1024;
}
// 将MB单位转为为字节B
int m = size * 1024 * 1024;
double m1 = m;
// 计算最终会分成几个文件
int count = (int) Math.ceil(fileSize / m1);
System.out.println("文件分配：" + count + "块");
ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2 - 1);
try {
CountDownLatch countDown = new CountDownLatch(count);
for (int i = 0; i < count; i++) {
int index = i;
service.execute(() -> {
int beginPoint = index * m;
int endPoint;
if (index == (count - 1)) {
endPoint = fileSize;
} else {
endPoint = beginPoint + m;
}
InputStream inputStream = null;
FileOutputStream fileOutputStream = null;
try {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
//设置允许接收消息
conn.setDoInput(true);
conn.connect();
inputStream = conn.getInputStream();
/**inputStream = urlConnection.getInputStream(); **/
inputStream.skip(beginPoint);
int start = beginPoint;
fileOutputStream = new FileOutputStream(to + "/" + index);
byte[] bytes = new byte[m];
int len;
System.out.println("第" + index + "分片开始下载: 起始字节数：" + beginPoint + ",结束字节数：" + endPoint);
while (start < endPoint) {
len = inputStream.read(bytes);
if ((start + len) >= endPoint) {
len = endPoint - start;
start = endPoint;
} else {
start += len;
}
if (len < 0) {
System.out.println("len=" + len + ",index=" + index);
start = endPoint;
break;
}
// System.out.println("线程名:"+Thread.currentThread().getName()+
// "--bytef="+bytes.length+",i="+ finalI +",len="+len);
fileOutputStream.write(bytes, 0, len);
}
System.out.println("------------------第" + index + "分片,下载完成-------------------------");
countDown.countDown();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
inputStream.close();
if (null != fileOutputStream) {
fileOutputStream.flush();
fileOutputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
});
}
countDown.await();
service.shutdown();
System.out.println("分解成功");
} catch (Exception e) {
e.printStackTrace();
}
}
public static void merge(String from, String to, Boolean delete) throws IOException {
File t = new File(to);
FileInputStream in = null;
FileChannel inChannel = null;
FileOutputStream out = new FileOutputStream(t, true);
FileChannel outChannel = out.getChannel();
File f = new File(from);
// 获取目录下的每一个文件名，再将每个文件一次写入目标文件
if (f.isDirectory()) {
List<File> list = getAllFileAndSort(from);
// 记录新文件最后一个数据的位置
long start = 0;
for (File file : list) {
in = new FileInputStream(file);
inChannel = in.getChannel();
// 从inChannel中读取file.length()长度的数据，写入outChannel的start处
outChannel.transferFrom(inChannel, start, file.length());
start += file.length();
in.close();
inChannel.close();
}
}
if (delete) {
deleteFile(f);
}
System.out.println("合成成功");
out.close();
outChannel.close();
}
private static List<File> getAllFileAndSort(String dirPath) {
File dirFile = new File(dirPath);
File[] listFiles = dirFile.listFiles();
List<File> list = Arrays.asList(listFiles);
Collections.sort(list, (o1, o2) -> {
return Integer.parseInt(o1.getName()) - Integer.parseInt(o2.getName());
});
return list;
}
//删除File对象中抽象的路径方法
private static void deleteFile(File dir) {
//将file封装的路径下对象转换为数组
File[] files = dir.listFiles();
//判断这个数组为不为空,如果不为空,就执行内部代码
if (files != null) {
for (File file : files) {
//判断是否为文件
if (file.isFile()) {
//如果为文件,执行删除
file.delete();
} else {
//如果不为文件,就(递归)进入这个文件夹,删除文件
deleteFile(file);
}
}
//删除全部文件后删除空文件夹,最后删除自己
dir.delete();
}
}
多线程本地文件传输
多线程网络文件都会了，本地文件传输就so easy了
合并文件代码用多线程合并代码，都一样的。
代码如下
public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException {
//是否删除分片的目录
boolean delete = true;
//获取桌面页面
File desktopDir = FileSystemView.getFileSystemView().getHomeDirectory();
//指定文件的名字
String name = "";
//如果url没有后缀，请自己指定
suffix = "mp4";
System.out.println("后缀名:" + suffix);
//从连接获取文件
long start = System.currentTimeMillis();
String data = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
//本地文件传输
spilt("D:\\software\\test\\netty\\io\\src\\main\\java\\com\\yj/io/nio/img/" + name + "." + suffix,
"D:\\software\\test\\netty\\io\\src\\main\\java\\com\\yj/io/nio/img/yj");
merge("D:\\software\\test\\netty\\io\\src\\main\\java\\com\\yj/io/nio/img/yj",
"D:\\software\\test\\netty\\io\\src\\main\\java\\com\\yj/io/nio/img/" + name + "下载文件" + "." + suffix, delete);
System.out.println("下载文件名:" + name + data + "." + suffix);
System.out.println("耗时时间:" + (System.currentTimeMillis() - start));
}
public static void spilt(String from, String to) throws IOException, InterruptedException {
File f = new File(from);
FileInputStream in = new FileInputStream(f);
FileOutputStream out = null;
FileChannel inChannel = in.getChannel();
FileChannel outChannel = null;
int size = 1;//默认1m
ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
long mb100 = 1024 * 1024 * 100; //100mb
long gb1 = 1024 * 1024 * 1024; //1gb
if (mb100 <= f.length() && gb1 > f.length()) {
size = 10;
} else if (gb1 <= f.length()) {
size = 100;
}
// 将MB单位转为为字节B
long m = size * 1024 * 1024;
// 计算最终会分成几个文件
int count = (int) (f.length() / m);
CountDownLatch countDown = new CountDownLatch(count);
for (int i = 0; i <= count; i++) {
// 生成文件的路径
String t = to + "/" + i;
File file = new File(to);
if (!file.exists()) {
file.mkdirs();
}
out = new FileOutputStream(t);
outChannel = out.getChannel();
int finalI = i;
FileChannel finalOutChannel = outChannel;
executorService.execute(() -> {
// 从inChannel的m*i处，读取固定长度的数据，写入outChannel
try {
if (finalI != count) {
inChannel.transferTo(m * finalI, m, finalOutChannel);
} else {// 最后一个文件，大小不固定，所以需要重新计算长度
inChannel.transferTo(m * finalI, f.length() - m * count, finalOutChannel);
}
countDown.countDown();
} catch (IOException e) {
e.printStackTrace();
}
});
}
countDown.await();
executorService.shutdown();
System.out.println("分解成功");
out.close();
outChannel.close();
in.close();
inChannel.close();
}
git项目地址
好了学习到处结束，如果对你有帮助可以给我一个小小赞，感谢。
139
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
