这段时间研究java的io与nio框架,一时兴起决定用java实现一个下载工具,主要有下面几个功能
1)支持多任务下载
2)支持多线程下载
3) 支持断点续传
4)错误线程任务的重新调度
用到的技术点
1) http协议的range头技术
2)java的多线程
3)java的网络编程,主要是HttpUrlConnection类
4)java的io文件操作, 如RandomAccessFile类
速度还不错,基本上和浏览器下载差不多,可能比它还要快,我只开了10个线程。
-
package org.blackfoxer.cat;
-
import java.io.File;
-
import java.io.IOException;
-
import java.io.RandomAccessFile;
-
import java.io.UnsupportedEncodingException;
-
import java.HttpURLConnection;
-
import java.URL;
-
import java.URLDecoder;
-
import java.text.DecimalFormat;
-
import java.util.concurrent.CountDownLatch;
-
import java.util.concurrent.TimeUnit;
-
import java.util.regex.Matcher;
-
import java.util.regex.Pattern;
-
public class Job {
-
private int fileSize;
-
private String fileName;
-
private int connectTimeout = 10000;
-
private int readTimeout = 20000;
-
private String url;
-
private String storeDir;
-
private int taskNum;
-
private String jobId;
-
private int[] startIndexes;
-
private int[] endIndexes;
-
private int[] progress;
-
private Task[] tasks;
-
private File storeDirFile;
-
private File dtDirFile;
-
private File localFile;
-
private ThreadLocal<RandomAccessFile> rafLocalTl;
-
private ThreadLocal<RandomAccessFile> rafOffsetTl;
-
private CountDownLatch latch;
-
private ProgressThread pt;
-
public Job(String url, String storeDir, int taskNum) throws IOException {
-
this.url = url;
-
this.storeDir = storeDir;
-
this.taskNum = taskNum;
-
this.startIndexes = new int[taskNum];
-
this.endIndexes = new int[taskNum];
-
this.progress = new int[taskNum];
-
this.tasks = new Task[taskNum];
-
this.latch = new CountDownLatch(taskNum);
-
this.jobId = Math.abs(url.hashCode()) + "_" + taskNum;
-
this.rafLocalTl = new ThreadLocal<RandomAccessFile>();
-
this.rafOffsetTl = new ThreadLocal<RandomAccessFile>();
-
this.pt = new ProgressThread();
-
}
-
public void startJob() throws Exception {
-
long start = System.currentTimeMillis();
-
System.out.println("开始下载文件...");
-
boolean j = fetchFileMetaInfo();
-
if (j) {
-
assignTasks();
-
createFiles();
-
startTasks();
-
openProgressThread();
-
waitForCompeletion();
-
long end = System.currentTimeMillis();
-
System.out.println("下载完毕,全程耗时" + (end - start) + "ms");
-
} else {
-
System.out.println("获取文件长度或文件名失败,请重试");
-
}
-
}
-
private void openProgressThread() {
-
this.pt.start();
-
}
-
private void waitForCompeletion() throws Exception {
-
latch.await();
-
deleteFiles();
-
pt.join();
-
}
-
private void deleteFiles() {
-
if (dtDirFile != null) {
-
File[] subFiles = dtDirFile.listFiles();
-
for (File subFile : subFiles) {
-
subFile.delete();
-
}
-
dtDirFile.delete();
-
}
-
}
-
// 1.fetch file size and file name
-
private boolean fetchFileMetaInfo() throws IOException {
-
HttpURLConnection connection = createConnection();
-
connection.setRequestMethod("GET");
-
if (connection.getResponseCode() == 200) {
-
this.fileSize = connection.getContentLength();
-
String disposition = connection.getHeaderField("Content-Disposition");
-
if (disposition == null) {
-
parseFileNameFromUrl(url);
-
} else {
-
parseFileNameFromDisposition(disposition);
-
}
-
if (this.fileName == null || this.fileSize < 0) {
-
return false;
-
}
-
System.out.println("找到文件资源,长度为" + fileSize + ",资源名称为" + fileName);
-
return true;
-
}
-
return false;
-
}
-
private void parseFileNameFromUrl(String url) throws UnsupportedEncodingException {
-
this.fileName = url.substring(url.lastIndexOf("/") + 1, url.length());
-
if (this.fileName.contains("%")) {
-
this.fileName = URLDecoder.decode(this.fileName, "UTF-8");
-
}
-
}
-
private void parseFileNameFromDisposition(String disposition) throws UnsupportedEncodingException {
-
Pattern pattern = Patternpile(".+filename=\"(.+?)\".*");
-
Matcher matcher = pattern.matcher(disposition);
-
if (matcher.matches()) {
-
this.fileName = new String(matcher.group(1).getBytes("ISO-8859-1"), "UTF-8");
-
} else {
-
parseFileNameFromUrl(url);
-
}
-
}
-
public HttpURLConnection createConnection() throws IOException {
-
URL urlObj = new URL(url);
-
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
-
connection.setConnectTimeout(connectTimeout);
-
connection.setReadTimeout(readTimeout);
-
connection.setRequestProperty("Accept-Charset", "UTF-8");
-
connection.setRequestProperty("contentType", "UTF-8");
-
return connection;
-
}
-
// 2.assign every task start index and end index out of the file
-
private void assignTasks() throws IOException {
-
for (int i = 0; i < taskNum; i++) {
-
int size = fileSize / taskNum;
-
int startIndex = i * size;
-
int endIndex = i == taskNum - 1 ? fileSize - 1 : i * size + size - 1;
-
this.startIndexes[i] = startIndex;
-
this.endIndexes[i] = endIndex;
-
}
-
}
-
// 3.create the local file and temp directory
-
private void createFiles() throws IOException {
-
storeDirFile = new File(storeDir);
-
storeDirFile.mkdirs();
-
localFile = new File(storeDirFile, fileName);
-
dtDirFile = new File(storeDirFile, "." + jobId);
-
dtDirFile.mkdirs();
-
if (!localFile.exists()) {
-
RandomAccessFile raf = new RandomAccessFile(localFile, "rw");
-
raf.setLength(fileSize);
-
raf.close();
-
}
-
}
-
// 4.let the task start to do their work
-
private void startTasks() throws IOException {
-
for (int i = 0; i < taskNum; i++) {
-
Task task = new Task(this, i);
-
tasks[i] = task;
-
task.start();
-
}
-
}
-
private int totalReadBytes() {
-
int totalReadBytes = 0;
-
for (int i = 0; i < progress.length; i++) {
-
totalReadBytes += progress[i];
-
}
-
return totalReadBytes;
-
}
-
public int[] getStartIndexes() {
-
return startIndexes;
-
}
-
public int[] getEndIndexes() {
-
return endIndexes;
-
}
-
public void writeLocalFile(int startIndex, byte[] buf, int off, int len) throws IOException {
-
if (rafLocalTl.get() == null) {
-
RandomAccessFile raf = new RandomAccessFile(localFile, "rw");
-
rafLocalTl.set(raf);
-
}
-
RandomAccessFile raf = rafLocalTl.get();
-
raf.seek(startIndex);
-
raf.write(buf, off, len);
-
}
-
// 5.let task to report their progress
-
public void reportProgress(int index, int readBytes) {
-
progress[index] = readBytes;
-
}
-
public void closeTaskResource(int index) throws IOException {
-
RandomAccessFile raf = rafLocalTl.get();
-
if (raf != null) {
-
raf.close();
-
}
-
raf = rafOffsetTl.get();
-
if (raf != null) {
-
raf.close();
-
}
-
}
-
public void commitOffset(int index, int offset) throws IOException {
-
File offsetFile = new File(dtDirFile, String.valueOf(index));
-
if (rafOffsetTl.get() == null) {
-
RandomAccessFile raf = new RandomAccessFile(offsetFile, "rw");
-
rafOffsetTl.set(raf);
-
}
-
RandomAccessFile raf = rafOffsetTl.get();
-
raf.seek(0);
-
raf.writeInt(offset);
-
}
-
public int readOffset(int index) throws IOException {
-
File offsetFile = new File(dtDirFile, String.valueOf(index));
-
if (offsetFile.exists()) {
-
RandomAccessFile raf = new RandomAccessFile(offsetFile, "rw");
-
raf.seek(0);
-
int offset = raf.readInt();
-
raf.close();
-
return offset;
-
}
-
return 0;
-
}
-
public void reStartTask(int index) throws IOException {
-
Task task = new Task(this, index);
-
tasks[index] = task;
-
task.start();
-
System.out.println("任务" + index + "发生错误,重新调度该任务");
-
}
-
public void taskFinished() {
-
latch.countDown();
-
}
-
private class ProgressThread extends Thread {
-
private DecimalFormat decimalFormat = new DecimalFormat();
-
public void run() {
-
decimalFormat.applyPattern("0.0");
-
while (true) {
-
try {
-
int endPointX = totalReadBytes();
-
TimeUnit.SECONDS.sleep(1);
-
int endPointY = totalReadBytes();
-
int waitSeconds = 1;
-
while (endPointY - endPointX == 0) {
-
TimeUnit.SECONDS.sleep(1);
-
waitSeconds++;
-
endPointY = totalReadBytes();
-
}
-
int speed = (endPointY - endPointX) / waitSeconds;
-
String speedStr = speed > 1024 ? speed/1024+"kb/s":speed+"b/s";
-
String percent = decimalFormat.format(endPointY * 100.0 / fileSize);
-
int remainSeconds = (fileSize - endPointY)/speed;
-
System.out.println("下载完成"+percent+"%,速度"+speedStr+",估计还需要"+remainSeconds+"秒");
-
if("100.0".equals(percent)) {
-
break;
-
}
-
} catch (InterruptedException e) {
-
e.printStackTrace();
-
}
-
}
-
}
-
}
-
}
-
package org.blackfoxer.cat;
-
import java.io.IOException;
-
import java.io.InputStream;
-
import java.HttpURLConnection;
-
public class Task extends Thread {
-
private Job owner;
-
private int index;
-
private int readBytes;
-
private int startIndex;
-
private int endIndex;
-
public Task(Job owner,int index) throws IOException {
-
this.owner = owner;
-
this.index = index;
-
if(owner.readOffset(index)!=0) {
-
this.readBytes = owner.readOffset(index)-owner.getStartIndexes()[index];
-
owner.reportProgress(index, readBytes);
-
}
-
this.startIndex = owner.getStartIndexes()[index]+readBytes;
-
this.endIndex = owner.getEndIndexes()[index];
-
}
-
public void run() {
-
InputStream inputStream = null;
-
HttpURLConnection connection = null;
-
try {
-
if(startIndex > endIndex) {
-
owner.taskFinished();
-
return;
-
}
-
connection = owner.createConnection();
-
connection.setRequestMethod("GET");
-
String range = "bytes="+startIndex+"-"+endIndex ;
-
connection.setRequestProperty("Range", range);
-
if(connection.getResponseCode()==206) {
-
inputStream = connection.getInputStream();
-
int len = -1;
-
byte buf[] = new byte[1024];
-
int offset = startIndex;
-
while((len=inputStream.read(buf))!=-1) {
-
owner.writeLocalFile(offset,buf,0,len);
-
readBytes+=len;
-
offset+=len;
-
ownermitOffset(index,offset);
-
owner.reportProgress(index,readBytes);
-
}
-
owner.taskFinished();
-
}
-
} catch (IOException e) {
-
e.printStackTrace();
-
try {
-
owner.reStartTask(index);
-
} catch (IOException e1) {
-
e1.printStackTrace();
-
}
-
} finally {
-
if(inputStream != null) {
-
try {
-
inputStream.close();
-
} catch (IOException e) {
-
e.printStackTrace();
-
}
-
}
-
if(connection != null) {
-
connection.disconnect();
-
}
-
try {
-
owner.closeTaskResource(index);
-
} catch (IOException e) {
-
e.printStackTrace();
-
}
-
}
-
}
-
}
-
package org.blackfoxer.cat;
-
import java.io.BufferedReader;
-
import java.io.IOException;
-
import java.io.InputStreamReader;
-
public class JavaXunlei {
-
private static final BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
-
private static String storeDir = null;
-
public static void main(String args[]) throws Exception {
-
storeDir = getInput("请先设置你的文件存储目录:");
-
int taskNum = getIntInput("请输入你的开启下载的线程数:");
-
while(true) {
-
String url = getInput("请输入文件链接地址:");
-
Job job = new Job(url,storeDir,taskNum);
-
job.startJob();
-
}
-
}
-
private static int getIntInput(String message) throws IOException {
-
String number = getInput(message);
-
while(!number.matches("\\d+")) {
-
System.out.println("线程数必须是1个整数");
-
number = getInput(message);
-
}
-
return Integer.parseInt(number);
-
}
-
private static String getInput(String message) throws IOException {
-
System.out.print(message);
-
String line = in.readLine();
-
while(line == null || line.trim().length()<1) {
-
System.out.print(message);
-
line = in.readLine();
-
}
-
return line.trim();
-
}
-
}
更多推荐
java版迅雷下载源码分享
发布评论