千万级别瓦片文件夹遍历及导入mongodb

本示例暂时使用了GridFS的存储方式，目前还没有很好的考虑大数据量下，瓦片索引的机制

import java.io.File;

import java.net.UnknownHostException;

import java.util.Queue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.LinkedBlockingQueue;

import java.util.concurrent.TimeUnit;

import com.mongodb.DB;

import com.mongodb.Mongo;

import com.mongodb.MongoException;

import com.mongodb.gridfs.GridFS;

import com.mongodb.gridfs.GridFSInputFile;

public class PackageManager {

private final ExecutorService mThreadPool;

private final Queue<File> mQueue = new LinkedBlockingQueue<File>();

Mongo mg;

DB db;

GridFS gridFS;

// ===========================================================

// Constructors

// ===========================================================

public PackageManager(final int mThreads) {

this.mThreadPool = Executors.newFixedThreadPool(mThreads);

try {

mg = new Mongo();

//mg = new Mongo("localhost", 27017);

} catch (UnknownHostException e) {

e.printStackTrace();

} catch (MongoException e) {

e.printStackTrace();

}

db = mg.getDB("beijingImg");

gridFS= new GridFS(db);

}

// ===========================================================

// Getter & Setter

// ===========================================================

public synchronized void add(final File pTileInfo){

this.mQueue.add(pTileInfo);

spawnNewThread();

}

private synchronized File getNext(){

final File tile = this.mQueue.poll();

final int remaining = this.mQueue.size();

if(remaining % 100 == 0 && remaining > 0) {

System.out.print("(" + remaining +")");

} else {

System.out.print(".");

}

this.notify();

return tile;

}

public synchronized void waitEmpty() throws InterruptedException {

while(this.mQueue.size() > 0){

this.wait();

}

}

public void waitFinished() throws InterruptedException {

waitEmpty();

this.mThreadPool.shutdown();

this.mThreadPool.awaitTermination(6, TimeUnit.HOURS);

}

// ===========================================================

// Methods from SuperClass/Interfaces

// ===========================================================

// ===========================================================

// Methods

// ===========================================================

private void spawnNewThread() {

this.mThreadPool.execute(new DownloadRunner());

}

// ===========================================================

// Inner and Anonymous Classes

// ===========================================================

private class DownloadRunner implements Runnable {

private File mTile;

private String saveFileName;

//private File mDestinationFile;

public DownloadRunner() {

}

private void init(final File pTileInfo) {

this.mTile = pTileInfo;

String fileName = mTile.getName().substring(0, mTile.getName().indexOf("."));

//System.out.println("fileName：" + fileName);

String row = mTile.getParentFile().getPath().substring(mTile.getParentFile().getPath().lastIndexOf("\\") + 1);

//System.out.println("row：" + row);

String level = mTile.getParentFile().getParentFile().getPath().substring(mTile.getParentFile().getParentFile().getPath().lastIndexOf("\\") + 1);

//System.out.println("level：" + level);

saveFileName = level + "_" + row + "_" + fileName;

}

@Override

public void run() {

try{

init(PackageManager.this.getNext());

// gridFS= new GridFS(db);

GridFSInputFile gfs = gridFS.createFile(mTile);

//GridFSInputFile gridFSInputFile = myFS.createFile(inputStream);

gfs.put("filename", saveFileName);

gfs.save();

}catch(Exception e){

}

}

}

}

 

 

import java.io.File;

import java.util.ArrayList;

import java.util.LinkedList;

import java.util.List;

import org.junit.Test;

public class TilePackager {

public static List<File> files = new ArrayList<File>();

PackageManager packageManager;

public int count = 0;

public static void main(String[] args) {

/*File ss = new File("E:\\瓦片数据\\北京影像");

String fileName = ss.getName().substring(0, ss.getName().indexOf("."));

System.out.println("fileName：" + fileName);

String row = ss.getParentFile().getPath().substring(ss.getParentFile().getPath().lastIndexOf("\\") + 1);

System.out.println("row：" + row);

String level = ss.getParentFile().getParentFile().getPath().substring(ss.getParentFile().getParentFile().getPath().lastIndexOf("\\") + 1);

System.out.println("level：" + level);*/

//入库

/*TilePackager tp = new TilePackager();

tp.packageManager = new PackageManager(30);

tp.getTiles(new File("E:\\瓦片数据\\北京影像"));

System.out.println(files.size());*/

TilePackager tp = new TilePackager();

tp.getDirectorySize(new File("E:\\瓦片数据\\北京矢量"), tp.list);

}

//测试成功

@Test

public void listFiles() {

long count = 0;

long countd = 0;

// 链表

LinkedList<File> list = new LinkedList<File>();

File dir = new File("D:\\opt");

File[] file = dir.listFiles();

for (int i = 0; i < file.length; i++) {

if (file[i].isDirectory())

// 把第一层的目录，全部放入链表

list.add(file[i]);

else

count++;

System.out.println("文件" + count + ":" + file[i].getAbsolutePath());

}

File tmp = null;

// 循环遍历链表

while (!list.isEmpty()) {

// 把链表的第一个记录删除

tmp = list.removeFirst();

// 如果删除的目录是一个路径的话

if (tmp.isDirectory()) {

// 列出这个目录下的文件到数组中

file = tmp.listFiles();

if (file == null)

continue;

// 遍历文件数组

for (int i = 0; i < file.length; i++) {

if (file[i].isDirectory())

// 如果遍历到的是目录，则继续加入链表

list.add(file[i]);

else

count++;

System.out.println("文件" + count + ":" + file[i].getAbsolutePath());

}

} else {

countd++;

System.out.println("目录[" + countd + "]路径:" + tmp.getAbsolutePath());

}

}

}

public void getTiles(File file){

File[] files = file.listFiles();

if (files == null || files.length == 0) {

return;

}

for(File childFile : files){

if(childFile.isDirectory()){

getTiles(childFile);

}else{

String fileName = childFile.getName();

String filePath = childFile.getPath();

if(fileName.endsWith("png") || fileName.endsWith("jpg")){

packageManager.add(childFile);

}

}

}

}

LinkedList<File> list = new LinkedList<File>(); // 保存待遍历文件夹的列表

// 非递归遍历

private void getDirectorySize(File file, LinkedList<File> list) {

getOneDir(file, list); // 调用遍历文件夹根目录文件的方法

File tmp;

while (!list.isEmpty()) {

tmp = (File) list.removeFirst();

// 这个地方的判断有点多余，但是为了保险还是给个判断了，正常情况列表中是只有文件夹的

// 但是不排除特殊情况，例如：本身是文件夹的目标在压入堆栈之后变成了文件

if (tmp.isDirectory()) {

getDirectorySize(tmp, list);

} else {

System.out.println("file==>" + tmp.getAbsolutePath());

}

}

}

int sumdir = 0;

// 遍历指定文件夹根目录下的文件

private void getOneDir(File file, LinkedList<File> list) {

// 每个文件夹遍历都会调用该方法

System.out.println("Dir==>" + file.getAbsolutePath());

File[] files = file.listFiles();

sumdir += 1;

if (files == null || files.length == 0) {

return;

}

for (File f : files) {

if (f.isDirectory()) {

list.add(f);

} else {

// 这里列出当前文件夹根目录下的所有文件

System.out.println("file==>" + f.getAbsolutePath());

}

}

}

}