GetHFileFileInfo.java

package hfile.inputformat;

import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.HFile;

public class GetHFileFileInfo {
	/**
	 * ���Ŀ���ļ��ڼ�Ⱥ���Ƿ����
	 * 
	 * @param path_str
	 * @param cluster_conf
	 * @return
	 */
	public static boolean checkPath(String cluster_str,
			Configuration cluster_conf) {
		boolean result_bool = false;
		FileSystem cluster_fs;
		Path cluster_path = new Path(cluster_str);
		try {
			cluster_fs = FileSystem.get(cluster_conf);
			result_bool = cluster_fs.exists(cluster_path);

			cluster_fs.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return result_bool;
	}

	public static void main(String[] args) throws IOException {
		// Configuration conf = new Configuration();
		// Path hdfs = new Path(args[0]);
		// FileSystem fs = FileSystem.get(conf);
		// System.out.println("hdfs get name = " + hdfs.getName());
		// System.out.println("hdfs get parent = " +
		// hdfs.toString().toString());
		// if (hdfs.toString().contains("/c/")) {
		// System.out.println("contain /c/");
		// }

		// // ------------write hfile
		// // create local file system
		// FileSystem fs = new RawLocalFileSystem();
		// fs.setConf(new Configuration());
		//
		// // block size = 1kb
		// HFile.Writer hwriter = new HFile.Writer(fs, new Path(args[0]), 1,
		// (Compression.Algorithm) null, null);
		//
		// // create key & value, the value is 8kb, larger than 1kb
		// byte[] key = "www.data-works.org".getBytes();
		// byte[] value = new byte[8 * 1024];
		// for (int i = 0; i < 8 * 1024; i++) {
		// value[i] = '0';
		// }
		// // add values to hfile
		// for (int i = 0; i < 10; i++) {
		// hwriter.append(key, value);
		// }
		//
		// // close hfile
		// hwriter.close();

		// -------------read hfile
		Configuration conf = new Configuration();
		// create local file system
		// FileSystem fs = new RawLocalFileSystem();
		// fs.initialize(URI.create("file:///"), conf);
		// create hdfs file system
		String hdfs = "hdfs://m0:9000/hbase/test/18703084e9def72a245c531b20857986/";
		FileSystem fs = FileSystem.get(conf);
		fs.initialize(URI.create(hdfs), conf);
		fs.setConf(conf);
		Path hdfsPath = new Path(hdfs);

		System.out.println("the all count = "
				+ totalHFileKeyValue(fs, hdfsPath));

		// ArrayList<Path> list = getHDFSHFilePath(fs, hdfsPath);
		// for (Path tmp : list) {
		// System.out.println("the list path = " + tmp.toString());
		// }

		// HFile.Reader hreader = new HFile.Reader(fs, new Path(args[0]), null,
		// false);

		// args: hdfs://ibm-m0:9000/hbase/rowcounterResult/*/c/*
		// HFile.Reader hreader = new HFile.Reader(fs, new Path(args[0]), null,
		// false);

		// // // loadFileInfo
		// hreader.loadFileInfo();
		// // number of KV entries in this HFile
		// int count = hreader.getEntries();
		// System.out.println("the count = " + count);

		// HFileScanner hscanner = hreader.getScanner(false, false);
		// // seek to the start position of the hfile.
		// hscanner.seekTo();
		// // print values.
		// int index = 1;
		// while (hscanner.next()) {
		// System.out.println("index: " + index++);
		// System.out.println("key: " + hscanner.getKeyString());
		// System.out.println("value: " + hscanner.getValueString());
		// }
		// // close hfile.
		// hreader.close();
	}

	public static long totalHFileKeyValue(FileSystem fs, Path hdfsPath)
			throws IOException {
		long count = 0;
		for (FileStatus tmp : fs.listStatus(hdfsPath)) {
			for (FileStatus tmp1 : fs.listStatus(tmp.getPath())) {
				for (FileStatus tmp2 : fs.listStatus(tmp1.getPath())) {
					if (tmp2.getPath().toString().contains("/c1/")
							&& fs.isFile(tmp2.getPath())) {
						HFile.Reader hreader = new HFile.Reader(fs, tmp2
								.getPath(), null, false);
						// // loadFileInfo
						hreader.loadFileInfo();
						// number of KV entries in this HFile
						count += hreader.getEntries();
						System.out.println("tmp2 path = "
								+ tmp2.getPath().toString());
						System.out.println("the tmp count = "
								+ hreader.getEntries());
						System.out.println("the count = " + count);
					}
				}

			}
		}
		return count;
	}

	public static int sumOneHFileKeyValue(FileSystem fs, Path hdfsPath)
			throws IOException {
		int count = 0;
		for (FileStatus tmp : fs.listStatus(hdfsPath)) {
			if (tmp.getPath().toString().contains("/c1/")
					&& fs.isFile(tmp.getPath())) {
				HFile.Reader hreader = new HFile.Reader(fs, tmp.getPath(),
						null, false);
				// // loadFileInfo
				hreader.loadFileInfo();
				// number of KV entries in this HFile
				count += hreader.getEntries();
				System.out.println("the count = " + count);
			} else {
				count += sumOneHFileKeyValue(fs, tmp.getPath());
			}
		}
		return count;
	}

	public static ArrayList<Path> getHDFSHFilePath(FileSystem fs, Path hdfsPath)
			throws IOException {
		ArrayList<Path> list = new ArrayList<Path>();
		for (FileStatus tmp : fs.listStatus(hdfsPath)) {
			if (!tmp.getPath().toString().contains("/c1/")
					|| !fs.isFile(tmp.getPath())) {
				getHDFSHFilePath(fs, tmp.getPath());
			} else {
				list.add(tmp.getPath());
				System.out
						.println("the tmp path = " + tmp.getPath().toString());
			}

			// if (tmp.getPath().toString().contains("/c/") &&
			// fs.isFile(tmp.getPath())) {
			// list.add(tmp.getPath());
			// System.out
			// .println("the tmp path = " + tmp.getPath().toString());
			// } else {
			// getHDFSHFilePath(fs, tmp.getPath());
			// }

			// for(FileStatus tmp1: fs.listStatus(hdfsPath)){
			// if(tmp1.getPath())
			//				
			// }
		}
		return list;
	}

}