hadoop hdfs的一些操作

[java]  view plain copy

1.  package com.picc.test;  
2.   
3. import java.io.BufferedReader;  
4. import java.io.IOException;  
5. import java.io.InputStream;  
6. import java.io.InputStreamReader;  
7. import java.net.URI;  
8. import java.net.URL;  
9.   
10. import org.apache.hadoop.conf.Configuration;  
11. import org.apache.hadoop.fs.FileSystem;  
12. import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;  
13. import org.apache.hadoop.fs.Path;  
14.   
15. public class UrlCat {  
16.    static{  
17.        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());  
18.    }  
19.    public static void main(String[] args) {  
20.        InputStream in =null;  
21.         try {//new FileInputStream("/home/liucheng/file/student.txt");  
22.             in=  new URL("hdfs://localhost:9000/user/liucheng/input/student.txt").openStream();  
23.         }  catch (IOException e) {  
24.             e.printStackTrace();  
25.         }  
26.         BufferedReader read = new BufferedReader(new InputStreamReader(in));  
27.         String line=null;  
28.         try {  
29.             while((line=read.readLine())!=null){  
30.                 System.out.println("result:"+line.trim());  
31.             }  
32.         } catch (IOException e) {  
33.             e.printStackTrace();  
34.         }  
35.         try {  
36.             read.close();  
37.         } catch (IOException e) {  
38.             e.printStackTrace();  
39.         }  
40.     run("hdfs://localhost:9000/user/liucheng/input/student.txt");  
41.    }  
42.    public static void run(String hdfs){  
43.        Configuration conf = new Configuration();  
44.        InputStream in =null;  
45.         try {  
46.           FileSystem fs = FileSystem.get(URI.create(hdfs), conf);  
47.           in = fs.open(new Path(hdfs));  
48.           BufferedReader read = new BufferedReader(new InputStreamReader(in));  
49.             String line=null;  
50.             try {  
51.                 while((line=read.readLine())!=null){  
52.                     System.out.println("result:"+line.trim());  
53.                 }  
54.             } catch (IOException e) {  
55.                 e.printStackTrace();  
56.             }  
57.         } catch (IOException e) {  
58.             e.printStackTrace();  
59.         }finally{  
60.             try {  
61.                 in.close();  
62.             } catch (IOException e) {  
63.                 e.printStackTrace();  
64.             }  
65.         }  
66.    }  
67. }  

示例是读取hdfs 的Input 文件夹下的student.txt 文件。示例。

java 操作 hdfs 

[java]  view plain copy

1.  package com.picc.test;  
2.   
3. import java.io.BufferedReader;  
4. import java.io.IOException;  
5. import java.io.InputStream;  
6. import java.io.InputStreamReader;  
7. import java.net.URI;  
8. import java.net.URL;  
9. import java.util.Arrays;  
10.   
11. import org.apache.hadoop.conf.Configuration;  
12. import org.apache.hadoop.fs.FSDataInputStream;  
13. import org.apache.hadoop.fs.FileStatus;  
14. import org.apache.hadoop.fs.FileSystem;  
15. import org.apache.hadoop.fs.FileUtil;  
16. import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;  
17. import org.apache.hadoop.fs.Path;  
18.   
19. public class UrlCat {  
20.    static{  
21.        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());  
22.    }  
23.      
24.    public static void main(String[] args) {  
25.       
26.      run("hdfs://localhost:9000/user/liucheng/input/file01.txt");  
27.     //doubleCat("hdfs://localhost:9000/user/liucheng/input/student.txt");  
28.     //   statusTest("hdfs://localhost:9000/user/liucheng/input");  
29.    }  
30.    public static void run(String hdfs){  
31.        Configuration conf = new Configuration();  
32.        InputStream in =null;  
33.         try {  
34.           FileSystem fs = FileSystem.get(URI.create(hdfs), conf);  
35.           in = fs.open(new Path(hdfs));  
36.           BufferedReader read = new BufferedReader(new InputStreamReader(in));  
37.             String line=null;  
38.             try {  
39.                 while((line=read.readLine())!=null){  
40.                     System.out.println("result:"+line.trim());  
41.                 }  
42.             } catch (IOException e) {  
43.                 e.printStackTrace();  
44.             }  
45.         } catch (IOException e) {  
46.             e.printStackTrace();  
47.         }finally{  
48.             try {  
49.                 in.close();  
50.             } catch (IOException e) {  
51.                 e.printStackTrace();  
52.             }  
53.         }  
54.    }  
55.    //FSDataInputStream extents DataInputStream extents InputStream  
56.    /** 
57.     *  其实doubleCat 和 getInputStreamtoText 方法需要放到一起 
58.     *  在有异常时，把位置保存到一个properties文件中 
59.     * @param hdfs 
60.     */  
61.    public static void doubleCat(String hdfs){  
62.        Configuration conf = new Configuration();  
63.        FSDataInputStream fsdis =null;  
64.        try {  
65.         FileSystem fs = FileSystem.get(URI.create(hdfs),conf);  
66.         fsdis= fs.open(new Path(hdfs));  
67.         long rs = fsdis.skip(1000);//跳过1000个位置  
68.         fsdis.seek(rs);//查找rs的位子  
69.         String result = getInputStreamtoText(fsdis);  
70.         System.out.println("result:"+result);  
71.     } catch (IOException e) {  
72.         e.printStackTrace();  
73.     }finally{  
74.         if(fsdis!=null){  
75.             try {  
76.                 fsdis.close();  
77.             } catch (IOException e) {  
78.                 e.printStackTrace();  
79.             }  
80.         }  
81.     }  
82.    }  
83.    /** 
84.     * inputStream to Text 
85.     * @param in 
86.     * @return 
87.     */  
88.    public static String getInputStreamtoText(InputStream in ){  
89.        BufferedReader read = new BufferedReader(new InputStreamReader(in));  
90.         String line=null;  
91.         StringBuffer result=new StringBuffer();  
92.         try {  
93.             while((line=read.readLine())!=null){  
94.             //    System.out.println("result:"+line.trim());  
95.                 String lineSeparator =System.getProperty("line.separator");  
96.                 result.append(line).append(lineSeparator);  
97.                   
98.             }  
99.         } catch (IOException e) {  
100.             e.printStackTrace();  
101.         }finally{  
102.             try {  
103.                 in.close();  
104.             } catch (IOException e) {  
105.                 e.printStackTrace();  
106.             }  
107.         }  
108.         return result.toString();  
109.    }  
110.    public static void statusTest(String url){  
111.        Configuration conf = new Configuration();  
112.      try {  
113.         FileSystem fs =   FileSystem.get(URI.create(url), conf);  
114.         Path [] paths = new Path[url.length()];  
115.         for(int i=0;i<paths.length;i++){  
116. //            paths[i]=new Path[String.url];  
117.             FileStatus [] filestatus = fs.listStatus(paths[i]);  
118.             Path [] listedPaths = FileUtil.stat2Paths(filestatus);  
119.             for(Path p:listedPaths){  
120.                 System.out.println(p);  
121.             }  
122.         }  
123.          // /tmp/hadoop-liucheng/dfs/name/current/edits  
124.     //    bin/mysqld  
125.     } catch (IOException e) {  
126.         e.printStackTrace();  
127.     }  
128.    }  
129.      
130. }