Eclipse中创建Hadoop项目的建议
– 没有Hadoop插件
– 下载并解压缩hadoop发布包
– 创建Java项目
– 将解压后hadoop源码包/src目录中core, hdfs, mapred, tool几个目录(其它几个源码根据需
要进行选择)copy到eclipse新建项目的src目录
– 右键点击eclipse项目,选择“ Properties” ,在弹出对话框中左边菜单选择“ Java Build
Path”
• 点击“ Source” 标签。先删除src这个目录,然后依次添加刚才copy过来的目录
• 点击当前对话框“ Libaries” ,点击“ Add External JARs” ,在弹出窗口中添加$HADOOP_HOME下
几个hadoop程序jar包,然后再次添加$HADOOP_HOME /lib、 $HADOOP_HOME /lib/jsp-2.1两个
目录下所有jar包,最后还要添加ANT项目lib目录下ant.jar文件
– 没有Hadoop插件
– 下载并解压缩hadoop发布包
– 创建Java项目
– 将解压后hadoop源码包/src目录中core, hdfs, mapred, tool几个目录(其它几个源码根据需
要进行选择)copy到eclipse新建项目的src目录
– 右键点击eclipse项目,选择“ Properties” ,在弹出对话框中左边菜单选择“ Java Build
Path”
• 点击“ Source” 标签。先删除src这个目录,然后依次添加刚才copy过来的目录
• 点击当前对话框“ Libaries” ,点击“ Add External JARs” ,在弹出窗口中添加$HADOOP_HOME下
几个hadoop程序jar包,然后再次添加$HADOOP_HOME /lib、 $HADOOP_HOME /lib/jsp-2.1两个
目录下所有jar包,最后还要添加ANT项目lib目录下ant.jar文件
– 此时源码项目应该只有关于找不到sun.security包的错误了。这时我们还是在“ Libraries” 这
个标签中,展开jar包列表最低下的 “ JRE System Library” ,双击” Access rules” ,在弹
出窗口中点击“ add按钮”,然后在新对话框中“Resolution” 下拉框选择“Accessible” ,
“Rule Pattern” 填写*/,保存
个标签中,展开jar包列表最低下的 “ JRE System Library” ,双击” Access rules” ,在弹
出窗口中点击“ add按钮”,然后在新对话框中“Resolution” 下拉框选择“Accessible” ,
“Rule Pattern” 填写*/,保存
正则表达式
普通匹配
– 单词,例如cat,匹配cat、 catalog等等
通配符”.”
– 匹配0至多个任意字符,如c.t,可匹配cat、 c#t、 coot、 ct等
方括号
– 仅匹配方括号内的字符,如t[aeiou]n,可匹配tan、 ten、 tin、 ton、 tun
– 方括号内仅能匹配单个字符
或括号(|)
– 可匹配任意一个选项,如t(a|oo)n,可匹配tan、 toon
表示匹配次数的符号
– +: 1 – n次
– *: 0 – n次
– ?: 0或1次
– {n}:恰好n次
– {n, m}:n-m次
普通匹配
– 单词,例如cat,匹配cat、 catalog等等
通配符”.”
– 匹配0至多个任意字符,如c.t,可匹配cat、 c#t、 coot、 ct等
方括号
– 仅匹配方括号内的字符,如t[aeiou]n,可匹配tan、 ten、 tin、 ton、 tun
– 方括号内仅能匹配单个字符
或括号(|)
– 可匹配任意一个选项,如t(a|oo)n,可匹配tan、 toon
表示匹配次数的符号
– +: 1 – n次
– *: 0 – n次
– ?: 0或1次
– {n}:恰好n次
– {n, m}:n-m次
常见符号
– d:数字,等价于[0-9]
– D:非数字,等价于[^0-9]
– s:空白符,等价于[\t\n\x0B\f\r]
– S:非空白符,等价于[^\t\n\x0B\f\r]
– w:单独字符,等价于[a-zA-Z0-9]
– W:非单独字符,等价于[^a-zA-Z0-9]
– ^:开头,如^java,为java开头的字符串
◆比如,在字符串包含验证时
//查找以Java开头,任意结尾的字符串
Pattern pattern = Pattern.compile("^Java.*");
Matcher matcher = pattern.matcher("Java不是人");
boolean b= matcher.matches();
//当条件满足时,将返回true,否则返回false
System.out.println(b);
◆以多条件分割字符串时
Pattern pattern = Pattern.compile("[, |]+");
String[] strs = pattern.split("Java Hello World Java,Hello,,World|Sun");
for (int i=0;i<strs.length;i++) {
System.out.println(strs[i]);
}
◆文字替换(首次出现字符)
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World");
//替换第一个符合正则的数据
System.out.println(matcher.replaceFirst("Java"));
◆文字替换(全部)
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World");
//替换第一个符合正则的数据
System.out.println(matcher.replaceAll("Java"));
◆文字替换(置换字符)
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World ");
StringBuffer sbr = new StringBuffer();
while (matcher.find()) {
matcher.appendReplacement(sbr, "Java");
}
matcher.appendTail(sbr);
System.out.println(sbr.toString());
◆验证是否为邮箱地址
String str="ceponline@yahoo.com.cn";
Pattern pattern = Pattern.compile("[\w\.\-]+@([\w\-]+\.)+[\w\-]+",Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(str);
System.out.println(matcher.matches());
◆去除html标记
Pattern pattern = Pattern.compile("<.+?>", Pattern.DOTALL);
Matcher matcher = pattern.matcher("<a href=\"index.html\">主页</a>");
String string = matcher.replaceAll("");
System.out.println(string);
◆查找html中对应条件字符串
Pattern pattern = Pattern.compile("href=\"(.+?)\"");
Matcher matcher = pattern.matcher("<a href=\"index.html\">主页</a>");
if(matcher.find())
System.out.println(matcher.group(1));
}
◆截取http://地址
//截取url
Pattern pattern = Pattern.compile("(http://|https://){1}[//w//.//-/:]+");
Matcher matcher = pattern.matcher("dsdsds<http://dsds//gfgffdfd>fdf");
StringBuffer buffer = new StringBuffer();
while(matcher.find()){
buffer.append(matcher.group());
buffer.append("/r/n");
System.out.println(buffer.toString());
}
◆替换指定{}中文字
String str = "Java目前的发展史是由{0}年-{1}年";
String[][] object={new String[]{"//{0//}","1995"},new String[]{"//{1//}","2007"}};
System.out.println(replace(str,object));
public static String replace(final String sourceString,Object[] object) {
String temp=sourceString;
for(int i=0;i<object.length;i++){
String[] result=(String[])object[i];
Pattern pattern = Pattern.compile(result[0]);
Matcher matcher = pattern.matcher(temp);
temp=matcher.replaceAll(result[1]);
}
return temp;
}
◆以正则条件查询指定目录下文件
//用于缓存文件列表
private ArrayList files = new ArrayList();
//用于承载文件路径
private String _path;
//用于承载未合并的正则公式
private String _regexp;
class MyFileFilter implements FileFilter {
/**
* 匹配文件名称
*/
public boolean accept(File file) {
try {
Pattern pattern = Pattern.compile(_regexp);
Matcher match = pattern.matcher(file.getName());
return match.matches();
} catch (Exception e) {
return true;
}
}
}
/**
* 解析输入流
* @param inputs
*/
FilesAnalyze (String path,String regexp){
getFileName(path,regexp);
}
/**
* 分析文件名并加入files
* @param input
*/
private void getFileName(String path,String regexp) {
//目录
_path=path;
_regexp=regexp;
File directory = new File(_path);
File[] filesFile = directory.listFiles(new MyFileFilter());
if (filesFile == null) return;
for (int j = 0; j < filesFile.length; j++) {
files.add(filesFile[j]);
}
return;
}
/**
* 显示输出信息
* @param out
*/
public void print (PrintStream out) {
Iterator elements = files.iterator();
while (elements.hasNext()) {
File file=(File) elements.next();
out.println(file.getPath());
}
}
public static void output(String path,String regexp) {
FilesAnalyze fileGroup1 = new FilesAnalyze(path,regexp);
fileGroup1.print(System.out);
}
public static void main (String[] args) {
output("C://","[A-z|.]*");
}
– d:数字,等价于[0-9]
– D:非数字,等价于[^0-9]
– s:空白符,等价于[\t\n\x0B\f\r]
– S:非空白符,等价于[^\t\n\x0B\f\r]
– w:单独字符,等价于[a-zA-Z0-9]
– W:非单独字符,等价于[^a-zA-Z0-9]
– ^:开头,如^java,为java开头的字符串
– $:结尾,java$,以java为结尾的行
package org.myorg;
import java.io.*;
import java.util.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.*;
public class WordCount extends Configured implements Tool {
public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
static enum Counters { INPUT_WORDS }
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
private boolean caseSensitive = true;
private Set<String> patternsToSkip = new HashSet<String>();
private long numRecords = 0;
private String inputFile;
public void configure(JobConf job) {
caseSensitive = job.getBoolean("wordcount.case.sensitive", true);
inputFile = job.get("map.input.file");
if (job.getBoolean("wordcount.skip.patterns", false)) {
Path[] patternsFiles = new Path[0];
try {
patternsFiles = DistributedCache.getLocalCacheFiles(job);
} catch (IOException ioe) {
System.err.println("Caught exception while getting cached files: " + StringUtils.stringifyException(ioe));
}
for (Path patternsFile : patternsFiles) {
parseSkipFile(patternsFile);
}
}
}
private void parseSkipFile(Path patternsFile) {
try {
BufferedReader fis = new BufferedReader(new FileReader(patternsFile.toString()));
String pattern = null;
while ((pattern = fis.readLine()) != null) {
patternsToSkip.add(pattern);
}
} catch (IOException ioe) {
System.err.println("Caught exception while parsing the cached file '" + patternsFile + "' : " + StringUtils.stringifyException(ioe));
}
}
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
String line = (caseSensitive) ? value.toString() : value.toString().toLowerCase();
for (String pattern : patternsToSkip) {
line = line.replaceAll(pattern, "");
}
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
output.collect(word, one);
reporter.incrCounter(Counters.INPUT_WORDS, 1);
}
if ((++numRecords % 100) == 0) {
reporter.setStatus("Finished processing " + numRecords + " records " + "from the input file: " + inputFile);
}
}
}
public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
int sum = 0;
while (values.hasNext()) {
sum += values.next().get();
}
output.collect(key, new IntWritable(sum));
}
}
public int run(String[] args) throws Exception {
JobConf conf = new JobConf(getConf(), WordCount.class);
conf.setJobName("wordcount");
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(Map.class);
conf.setCombinerClass(Reduce.class);
conf.setReducerClass(Reduce.class);
conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
List<String> other_args = new ArrayList<String>();
for (int i=0; i < args.length; ++i) {
if ("-skip".equals(args[i])) {
DistributedCache.addCacheFile(new Path(args[++i]).toUri(), conf);
conf.setBoolean("wordcount.skip.patterns", true);
} else {
other_args.add(args[i]);
}
}
FileInputFormat.setInputPaths(conf, new Path(other_args.get(0)));
FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
JobClient.runJob(conf);
return 0;
}
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new WordCount(), args);
System.exit(res);
}
}
◆比如,在字符串包含验证时
//查找以Java开头,任意结尾的字符串
Pattern pattern = Pattern.compile("^Java.*");
Matcher matcher = pattern.matcher("Java不是人");
boolean b= matcher.matches();
//当条件满足时,将返回true,否则返回false
System.out.println(b);
◆以多条件分割字符串时
Pattern pattern = Pattern.compile("[, |]+");
String[] strs = pattern.split("Java Hello World Java,Hello,,World|Sun");
for (int i=0;i<strs.length;i++) {
System.out.println(strs[i]);
}
◆文字替换(首次出现字符)
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World");
//替换第一个符合正则的数据
System.out.println(matcher.replaceFirst("Java"));
◆文字替换(全部)
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World");
//替换第一个符合正则的数据
System.out.println(matcher.replaceAll("Java"));
◆文字替换(置换字符)
Pattern pattern = Pattern.compile("正则表达式");
Matcher matcher = pattern.matcher("正则表达式 Hello World,正则表达式 Hello World ");
StringBuffer sbr = new StringBuffer();
while (matcher.find()) {
matcher.appendReplacement(sbr, "Java");
}
matcher.appendTail(sbr);
System.out.println(sbr.toString());
◆验证是否为邮箱地址
String str="ceponline@yahoo.com.cn";
Pattern pattern = Pattern.compile("[\w\.\-]+@([\w\-]+\.)+[\w\-]+",Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(str);
System.out.println(matcher.matches());
◆去除html标记
Pattern pattern = Pattern.compile("<.+?>", Pattern.DOTALL);
Matcher matcher = pattern.matcher("<a href=\"index.html\">主页</a>");
String string = matcher.replaceAll("");
System.out.println(string);
◆查找html中对应条件字符串
Pattern pattern = Pattern.compile("href=\"(.+?)\"");
Matcher matcher = pattern.matcher("<a href=\"index.html\">主页</a>");
if(matcher.find())
System.out.println(matcher.group(1));
}
◆截取http://地址
//截取url
Pattern pattern = Pattern.compile("(http://|https://){1}[//w//.//-/:]+");
Matcher matcher = pattern.matcher("dsdsds<http://dsds//gfgffdfd>fdf");
StringBuffer buffer = new StringBuffer();
while(matcher.find()){
buffer.append(matcher.group());
buffer.append("/r/n");
System.out.println(buffer.toString());
}
◆替换指定{}中文字
String str = "Java目前的发展史是由{0}年-{1}年";
String[][] object={new String[]{"//{0//}","1995"},new String[]{"//{1//}","2007"}};
System.out.println(replace(str,object));
public static String replace(final String sourceString,Object[] object) {
String temp=sourceString;
for(int i=0;i<object.length;i++){
String[] result=(String[])object[i];
Pattern pattern = Pattern.compile(result[0]);
Matcher matcher = pattern.matcher(temp);
temp=matcher.replaceAll(result[1]);
}
return temp;
}
◆以正则条件查询指定目录下文件
//用于缓存文件列表
private ArrayList files = new ArrayList();
//用于承载文件路径
private String _path;
//用于承载未合并的正则公式
private String _regexp;
class MyFileFilter implements FileFilter {
/**
* 匹配文件名称
*/
public boolean accept(File file) {
try {
Pattern pattern = Pattern.compile(_regexp);
Matcher match = pattern.matcher(file.getName());
return match.matches();
} catch (Exception e) {
return true;
}
}
}
/**
* 解析输入流
* @param inputs
*/
FilesAnalyze (String path,String regexp){
getFileName(path,regexp);
}
/**
* 分析文件名并加入files
* @param input
*/
private void getFileName(String path,String regexp) {
//目录
_path=path;
_regexp=regexp;
File directory = new File(_path);
File[] filesFile = directory.listFiles(new MyFileFilter());
if (filesFile == null) return;
for (int j = 0; j < filesFile.length; j++) {
files.add(filesFile[j]);
}
return;
}
/**
* 显示输出信息
* @param out
*/
public void print (PrintStream out) {
Iterator elements = files.iterator();
while (elements.hasNext()) {
File file=(File) elements.next();
out.println(file.getPath());
}
}
public static void output(String path,String regexp) {
FilesAnalyze fileGroup1 = new FilesAnalyze(path,regexp);
fileGroup1.print(System.out);
}
public static void main (String[] args) {
output("C://","[A-z|.]*");
}