Day7-电商实战项目使用Hadoop实现（二）

具体代码

1.GetPageld.java

import org.apache.commons.lang3.StringUtils;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class GetPageId {

    public static String getPageId(String url) {
        String pageId = "";
        if (StringUtils.isBlank(url)) {
            return pageId;
        }
        Pattern pat = Pattern.compile("topicId=[0-9]+");
        Matcher matcher = pat.matcher(url);

        if (matcher.find()) {
            pageId = matcher.group().split("topicId=")[1];
        }

        return pageId;
    }

    public static void main(String[] args) {
        System.out.println(getPageId("http://www.yihaodian.com/cms/view.do?topicId=14572"));
        System.out.println(getPageId("http://www.yihaodian.com/cms/view.do?topicId=22372&merchant=1"));

    }
}

2.IPParser.java

public class IPParser extends IPSeeker {
    // 地址 仅仅只是在ecplise环境中使用，部署在服务器上，需要先将trackinfo_20130721.txt放在集群的各个节点某个有读取权限目录，
    // 然后在这里指定全路径
    private static final String ipFilePath = "/root/data/qqwry.dat";
    // 部署在服务器上
    //private static final String ipFilePath = "/opt/datas/qqwry.dat";
    private static IPParser obj = new IPParser(ipFilePath);



    protected IPParser(String ipFilePath) {
        super(ipFilePath);
    }

    public static IPParser getInstance() {
        return obj;
    }

    /**
     * 解析ip地址
     *
     * @param ip
     * @return
     */
    public RegionInfo analyseIp(String ip) {
        if (ip == null || "".equals(ip.trim())) {
            return null;
        }

        RegionInfo info = new RegionInfo();
        try {
            String country = super.getCountry(ip);
            if ("局域网".equals(country) || country == null || country.isEmpty() || country.trim().startsWith("CZ88")) {
                // 设置默认值
                info.setCountry("中国");
                info.setProvince("上海市");
            } else {
                int length = country.length();
                int index = country.indexOf('省');
                if (index > 0) { // 表示是国内的某个省
                    info.setCountry("中国");
                    info.setProvince(country.substring(0, Math.min(index + 1, length)));
                    int index2 = country.indexOf('市', index);
                    if (index2 > 0) {
                        // 设置市
                        info.setCity(country.substring(index + 1, Math.min(index2 + 1, length)));
                    }
                } else {
                    String flag = country.substring(0, 2);
                    switch (flag) {
                        case "内蒙":
                            info.setCountry("中国");
                            info.setProvince("内蒙古自治区");
                            country = country.substring(3);
                            if (country != null && !country.isEmpty()) {
                                index = country.indexOf('市');
                                if (index > 0) {
                                    // 设置市
                                    info.setCity(country.substring(0, Math.min(index + 1, length)));
                                }
                                // TODO:针对其他旗或者盟没有进行处理
                            }
                            break;
                        case "广西":
                        case "西藏":
                        case "宁夏":
                        case "新疆":
                            info.setCountry("中国");
                            info.setProvince(flag);
                            country = country.substring(2);
                            if (country != null && !country.isEmpty()) {
                                index = country.indexOf('市');
                                if (index > 0) {
                                    // 设置市
                                    info.setCity(country.substring(0, Math.min(index + 1, length)));
                                }
                            }
                            break;
                        case "上海":
                        case "北京":
                        case "重庆":
                        case "天津":
                            info.setCountry("中国");
                            info.setProvince(flag + "市");
                            country = country.substring(3);
                            if (country != null && !country.isEmpty()) {
                                index = country.indexOf('区');
                                if (index > 0) {
                                    // 设置市
                                    char ch = country.charAt(index - 1);
                                    if (ch != '小' || ch != '校') {
                                        info.setCity(country.substring(0, Math.min(index + 1, length)));
                                    }
                                }

                                if ("unknown".equals(info.getCity())) {
                                    // 现在city还没有设置，考虑县
                                    index = country.indexOf('县');
                                    if (index > 0) {
                                        // 设置市
                                        info.setCity(country.substring(0, Math.min(index + 1, length)));
                                    }
                                }
                            }
                            break;
                        case "香港":
                        case "澳门":
                            info.setCountry("中国");
                            info.setProvince(flag + "特别行政区");
                            break;
                        default:
                            info.setCountry(country); // 针对其他国外的ip
                    }
                }
            }
        } catch (Exception e) {
            // nothing
        }
        return info;
    }

    /**
     * ip地址对应的info类
     *
     */
    public static class RegionInfo {
        private String country ;
        private String province ;
        private String city ;

        public String getCountry() {
            return country;
        }

        public void setCountry(String country) {
            this.country = country;
        }

        public String getProvince() {
            return province;
        }

        public void setProvince(String province) {
            this.province = province;
        }

        public String getCity() {
            return city;
        }

        public void setCity(String city) {
            this.city = city;
        }

        @Override
        public String toString() {
            return "RegionInfo [country=" + country + ", province=" + province + ", city=" + city + "]";
        }
    }
}

3.IPSeeker.java

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.List;

public class IPSeeker {
    public static final String ERROR_RESULT = "错误的IP数据库文件";
    // 一些固定常量，比如记录长度等等
    private static final int IP_RECORD_LENGTH = 7;
    private static final byte AREA_FOLLOWED = 0x01;
    private static final byte NO_AREA = 0x2;

    // 用来做为cache，查询一个ip时首先查看cache，以减少不必要的重复查找
    private Hashtable ipCache;
    // 随机文件访问类
    private RandomAccessFile ipFile;
    // 内存映射文件
    private MappedByteBuffer mbb;
    // 单一模式实例
    private static IPSeeker instance = null;
    // 起始地区的开始和结束的绝对偏移
    private long ipBegin, ipEnd;
    // 为提高效率而采用的临时变量
    private IPLocation loc;
    private byte[] buf;
    private byte[] b4;
    private byte[] b3;

    /** */
    /**
     * 私有构造函数
     */
    protected IPSeeker(String ipFilePath) {
        ipCache = new Hashtable();
        loc = new IPLocation();
        buf = new byte[100];
        b4 = new byte[4];
        b3 = new byte[3];
        try {
            ipFile = new RandomAccessFile(ipFilePath, "r");
        } catch (FileNotFoundException e) {
            System.out.println("IP地址信息文件没有找到，IP显示功能将无法使用");
            ipFile = null;

        }
        // 如果打开文件成功，读取文件头信息
        if (ipFile != null) {
            try {
                ipBegin = readLong4(0);
                ipEnd = readLong4(4);
                if (ipBegin == -1 || ipEnd == -1) {
                    ipFile.close();
                    ipFile = null;
                }
            } catch (IOException e) {
                System.out.println("IP地址信息文件格式有错误，IP显示功能将无法使用");
                ipFile = null;
            }
        }
    }

    /** */
    /**
     * @return 单一实例
     */
    public static IPSeeker getInstance(String ipFilePath) {
        if (instance == null) {
            instance = new IPSeeker(ipFilePath);
        }
        return instance;
    }

    /** */
    /**
     * 给定一个地点的不完全名字，得到一系列包含s子串的IP范围记录
     *
     * @param s
     *            地点子串
     * @return 包含IPEntry类型的List
     */
    public List getIPEntriesDebug(String s) {
        List ret = new ArrayList();
        long endOffset = ipEnd + 4;
        for (long offset = ipBegin + 4; offset <= endOffset; offset += IP_RECORD_LENGTH) {
            // 读取结束IP偏移
            long temp = readLong3(offset);
            // 如果temp不等于-1，读取IP的地点信息
            if (temp != -1) {
                IPLocation loc = getIPLocation(temp);
                // 判断是否这个地点里面包含了s子串，如果包含了，添加这个记录到List中，如果没有，继续
                if (loc.country.indexOf(s) != -1 || loc.area.indexOf(s) != -1) {
                    IPEntry entry = new IPEntry();
                    entry.country = loc.country;
                    entry.area = loc.area;
                    // 得到起始IP
                    readIP(offset - 4, b4);
                    entry.beginIp = IPSeekerUtils.getIpStringFromBytes(b4);
                    // 得到结束IP
                    readIP(temp, b4);
                    entry.endIp = IPSeekerUtils.getIpStringFromBytes(b4);
                    // 添加该记录
                    ret.add(entry);
                }
            }
        }
        return ret;
    }

    /** */
    /**
     * 给定一个地点的不完全名字，得到一系列包含s子串的IP范围记录
     *
     * @param s
     *            地点子串
     * @return 包含IPEntry类型的List
     */
    public List getIPEntries(String s) {
        List ret = new ArrayList();
        try {
            // 映射IP信息文件到内存中
            if (mbb == null) {
                FileChannel fc = ipFile.getChannel();
                mbb = fc.map(FileChannel.MapMode.READ_ONLY, 0, ipFile.length());
                mbb.order(ByteOrder.LITTLE_ENDIAN);
            }

            int endOffset = (int) ipEnd;
            for (int offset = (int) ipBegin + 4; offset <= endOffset; offset += IP_RECORD_LENGTH) {
                int temp = readInt3(offset);
                if (temp != -1) {
                    IPLocation loc = getIPLocation(temp);
                    // 判断是否这个地点里面包含了s子串，如果包含了，添加这个记录到List中，如果没有，继续
                    if (loc.country.indexOf(s) != -1 || loc.area.indexOf(s) != -1) {
                        IPEntry entry = new IPEntry();
                        entry.country = loc.country;
                        entry.area = loc.area;
                        // 得到起始IP
                        readIP(offset - 4, b4);
                        entry.beginIp = IPSeekerUtils.getIpStringFromBytes(b4);
                        // 得到结束IP
                        readIP(temp, b4);
                        entry.endIp = IPSeekerUtils.getIpStringFromBytes(b4);
                        // 添加该记录
                        ret.add(entry);
                    }
                }
            }
        } catch (IOException e) {
            System.out.println(e.getMessage());
        }
        return ret;
    }

    /** */
    /**
     * 从内存映射文件的offset位置开始的3个字节读取一个int
     *
     * @param offset
     * @return
     */
    private int readInt3(int offset) {
        mbb.position(offset);
        return mbb.getInt() & 0x00FFFFFF;
    }

    /** */
    /**
     * 从内存映射文件的当前位置开始的3个字节读取一个int
     *
     * @return
     */
    private int readInt3() {
        return mbb.getInt() & 0x00FFFFFF;
    }

    /** */
    /**
     * 根据IP得到国家名
     *
     * @param ip
     *            ip的字节数组形式
     * @return 国家名字符串
     */
    public String getCountry(byte[] ip) {
        // 检查ip地址文件是否正常
        if (ipFile == null)
            return ERROR_RESULT;
        // 保存ip，转换ip字节数组为字符串形式
        String ipStr = IPSeekerUtils.getIpStringFromBytes(ip);
        // 先检查cache中是否已经包含有这个ip的结果，没有再搜索文件
        if (ipCache.containsKey(ipStr)) {
            IPLocation loc = (IPLocation) ipCache.get(ipStr);
            return loc.country;
        } else {
            IPLocation loc = getIPLocation(ip);
            ipCache.put(ipStr, loc.getCopy());
            return loc.country;
        }
    }

    /** */
    /**
     * 根据IP得到国家名
     *
     * @param ip
     *            IP的字符串形式
     * @return 国家名字符串
     */
    public String getCountry(String ip) {
        return getCountry(IPSeekerUtils.getIpByteArrayFromString(ip));
    }

    /** */
    /**
     * 根据IP得到地区名
     *
     * @param ip
     *            ip的字节数组形式
     * @return 地区名字符串
     */
    public String getArea(byte[] ip) {
        // 检查ip地址文件是否正常
        if (ipFile == null)
            return ERROR_RESULT;
        // 保存ip，转换ip字节数组为字符串形式
        String ipStr = IPSeekerUtils.getIpStringFromBytes(ip);
        // 先检查cache中是否已经包含有这个ip的结果，没有再搜索文件
        if (ipCache.containsKey(ipStr)) {
            IPLocation loc = (IPLocation) ipCache.get(ipStr);
            return loc.area;
        } else {
            IPLocation loc = getIPLocation(ip);
            ipCache.put(ipStr, loc.getCopy());
            return loc.area;
        }
    }

    /**
     * 根据IP得到地区名
     *
     * @param ip
     *            IP的字符串形式
     * @return 地区名字符串
     */
    public String getArea(String ip) {
        return getArea(IPSeekerUtils.getIpByteArrayFromString(ip));
    }

    /** */
    /**
     * 根据ip搜索ip信息文件，得到IPLocation结构，所搜索的ip参数从类成员ip中得到
     *
     * @param ip
     *            要查询的IP
     * @return IPLocation结构
     */
    public IPLocation getIPLocation(byte[] ip) {
        IPLocation info = null;
        long offset = locateIP(ip);
        if (offset != -1)
            info = getIPLocation(offset);
        if (info == null) {
            info = new IPLocation();
            info.country = "未知国家";
            info.area = "未知地区";
        }
        return info;
    }

    /**
     * 从offset位置读取4个字节为一个long，因为java为big-endian格式，所以没办法 用了这么一个函数来做转换
     *
     * @param offset
     * @return 读取的long值，返回-1表示读取文件失败
     */
    private long readLong4(long offset) {
        long ret = 0;
        try {
            ipFile.seek(offset);
            ret |= (ipFile.readByte() & 0xFF);
            ret |= ((ipFile.readByte() << 8) & 0xFF00);
            ret |= ((ipFile.readByte() << 16) & 0xFF0000);
            ret |= ((ipFile.readByte() << 24) & 0xFF000000);
            return ret;
        } catch (IOException e) {
            return -1;
        }
    }

    /**
     * 从offset位置读取3个字节为一个long，因为java为big-endian格式，所以没办法 用了这么一个函数来做转换
     *
     * @param offset
     * @return 读取的long值，返回-1表示读取文件失败
     */
    private long readLong3(long offset) {
        long ret = 0;
        try {
            ipFile.seek(offset);
            ipFile.readFully(b3);
            ret |= (b3[0] & 0xFF);
            ret |= ((b3[1] << 8) & 0xFF00);
            ret |= ((b3[2] << 16) & 0xFF0000);
            return ret;
        } catch (IOException e) {
            return -1;
        }
    }

    /**
     * 从当前位置读取3个字节转换成long
     *
     * @return
     */
    private long readLong3() {
        long ret = 0;
        try {
            ipFile.readFully(b3);
            ret |= (b3[0] & 0xFF);
            ret |= ((b3[1] << 8) & 0xFF00);
            ret |= ((b3[2] << 16) & 0xFF0000);
            return ret;
        } catch (IOException e) {
            return -1;
        }
    }

    /**
     * 从offset位置读取四个字节的ip地址放入ip数组中，读取后的ip为big-endian格式，但是
     * 文件中是little-endian形式，将会进行转换
     *
     * @param offset
     * @param ip
     */
    private void readIP(long offset, byte[] ip) {
        try {
            ipFile.seek(offset);
            ipFile.readFully(ip);
            byte temp = ip[0];
            ip[0] = ip[3];
            ip[3] = temp;
            temp = ip[1];
            ip[1] = ip[2];
            ip[2] = temp;
        } catch (IOException e) {
            System.out.println(e.getMessage());
        }
    }

    /**
     * 从offset位置读取四个字节的ip地址放入ip数组中，读取后的ip为big-endian格式，但是
     * 文件中是little-endian形式，将会进行转换
     *
     * @param offset
     * @param ip
     */
    private void readIP(int offset, byte[] ip) {
        mbb.position(offset);
        mbb.get(ip);
        byte temp = ip[0];
        ip[0] = ip[3];
        ip[3] = temp;
        temp = ip[1];
        ip[1] = ip[2];
        ip[2] = temp;
    }

    /**
     * 把类成员ip和beginIp比较，注意这个beginIp是big-endian的
     *
     * @param ip
     *            要查询的IP
     * @param beginIp
     *            和被查询IP相比较的IP
     * @return 相等返回0，ip大于beginIp则返回1，小于返回-1。
     */
    private int compareIP(byte[] ip, byte[] beginIp) {
        for (int i = 0; i < 4; i++) {
            int r = compareByte(ip[i], beginIp[i]);
            if (r != 0)
                return r;
        }
        return 0;
    }

    /**
     * 把两个byte当作无符号数进行比较
     *
     * @param b1
     * @param b2
     * @return 若b1大于b2则返回1，相等返回0，小于返回-1
     */
    private int compareByte(byte b1, byte b2) {
        if ((b1 & 0xFF) > (b2 & 0xFF)) // 比较是否大于
            return 1;
        else if ((b1 ^ b2) == 0)// 判断是否相等
            return 0;
        else
            return -1;
    }

    /**
     * 这个方法将根据ip的内容，定位到包含这个ip国家地区的记录处，返回一个绝对偏移 方法使用二分法查找。
     *
     * @param ip
     *            要查询的IP
     * @return 如果找到了，返回结束IP的偏移，如果没有找到，返回-1
     */
    private long locateIP(byte[] ip) {
        long m = 0;
        int r;
        // 比较第一个ip项
        readIP(ipBegin, b4);
        r = compareIP(ip, b4);
        if (r == 0)
            return ipBegin;
        else if (r < 0)
            return -1;
        // 开始二分搜索
        for (long i = ipBegin, j = ipEnd; i < j;) {
            m = getMiddleOffset(i, j);
            readIP(m, b4);
            r = compareIP(ip, b4);
            // log.debug(Utils.getIpStringFromBytes(b));
            if (r > 0)
                i = m;
            else if (r < 0) {
                if (m == j) {
                    j -= IP_RECORD_LENGTH;
                    m = j;
                } else
                    j = m;
            } else
                return readLong3(m + 4);
        }
        // 如果循环结束了，那么i和j必定是相等的，这个记录为最可能的记录，但是并非
        // 肯定就是，还要检查一下，如果是，就返回结束地址区的绝对偏移
        m = readLong3(m + 4);
        readIP(m, b4);
        r = compareIP(ip, b4);
        if (r <= 0)
            return m;
        else
            return -1;
    }

    /**
     * 得到begin偏移和end偏移中间位置记录的偏移
     *
     * @param begin
     * @param end
     * @return
     */
    private long getMiddleOffset(long begin, long end) {
        long records = (end - begin) / IP_RECORD_LENGTH;
        records >>= 1;
        if (records == 0)
            records = 1;
        return begin + records * IP_RECORD_LENGTH;
    }

    /**
     * 给定一个ip国家地区记录的偏移，返回一个IPLocation结构
     *
     * @param offset
     * @return
     */
    private IPLocation getIPLocation(long offset) {
        try {
            // 跳过4字节ip
            ipFile.seek(offset + 4);
            // 读取第一个字节判断是否标志字节
            byte b = ipFile.readByte();
            if (b == AREA_FOLLOWED) {
                // 读取国家偏移
                long countryOffset = readLong3();
                // 跳转至偏移处
                ipFile.seek(countryOffset);
                // 再检查一次标志字节，因为这个时候这个地方仍然可能是个重定向
                b = ipFile.readByte();
                if (b == NO_AREA) {
                    loc.country = readString(readLong3());
                    ipFile.seek(countryOffset + 4);
                } else
                    loc.country = readString(countryOffset);
                // 读取地区标志
                loc.area = readArea(ipFile.getFilePointer());
            } else if (b == NO_AREA) {
                loc.country = readString(readLong3());
                loc.area = readArea(offset + 8);
            } else {
                loc.country = readString(ipFile.getFilePointer() - 1);
                loc.area = readArea(ipFile.getFilePointer());
            }
            return loc;
        } catch (IOException e) {
            return null;
        }
    }

    /**
     * @param offset
     * @return
     */
    private IPLocation getIPLocation(int offset) {
        // 跳过4字节ip
        mbb.position(offset + 4);
        // 读取第一个字节判断是否标志字节
        byte b = mbb.get();
        if (b == AREA_FOLLOWED) {
            // 读取国家偏移
            int countryOffset = readInt3();
            // 跳转至偏移处
            mbb.position(countryOffset);
            // 再检查一次标志字节，因为这个时候这个地方仍然可能是个重定向
            b = mbb.get();
            if (b == NO_AREA) {
                loc.country = readString(readInt3());
                mbb.position(countryOffset + 4);
            } else
                loc.country = readString(countryOffset);
            // 读取地区标志
            loc.area = readArea(mbb.position());
        } else if (b == NO_AREA) {
            loc.country = readString(readInt3());
            loc.area = readArea(offset + 8);
        } else {
            loc.country = readString(mbb.position() - 1);
            loc.area = readArea(mbb.position());
        }
        return loc;
    }

    /**
     * 从offset偏移开始解析后面的字节，读出一个地区名
     *
     * @param offset
     * @return 地区名字符串
     * @throws IOException
     */
    private String readArea(long offset) throws IOException {
        ipFile.seek(offset);
        byte b = ipFile.readByte();
        if (b == 0x01 || b == 0x02) {
            long areaOffset = readLong3(offset + 1);
            if (areaOffset == 0)
                return "未知地区";
            else
                return readString(areaOffset);
        } else
            return readString(offset);
    }

    /**
     * @param offset
     * @return
     */
    private String readArea(int offset) {
        mbb.position(offset);
        byte b = mbb.get();
        if (b == 0x01 || b == 0x02) {
            int areaOffset = readInt3();
            if (areaOffset == 0)
                return "未知地区";
            else
                return readString(areaOffset);
        } else
            return readString(offset);
    }

    /**
     * 从offset偏移处读取一个以0结束的字符串
     *
     * @param offset
     * @return 读取的字符串，出错返回空字符串
     */
    private String readString(long offset) {
        try {
            ipFile.seek(offset);
            int i;
            for (i = 0, buf[i] = ipFile.readByte(); buf[i] != 0; buf[++i] = ipFile.readByte())
                ;
            if (i != 0)
                return IPSeekerUtils.getString(buf, 0, i, "GBK");
        } catch (IOException e) {
            System.out.println(e.getMessage());
        }
        return "";
    }

    /**
     * 从内存映射文件的offset位置得到一个0结尾字符串
     *
     * @param offset
     * @return
     */
    private String readString(int offset) {
        try {
            mbb.position(offset);
            int i;
            for (i = 0, buf[i] = mbb.get(); buf[i] != 0; buf[++i] = mbb.get())
                ;
            if (i != 0)
                return IPSeekerUtils.getString(buf, 0, i, "GBK");
        } catch (IllegalArgumentException e) {
            System.out.println(e.getMessage());
        }
        return "";
    }

    public String getAddress(String ip) {
        String country = getCountry(ip).equals(" CZ88.NET") ? "" : getCountry(ip);
        String area = getArea(ip).equals(" CZ88.NET") ? "" : getArea(ip);
        String address = country + " " + area;
        return address.trim();
    }

    /**
     * * 用来封装ip相关信息，目前只有两个字段，ip所在的国家和地区
     *
     *
     * @author swallow
     */
    public class IPLocation {
        public String country;
        public String area;

        public IPLocation() {
            country = area = "";
        }

        public IPLocation getCopy() {
            IPLocation ret = new IPLocation();
            ret.country = country;
            ret.area = area;
            return ret;
        }
    }

    /**
     * 一条IP范围记录，不仅包括国家和区域，也包括起始IP和结束IP *
     *
     *
     * @author gerry liu
     */
    public class IPEntry {
        public String beginIp;
        public String endIp;
        public String country;
        public String area;

        public IPEntry() {
            beginIp = endIp = country = area = "";
        }

        public String toString() {
            return this.area + " " + this.country + "IP  Χ:" + this.beginIp + "-" + this.endIp;
        }
    }

    /**
     * 操作工具类
     *
     * @author gerryliu
     *
     */
    public static class IPSeekerUtils {
        /**
         * 从ip的字符串形式得到字节数组形式
         *
         * @param ip
         *            字符串形式的ip
         * @return 字节数组形式的ip
         */
        public static byte[] getIpByteArrayFromString(String ip) {
            byte[] ret = new byte[4];
            java.util.StringTokenizer st = new java.util.StringTokenizer(ip, ".");
            try {
                ret[0] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
                ret[1] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
                ret[2] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
                ret[3] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
            } catch (Exception e) {
                System.out.println(e.getMessage());
            }
            return ret;
        }

        /**
         * 对原始字符串进行编码转换，如果失败，返回原始的字符串
         *
         * @param s
         *            原始字符串
         * @param srcEncoding
         *            源编码方式
         * @param destEncoding
         *            目标编码方式
         * @return 转换编码后的字符串，失败返回原始字符串
         */
        public static String getString(String s, String srcEncoding, String destEncoding) {
            try {
                return new String(s.getBytes(srcEncoding), destEncoding);
            } catch (UnsupportedEncodingException e) {
                return s;
            }
        }

        /**
         * 根据某种编码方式将字节数组转换成字符串
         *
         * @param b
         *            字节数组
         * @param encoding
         *            编码方式
         * @return 如果encoding不支持，返回一个缺省编码的字符串
         */
        public static String getString(byte[] b, String encoding) {
            try {
                return new String(b, encoding);
            } catch (UnsupportedEncodingException e) {
                return new String(b);
            }
        }

        /**
         * 根据某种编码方式将字节数组转换成字符串
         *
         * @param b
         *            字节数组
         * @param offset
         *            要转换的起始位置
         * @param len
         *            要转换的长度
         * @param encoding
         *            编码方式
         * @return 如果encoding不支持，返回一个缺省编码的字符串
         */
        public static String getString(byte[] b, int offset, int len, String encoding) {
            try {
                return new String(b, offset, len, encoding);
            } catch (UnsupportedEncodingException e) {
                return new String(b, offset, len);
            }
        }

        /**
         * @param ip
         *            ip的字节数组形式
         * @return 字符串形式的ip
         */
        public static String getIpStringFromBytes(byte[] ip) {
            StringBuffer sb = new StringBuffer();
            sb.append(ip[0] & 0xFF);
            sb.append('.');
            sb.append(ip[1] & 0xFF);
            sb.append('.');
            sb.append(ip[2] & 0xFF);
            sb.append('.');
            sb.append(ip[3] & 0xFF);
            return sb.toString();
        }
    }

    /**
     * 获取全部ip地址集合列表
     *
     * @return
     */
    public List<String> getAllIp() {
        List<String> list = new ArrayList<String>();
        byte[] buf = new byte[4];
        for (long i = ipBegin; i < ipEnd; i += IP_RECORD_LENGTH) {
            try {
                this.readIP(this.readLong3(i + 4), buf); // 读取ip，最终ip放到buf中
                String ip = IPSeekerUtils.getIpStringFromBytes(buf);
                list.add(ip);
            } catch (Exception e) {
                // nothing
            }
        }
        return list;
    }
}

4.LogParser.java

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;

public class LogParser {

    private Logger logger = LoggerFactory.getLogger(LogParser.class);



    public Map<String, String> parse2(String log)  {

        Map<String, String> logInfo = new HashMap<String,String>();
        IPParser ipParse = IPParser.getInstance();
        if(StringUtils.isNotBlank(log)) {
            String[] splits = log.split("\t");

            String ip = splits[0];
            String url = splits[1];
            String sessionId = splits[2];
            String time = splits[3];
            String country = splits[4];
            String province = splits[5];
            String city = splits[6];

            logInfo.put("ip",ip);
            logInfo.put("url",url);
            logInfo.put("sessionId",sessionId);
            logInfo.put("time",time);
            logInfo.put("country",country);
            logInfo.put("province",province);
            logInfo.put("city",city);

        } else{
            logger.error("日志记录的格式不正确：" + log);
        }

        return logInfo;
    }

    public Map<String, String> parse(String log)  {

        Map<String, String> logInfo = new HashMap<String,String>();
        IPParser ipParse = IPParser.getInstance();
        if(StringUtils.isNotBlank(log)) {
            String[] splits = log.split("\001");

            String ip = splits[13];
            String url = splits[1];
            String sessionId = splits[10];
            String time = splits[17];

            logInfo.put("ip",ip);
            logInfo.put("url",url);
            logInfo.put("sessionId",sessionId);
            logInfo.put("time",time);


            IPParser.RegionInfo regionInfo = ipParse.analyseIp(ip);

            logInfo.put("country",regionInfo.getCountry());
            logInfo.put("province",regionInfo.getProvince());
            logInfo.put("city",regionInfo.getCity());

        } else{
            logger.error("日志记录的格式不正确：" + log);
        }

        return logInfo;
    }

}

4.在写主类之前，按需求分成三个主类
（1）.MainApp1.java

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.Map;

public class MainApp1 {

    // Driver端的代码：八股文
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();

        // 如果输出目录已经存在，则先删除
        FileSystem fileSystem = FileSystem.get(configuration);
        Path outputPath = new Path("/output/xuqiu1");
        if(fileSystem.exists(outputPath)) {
            fileSystem.delete(outputPath,true);
        }

        Job job = Job.getInstance(configuration);
        job.setJarByClass(MainApp1.class);

        job.setMapperClass(MyMapper1.class);
        job.setReducerClass(MyReducer1.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        FileInputFormat.setInputPaths(job, new Path("/output/xuqiu3/part-r-00000"));
        FileOutputFormat.setOutputPath(job, new Path("/output/xuqiu1"));

        job.waitForCompletion(true);
    }




    //需求1
    static class MyMapper1 extends Mapper<LongWritable, Text, Text, LongWritable> {

        public MyMapper1() {
            // 添加一个无参构造函数
        }

        private LongWritable ONE = new LongWritable(1);
        private Text word = new Text("TotalLines");

        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            context.write(word, ONE);
        }
    }

    static class MyReducer1 extends Reducer<Text, LongWritable, Text, LongWritable> {

        public MyReducer1() {
            // 添加一个无参数构造函数
        }

        public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            long sum = 0;
            for (LongWritable val : values) {
                sum++;
            }
            context.write(key, new LongWritable(sum));
        }
    }

}

（2）MainApp2.java

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.Map;

public class MainApp2 {

    // Driver端的代码：八股文
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();

        // 如果输出目录已经存在，则先删除
        FileSystem fileSystem = FileSystem.get(configuration);
        Path outputPath = new Path("/output/xuqiu2");
        if(fileSystem.exists(outputPath)) {
            fileSystem.delete(outputPath,true);
        }

        Job job = Job.getInstance(configuration);
        job.setJarByClass(MainApp2.class);

        job.setMapperClass(MyMapper2.class);
        job.setReducerClass(MyReducer2.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        FileInputFormat.setInputPaths(job, new Path("/output/xuqiu3/part-r-00000"));
        FileOutputFormat.setOutputPath(job, new Path("/output/xuqiu2"));

        job.waitForCompletion(true);
    }


    //需求2
    static class MyMapper2 extends Mapper<LongWritable, Text, Text, LongWritable> {

        public MyMapper2() {
            // 添加一个无参构造函数
        }

        private LogParser parser;
        private LongWritable ONE = new LongWritable(1);

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            parser = new LogParser();
        }

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String log = value.toString();
            Map<String, String> logInfo = parser.parse2(log);
            context.write(new Text(logInfo.get("province")), ONE);
        }
    }

    static class MyReducer2 extends Reducer<Text, LongWritable, Text, LongWritable> {

        public MyReducer2() {
            // 添加一个无参数构造函数
        }

        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {

            long count = 0;
            for (LongWritable access : values) {
                count++;
            }
            context.write(key, new LongWritable(count));

        }
    }

}

(3).MainApp3.java

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import java.io.IOException;
import java.util.Map;

public class MainApp3 {

    // Driver端的代码：八股文
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();

        // 如果输出目录已经存在，则先删除
        FileSystem fileSystem = FileSystem.get(configuration);
        Path outputPath = new Path("/output/xuqiu3");
        if(fileSystem.exists(outputPath)) {
            fileSystem.delete(outputPath,true);
        }

        Job job = Job.getInstance(configuration);
        job.setJarByClass(MainApp3.class);

        job.setMapperClass(MyMapper3.class);

        job.setMapOutputKeyClass(NullWritable.class);
        job.setMapOutputValueClass(Text.class);

        FileInputFormat.setInputPaths(job, new Path("/input/trackinfo_20130721.txt"));
        FileOutputFormat.setOutputPath(job, new Path("/output/xuqiu3"));

        job.waitForCompletion(true);
    }



    //需求3
    static class MyMapper3 extends Mapper<LongWritable, Text, NullWritable, Text> {

        public MyMapper3() {
            // 添加一个无参构造函数
        }

        private LogParser parser;

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            parser = new LogParser();
        }

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String log = value.toString();
            Map<String, String> logInfo = parser.parse(log);
            Text etl = new Text(logInfo.get("ip")+"\t"+logInfo.get("url")+"\t"+logInfo.get("sessionId")+"\t"+logInfo.get("time")+"\t"+logInfo.get("country")+"\t"+logInfo.get("province")+"\t"+logInfo.get("city"));
            context.write(NullWritable.get(), etl);
        }
    }



}

将代码打成jar包，上传到HDFS中

hadoop jar /export/idea_workspace/dianshang/target/dianshang-1.0-SNAPSHOT.jar  MainApp3

hadoop jar /export/idea_workspace/dianshang/target/dianshang-1.0-SNAPSHOT.jar  MainApp2

hadoop jar /export/idea_workspace/dianshang/target/dianshang-1.0-SNAPSHOT.jar  MainApp1