Storm案例实战

• 需求：实时获取上网用户数据（APP的IP地址），并在地图展示出来用户的位置信息。
• 实时计算的解决方案：Storm0.8.2+Hadoop2.2.0+JDK1.7+ZooKeeper3.4.6
• 架构设计

数据文件A：用户，手机号，手机MAC地址，上网地址，访问网站，网址类型，上行数据包数量，下行数据包数量，上行总量，下行总量，HTTP访问返回值

141，3276006        18540852316      71-77-16-4c-41-b4:CMCC     10.116.136.202    alipay.com  支付   15     9       7161 4269 200

1413272713        15568385737      76-12-7d-b4-9e-6c:CMCC-EASY     158.219.67.93      iteye.com   门户   2       12     4613 1074 200

数据文件B：国家，省份，ISP提供商，最大IP,最小IP

中国   内蒙古                  联通   1039179776        1039187967

中国   福建省                  电信   3405938176        3405938687

数据文件C:国家，纬度，经度

安道尔         42.5   1.5

阿拉伯联合酋长国 24     54

• 第一步，创建Spout和Bolt,提交area-topology任务

package cyb.storm;

import backtype.storm.Config;

import backtype.storm.StormSubmitter;

import backtype.storm.topology.TopologyBuilder;

public class Main {      

         public static void main(String[] args) throws Exception {

                   TopologyBuilder builder = new TopologyBuilder();

                   builder.setSpout("spout", new ReaderSpout(), 4);

                   //4 the number of tasks that should be assigned to execute this spout

                   builder.setBolt("area-bolt", new GetAreaBolt()).shuffleGrouping("spout");

                   builder.setBolt("longitude-bolt", new GetLongitudeBolt()).shuffleGrouping("area-bolt");

                   Config config = new Config();

                   config.setNumWorkers(4);

                   config.setMaxSpoutPending(1000);

                   StormSubmitter.submitTopology("area-topology", config,

                                     builder.createTopology());

         }

}

• 第二步，创建读取HDFS文件，截取出IP地址，发射个下一个Bolt

package cyb.storm;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.URI;

import java.util.Map;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

import backtype.storm.spout.SpoutOutputCollector;

import backtype.storm.task.TopologyContext;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.topology.base.BaseRichSpout;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Values;

import backtype.storm.utils.Utils;

@SuppressWarnings("serial")

public class ReaderSpout extends BaseRichSpout {

         SpoutOutputCollector _collector;

         @SuppressWarnings("rawtypes")

         @Override

         public void open(Map conf, TopologyContext context,

                            SpoutOutputCollector collector) {

                   _collector = collector;

         }

         @Override

         public void nextTuple() {

                   String uri = "hdfs://master:9000/storm/m.txt";

                   InputStream in = null;

                   try {

                            Configuration conf = new Configuration();

                            FileSystem fs = FileSystem.get(URI.create(uri), conf);

                            in = fs.open(new Path(uri));

                            BufferedReader br = new BufferedReader(new InputStreamReader(in));

                            String line = null;

                            while (null != (line = br.readLine())) {

                                     _collector.emit(new Values(line));

                                     Utils.sleep(100);

                            }

                   } catch (IOException e) {

                            e.printStackTrace();

                   } finally {

                            IOUtils.closeStream(in);

                   }

         }

         @Override

         public void ack(Object id) {

         }

         @Override

         public void fail(Object id) {

         }

         @Override

         public void declareOutputFields(OutputFieldsDeclarer declarer) {

                   declarer.declare(new Fields("ip"));

         }

}

• 第三步，IP地址转换，比如127.0.0.1转换为1846789219，发射1846789219到下一个Bolt

package cyb.storm;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.Map;

import backtype.storm.task.TopologyContext;

import backtype.storm.topology.BasicOutputCollector;

import backtype.storm.topology.IBasicBolt;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Tuple;

import backtype.storm.tuple.Values;

@SuppressWarnings("serial")

public class GetAreaBolt implements IBasicBolt {

         static Connection conn;

         static Statement st;

         public static Connection getConnection() {

                   Connection con = null; // define Connection

                   try {

                            Class.forName("com.mysql.jdbc.Driver");// load Mysql driver

                            con = DriverManager.getConnection(

                                               "jdbc:mysql://192.168.32.72:3306/test", "hadoop", "hadoop");

                   } catch (Exception e) {

                            System.out.println("Connection failed! " + e.getMessage());

                   }

                   return con;

         }

         public static String select(long ipp) {

                   conn = getConnection(); // get connection

                   try {

                            String sql = "select area from ip where '" + ipp

                                               + "' between minip and maxip";

                            st = conn.createStatement();

                            ResultSet rs = st.executeQuery(sql);

                            String name = rs.getString("area");

                            return name;

                            // conn.close(); //close connection

                   } catch (SQLException e) {

                            System.out.println("failed! " + e.getMessage());

                            return null;

                   }

         }

         @Override

         public void execute(Tuple tuple, BasicOutputCollector collector) {

                   String line = tuple.toString();

                   String all[] = line.split("\t", -1);

                   long longIp = GetAreaBolt.ipToLong(all[3]);

                   collector.emit(new Values(select(longIp)));

         }

         public static long ipToLong(String strIp) {

                   // transfer ip like 127.0.0.1 to decimal integer

                   long[] ip = new long[4];

                   // find the position of dot

                   int position1 = strIp.indexOf(".");

                   int position2 = strIp.indexOf(".", position1 + 1);

                   int position3 = strIp.indexOf(".", position2 + 1);

                   // transfer string to integer

                   ip[0] = Long.parseLong(strIp.substring(0, position1));

                   ip[1] = Long.parseLong(strIp.substring(position1 + 1, position2));

                   ip[2] = Long.parseLong(strIp.substring(position2 + 1, position3));

                   ip[3] = Long.parseLong(strIp.substring(position3 + 1));

                   return (ip[0] << 24) + (ip[1] << 16) + (ip[2] << 8) + ip[3];

         }

         @Override

         public void declareOutputFields(OutputFieldsDeclarer declarer) {

                   declarer.declare(new Fields("area"));

         }

         @Override

         public Map<String, Object> getComponentConfiguration() {

                   return null;

         }

         @SuppressWarnings("rawtypes")

         @Override

         public void prepare(Map stormConf, TopologyContext context) {

                   conn = getConnection();

                   try {

                            st = conn.createStatement();

                   } catch (SQLException e) {

                            e.printStackTrace();

                   }

         }

         @Override

         public void cleanup() {

         }

}

• 第四步，地址和经纬度转换，从文件中根据地址，获取出经纬度

package chapter04.storm;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.URI;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.HashMap;

import java.util.Map;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import backtype.storm.task.TopologyContext;

import backtype.storm.topology.BasicOutputCollector;

import backtype.storm.topology.IBasicBolt;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Tuple;

public class GetLongitudeBolt implements IBasicBolt {

         private static final long serialVersionUID = 1L;

         private HashMap<String, String> longitude = new HashMap<String, String>();

         static Connection conn;

         static Statement st;

         private String uri = "hdfs://master:9000/storm/lng-lat-mapping.txt";

         public static Connection getConnection() {

                   Connection con = null; // get connection

                   try {

                            Class.forName("com.mysql.jdbc.Driver");// load Mysql driver

                            con = DriverManager.getConnection(

                                               "jdbc:mysql://192.168.32.72:3306/test", "hadoop", "hadoop");

                   } catch (Exception e) {

                            System.out.println("connect mysql failed! " + e.getMessage());

                   }

                   return con; // return connection

         }

         public static void insert(String area, String jing, String wei) {

                   conn = getConnection(); // get connection

                   try {

                            String sql = "INSERT INTO position(area,lng,lat)" + " VALUES ('"

                                               + area + "','" + jing + "','" + wei + "')";

                            st = (Statement) conn.createStatement(); // create static sql statement

                            st.executeUpdate(sql); // exec sql

                            conn.close(); // close connection

                   } catch (SQLException e) {

                            System.out.println("insert failed! " + e.getMessage());

                   }

         }

         public void execute(Tuple tuple, BasicOutputCollector collector) {

                   String word = tuple.toString();

                   if (longitude.get(word) != null) {

                            insert(longitude.get(word).split("\t", -1)[0], longitude.get(word)

                                               .split("\t", -1)[1], longitude.get(word).split("\t", -1)[2]);

                   }

         }

         public void declareOutputFields(OutputFieldsDeclarer declarer) {

                   declarer.declare(new Fields("area", "lng", "lat"));

         }

         @Override

         public Map<String, Object> getComponentConfiguration() {

                   return null;

         }

         @SuppressWarnings("rawtypes")

         @Override

         public void prepare(Map stormConf, TopologyContext context) {

                   conn = getConnection();

                   try {

                            st = (Statement) conn.createStatement();

                   } catch (SQLException e1) {

                            e1.printStackTrace();

                   }

                   InputStream in = null;

                   FileSystem fs = null;

                   try {

                            fs = FileSystem.get(URI.create(uri), new Configuration());

                            in = fs.open(new Path(uri));

                   } catch (IOException e) {

                            e.printStackTrace();

                   }

                   BufferedReader br = new BufferedReader(new InputStreamReader(in));

                   String line = null;

                   try {

                            while (null != (line = br.readLine())) {

                                     longitude.put(line.split("\t", -1)[0], line);

                            }

                   } catch (IOException e) {

                            e.printStackTrace();

                   }

         }

         @Override

         public void cleanup() {

         }

}

后续就是入库和显示JSP,不是核心技术，就不再赘述了。