使用事物TridentTopology 持久化数据到MySQL 1、构建拓扑JDBCTopology类 package storm.trident.mysql; import java.util.Arrays; import java.util.Map; import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.trident.TridentState; import org.apache.storm.trident.TridentTopology; import org.apache.storm.trident.operation.BaseFunction; import org.apache.storm.trident.operation.CombinerAggregator; import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.spout.IBatchSpout; import org.apache.storm.trident.state.StateType; import org.apache.storm.trident.testing.FixedBatchSpout; import org.apache.storm.trident.testing.MemoryMapState; import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; /** * 事物Trident-MySQL Topology * @author mengyao * */ @SuppressWarnings("all") public class JDBCTopology { public static void main(String[] args) { TridentTopology topology = new TridentTopology(); //Spout数据源 FixedBatchSpout spout = new FixedBatchSpout(new Fields("tels"), 7, new Values("189111 3"), new Values("135111 7"), new Values("189111 2"), new Values("158111 5"), new Values("159111 6"), new Values("159111 3"), new Values("158111 5") ); spout.setCycle(false); //State持久化配置属性 JDBCStateConfig config = new JDBCStateConfig(); config.setDriver("com.mysql.jdbc.Driver"); config.setUrl("jdbc:mysql://localhost:3306/test"); config.setUsername("root"); config.setPassword("123456"); config.setBatchSize(10); config.setCacheSize(10); config.setType(StateType.TRANSACTIONAL); config.setCols("tel"); config.setColVals("sum"); config.setTable("tbl_tel"); topology.newStream("spout", spout) .each(new Fields("tels"), new KeyValueFun(), new Fields("tel", "money")) .groupBy(new Fields("tel")) .persistentAggregate(JDBCState.getFactory(config), new Fields("money"), new SumCombinerAgg(), new Fields("sum")); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test1", new Config(), topology.build()); } } @SuppressWarnings("all") class KeyValueFun extends BaseFunction { @Override public void execute(TridentTuple tuple, TridentCollector collector) { String record = tuple.getString(0); collector.emit(new Values(record.split("\t")[0], record.split("\t")[1])); } } @SuppressWarnings("all") class SumCombinerAgg implements CombinerAggregator<Long> { @Override public Long init(TridentTuple tuple) { return Long.parseLong(tuple.getString(0)); } @Override public Long combine(Long val1, Long val2) { Long val = val1+val2; System.out.println(val); return val; } @Override public Long zero() { return 0L; } } 2、构建基于IBackingMap的JDBCState类 package storm.trident.mysql; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.storm.task.IMetricsContext; import org.apache.storm.trident.state.OpaqueValue; import org.apache.storm.trident.state.State; import org.apache.storm.trident.state.StateFactory; import org.apache.storm.trident.state.StateType; import org.apache.storm.trident.state.TransactionalValue; import org.apache.storm.trident.state.map.CachedMap; import org.apache.storm.trident.state.map.IBackingMap; import org.apache.storm.trident.state.map.NonTransactionalMap; import org.apache.storm.trident.state.map.OpaqueMap; import org.apache.storm.trident.state.map.TransactionalMap; @SuppressWarnings("all") public class JDBCState<T> implements IBackingMap<T> { private static JDBCStateConfig config; JDBCState(JDBCStateConfig config){ this.config = config; } @Override public List<T> multiGet(List<List<Object>> keys) { StringBuilder sqlBuilder = new StringBuilder("SELECT ").append(config.getCols()) .append(","+config.getColVals()) .append(",txid") .append(" FROM "+config.getTable()) .append(" WHERE ") .append(config.getCols()) .append("='"); JDBCUtil jdbcUtil = new JDBCUtil(config.getDriver(),config.getUrl(),config.getUsername(),config.getPassword()); List<Object> result = new ArrayList<Object>(); Map<String, Object> map = null; for (List<Object> list : keys) { Object key = list.get(0); map = jdbcUtil.queryForMap(sqlBuilder.toString()+key+"'"); System.out.println(sqlBuilder.toString()+key+"'"+" 【"+map); Bean itemBean = (Bean)map.get(key); long txid=0L; long val=0L; if (itemBean!=null) { val=itemBean.getSum(); txid=itemBean.getTxid(); } if (config.getType()==StateType.OPAQUE) { result.add(new OpaqueValue(txid, val)); } else if (config.getType()==StateType.TRANSACTIONAL) { result.add(new TransactionalValue(txid, val)); } else { result.add(val); } } return (List<T>) result; } @Override public void multiPut(List<List<Object>> keys, List<T> vals) { //构建新增SQL StringBuilder sqlBuilder = new StringBuilder("INSERT INTO ").append(config.getTable()) .append("("+config.getCols()) .append(","+config.getColVals()) .append(",txid") .append(",time") .append(") VALUES "); for (int i = 0; i < keys.size(); i++) { List<Object> key = keys.get(i); if (config.getType()==StateType.TRANSACTIONAL) { TransactionalValue val = (TransactionalValue)vals.get(i); sqlBuilder.append("("); sqlBuilder.append(key.get(0)); sqlBuilder.append(","); sqlBuilder.append(val.getVal()); sqlBuilder.append(","); sqlBuilder.append(val.getTxid()); sqlBuilder.append(",NOW()"); sqlBuilder.append("),"); } } sqlBuilder.setLength(sqlBuilder.length()-1); System.out.println(sqlBuilder.toString()); //新增数据 JDBCUtil jdbcUtil = new JDBCUtil(config.getDriver(),config.getUrl(),config.getUsername(),config.getPassword()); jdbcUtil.insert(sqlBuilder.toString()); } public static Factory getFactory(JDBCStateConfig config) { return new Factory(config); } static class Factory implements StateFactory { private static JDBCStateConfig config; public Factory(JDBCStateConfig config) { this.config = config; } @Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { final CachedMap map = new CachedMap(new JDBCState(config), config.getCacheSize()); System.out.println(config); if(config.getType()==StateType.OPAQUE) { return OpaqueMap.build(map); } else if(config.getType()==StateType.TRANSACTIONAL){ return TransactionalMap.build(map); }else { return NonTransactionalMap.build(map); } } } } 3、构建基于IBackingMap的JDBCStateConfig配置类 package storm.trident.mysql; import java.util.List; import org.apache.storm.trident.state.StateType; @SuppressWarnings("all") public class JDBCStateConfig { private String url; private String driver; private String username; private String password; private String table; private int batchSize; private String cols; private String colVals; private int cacheSize = 100; private StateType type = StateType.OPAQUE; public String getUrl() { return url; } public void setUrl(String url) { this.url = url; } public String getDriver() { return driver; } public void setDriver(String driver) { this.driver = driver; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } public String getTable() { return table; } public void setTable(String table) { this.table = table; } public int getBatchSize() { return batchSize; } public void setBatchSize(int batchSize) { this.batchSize = batchSize; } public String getCols() { return cols; } public void setCols(String cols) { this.cols = cols; } public String getColVals() { return colVals; } public void setColVals(String colVals) { this.colVals = colVals; } public int getCacheSize() { return cacheSize; } public void setCacheSize(int cacheSize) { this.cacheSize = cacheSize; } public StateType getType() { return type; } public void setType(StateType type) { this.type = type; } @Override public String toString() { return "Test2StateConfig [url=" + url + ", driver=" + driver + ", username=" + username + ", password=" + password + ", table=" + table + ", batchSize=" + batchSize + ", cols=" + cols + ", colVals=" + colVals + ", cacheSize=" + cacheSize + ", type=" + type + "]"; } } 4、构建JDBC工具类和实体Bean package storm.trident.mysql; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; public class JDBCUtil { private String driver; private String url; private String username; private String password; private Connection connection; private PreparedStatement ps; private ResultSet rs; public JDBCUtil(String driver, String url, String username, String password) { this.driver = driver; this.url = url; this.username = username; this.password = password; init(); } void init(){ try { Class.forName(driver); } catch (ClassNotFoundException e) { e.printStackTrace(); } } public boolean insert(String sql) { int state = 0; try { connection = DriverManager.getConnection(url, username, password); ps = connection.prepareStatement(sql); state = ps.executeUpdate(); } catch (SQLException e) { e.printStackTrace(); } finally { try { ps.close(); connection.close(); } catch (SQLException e) { e.printStackTrace(); } } if (state>0) { return true; } return false; } public Map<String, Object> queryForMap(String sql) { Map<String, Object> result = new HashMap<String, Object>(); try { connection = DriverManager.getConnection(url, username, password); ps = connection.prepareStatement(sql); rs = ps.executeQuery(); if(rs.next()){ Bean iteBean=new Bean(rs.getString("tel"), rs.getLong("sum"), rs.getLong("txid"), null); result.put(rs.getString("tel"), iteBean); } } catch (SQLException e) { e.printStackTrace(); } finally { try { ps.close(); connection.close(); } catch (SQLException e) { e.printStackTrace(); } } return result; } public String getDriver() { return driver; } public void setDriver(String driver) { this.driver = driver; } public String getUrl() { return url; } public void setUrl(String url) { this.url = url; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } } package storm.trident.mysql; public class Bean { private String tel; private long sum; private long txid; private String time; public Bean(){ } public Bean(String tel, long sum, long txid, String time) { super(); this.tel = tel; this.sum = sum; this.txid = txid; this.time = time; } public String getTel() { return tel; } public void setTel(String tel) { this.tel = tel; } public long getSum() { return sum; } public void setSum(long sum) { this.sum = sum; } public long getTxid() { return txid; } public void setTxid(long txid) { this.txid = txid; } public String getTime() { return time; } public void setTime(String time) { this.time = time; } @Override public String toString() { return "Bean [tel=" + tel + ", sum=" + sum + ", txid=" + txid + ", time=" + time + "]"; } }