kakfa没有相同的数据,stream.count() 跟kafka推来的数据总数是一致的,这样spark streaming取的应该也没有问题,就是通过spark streaming的rdd转换入库的时候成了两份,rdd转换代码如下
row.foreachRDD( rdd =>{
rdd.foreachPartition( x => {
x.foreach( pair =>{
val random = new Random
val sn = pair._1
val stations = pair._2
val tableName = "testgroupby"
if (!isTableIsEsxit(tableName)) {
HBaseTableCommon.createTable(tableName, SC, false, -1, 1)
}
val table = new HBaseTableFactory().getHBaseTableInstance(tableName)
val put = new Put((stations.mac.substring(0, 1) + stations.time + stations.mac + random.nextInt(100000) + System.currentTimeMillis()).getBytes());
// val put = new Put((stations.get(i).mac.substring(0, 1) + stations.get(i).time + stations.get(i).mac).getBytes());
put.addColumn(SC.getBytes(), fields(0), stations.id.getBytes()).addColumn(SC.getBytes(), fields(1), stations.mac.getBytes())
.addColumn(SC.getBytes(), fields(16), stations.longitude.getBytes()).addColumn(SC.getBytes(), fields(17), stations.latitude.getBytes())
.addColumn(SC.getBytes(), fields(18), Bytes.toBytes(stations.time))
table.put(put)
table.close()
})
})
})