操作
shell
命名空间操作
hbase(main):006:0> create_namespace "zzj"
hbase(main):007:0> create_namespace "shahe",{'subway'=>'bj'}
hbase(main):001:0> drop_namespace 'zzj'
hbase(main):001:0> describe_namespace 'name'
hbase(main):003:0> alter_namespace 'name',{METHOD => 'set', 'subway'=>'bj'}
hbase(main):005:0> list_namespace
表操作
hbase(main):007:0> create_namespace 'zzj'
hbase(main):008:0> create 'zzj:t_user','cf1','cf2'
hbase(main):002:0> drop 'zzj:t_user'
hbase(main):003:0> disable
hbase(main):004:0> drop 'zzj:t_user'
hbase(main):002:0> list
hbase(main):003:0> list_namespace_tables 'zzj'
数据操作
hbase(main):005:0> put 'zzj:t_user','1','cf1:name','zhangsan'
hbase(main):006:0> t = get_table 'zzj:t_user'
hbase(main):007:0> t.put '1','cf1:age',18
hbase(main):008:0> t.put '1','cf1:age',19
hbase(main):009:0> t.scan
hbase(main):013:0> get 'zzj:t_user' ,'1'
hbase(main):001:0> create 'zzj:t_user',{NAME=>'cf1',VERSIONS=>3}
hbase(main):009:0> t.delete '1','cf1:name'
hbase(main):012:0> t.get '1',{COLUMN=>'cf1:age',VERSIONS=>3}
hbase(main):002:0> deleteall 'zzj:t_user','1'
hbase(main):019:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'1'}
hbase(main):020:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'2'}
hbase(main):021:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'3'}
hbase(main):022:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'1',LIMIT=>1}
hbase(main):023:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'1',LIMIT=>2}
hbase(main):025:0> count 'zzj:t_user'
hbase(main):026:0> append 'zzj:t_user','1','cf1:name','110'
hbase(main):028:0> truncate 'zzj:t_user'
Java
依赖
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.4</version>
</dependency>
核心操作
@Before
public void getClient() throws Exception {
configuration = new Configuration();
configuration.set("hbase.zookeeper.quorum", "HadoopNode00");
configuration.set("hbase.zookeeper.property.clientPort", "2181");
conn = ConnectionFactory.createConnection(configuration);
admin = conn.getAdmin();
}
@After
public void close() throws Exception {
admin.close();
conn.close();
}
命名空间操作
@Test
public void createNameSpace() throws Exception {
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("zzj").addConfiguration("admin", "aaa").build();
admin.createNamespace(namespaceDescriptor);
}
@Test
public void listNameSpace() throws Exception {
NamespaceDescriptor[] listNamespaceDescriptors = admin.listNamespaceDescriptors();
for (NamespaceDescriptor listNamespaceDescriptor : listNamespaceDescriptors) {
System.out.println(listNamespaceDescriptor.getName());
}
}
@Test
public void modifyNameSpace() throws Exception {
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("zzj").addConfiguration("aa", "bb").removeConfiguration("admin").build();
admin.modifyNamespace(namespaceDescriptor);
}
@Test
public void deleteNameSpace() throws Exception{
admin.deleteNamespace("zzj");
}
表常规
@Test
public void createTables() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user1");
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
HColumnDescriptor cf1 = new HColumnDescriptor("cf1");
cf1.setMaxVersions(3);
=
HColumnDescriptor cf2 = new HColumnDescriptor("cf2");
cf2.setMaxVersions(3);
hTableDescriptor.addFamily(cf1);
hTableDescriptor.addFamily(cf2);
admin.createTable(hTableDescriptor);
}
@Test
public void dropTable() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user1");
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
数据操作
@Test
public void testPutOne() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user");
Table table = conn.getTable(tableName);
Put put1 = new Put("1".getBytes());
put1.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
put1.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
put1.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());
table.put(put1);
table.close();
}
@Test
public void testPutList() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user");
BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);
Put put1 = new Put("4".getBytes());
put1.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
put1.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
put1.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());
Put put2 = new Put("5".getBytes());
put2.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
put2.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
put2.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());
Put put3 = new Put("6".getBytes());
put3.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
put3.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
put3.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());
ArrayList<Put> puts = new ArrayList<Put>();
puts.add(put1);
puts.add(put2);
puts.add(put3);
bufferedMutator.mutate(puts);
bufferedMutator.close();
}
@Test
public void tetsDelete() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user");
Table table = conn.getTable(tableName);
Delete delete = new Delete("6".getBytes());
table.delete(delete);
table.close();
}
@Test
public void testDeleteList() throws Exception{
TableName tableName = TableName.valueOf("zzj:t_user");
BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);
Delete delete1 = new Delete("1".getBytes());
Delete delete2 = new Delete("2".getBytes());
Delete delete3 = new Delete("3".getBytes());
ArrayList<Delete> deletes = new ArrayList<Delete>();
deletes.add(delete1);
deletes.add(delete2);
deletes.add(delete3);
bufferedMutator.mutate(deletes);
bufferedMutator.close();
}
@Test
public void testGet() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user");
Table table = conn.getTable(tableName);
Get get = new Get("4".getBytes());
Result result = table.get(get);
byte[] name = result.getValue("cf1".getBytes(), "name".getBytes());
byte[] age = result.getValue("cf1".getBytes(), "age".getBytes());
byte[] sex = result.getValue("cf1".getBytes(), "sex".getBytes());
System.out.println(new String(name) + "-" + new String(age) + "-" + new String(sex));
}
@Test
public void testGet02() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user");
Table table = conn.getTable(tableName);
Get get = new Get("4".getBytes());
get.setMaxVersions(2);
get.addColumn("cf1".getBytes(), "name".getBytes());
Result result = table.get(get);
List<Cell> cellList = result.getColumnCells("cf1".getBytes(), "name".getBytes());
for (Cell cell : cellList) {
byte[] rowkey = CellUtil.cloneRow(cell);
byte[] cf = CellUtil.cloneFamily(cell);
byte[] qualifier = CellUtil.cloneQualifier(cell);
byte[] value = CellUtil.cloneValue(cell);
long timestamp = cell.getTimestamp();
System.out.println(new String(rowkey) + "--" + new String(cf) + "--" + new String(qualifier) + "--" + new String(value) +"--" +timestamp);
}
@Test
public void testScan() throws Exception {
TableName tableName = TableName.valueOf("zzj:t_user");
Table table = conn.getTable(tableName);
Scan scan = new Scan();
PrefixFilter prefixFilter1 = new PrefixFilter("4".getBytes());
PrefixFilter prefixFilter2 = new PrefixFilter("5".getBytes());
FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ONE,prefixFilter1, prefixFilter2);
scan.setFilter(list);
ResultScanner results = table.getScanner(scan);
for (Result result : results) {
byte[] row = result.getRow();
byte[] name = result.getValue("cf1".getBytes(), "name".getBytes());
byte[] age = result.getValue("cf1".getBytes(), "age".getBytes());
byte[] sex = result.getValue("cf1".getBytes(), "sex".getBytes());
System.out.println(new String(row) + "--" + new String(name) + "-" + new String(age) + "-" + new String(sex));
}
}
MapReduce
依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.4</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.2.4</version>
</dependency>
public class JobRunner {
public static void main(String[] args) throws Exception {
System.setProperty("HADOOP_USER_NAME", "root");
Configuration configuration = new Configuration();
configuration.set("hbase.zookeeper.quorum", "HadoopNode00");
configuration.set("hbase.zookeeper.property.clientPort", "2181");
configuration.addResource("conf2/core-site.xml");
configuration.addResource("conf2/hdfs-site.xml");
configuration.addResource("conf2/mapred-site.xml");
configuration.addResource("conf2/yarn-site.xml");
configuration.set(MRJobConfig.JAR, "G:\\IDEA_WorkSpace\\BigData\\HBase_Test\\target\\HBase_Test-1.0-SNAPSHOT.jar");
configuration.set("mapreduce.app-submission.cross-platform", "true");
Job job = Job.getInstance(configuration);
job.setJarByClass(JobRunner.class);
job.setInputFormatClass(TableInputFormat.class);
job.setOutputFormatClass(TableOutputFormat.class);
TableMapReduceUtil.initTableMapperJob(
"zzj:t_user",
new Scan(),
UserMapper.class,
Text.class,
DoubleWritable.class,
job
);
TableMapReduceUtil.initTableReducerJob(
"zzj:t_result",
UserReducer.class,
job);
job.waitForCompletion(true);
}
}
public class UserMapper extends TableMapper<Text, DoubleWritable> {
@Override
protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {
byte[] bytes = key.get();
String rowkey = Bytes.toString(bytes);
String company = rowkey.split(":")[0];
byte[] salaryByte = value.getValue("cf1".getBytes(), "salary".getBytes());
double salary = Bytes.toDouble(salaryByte);
context.write(new Text(company), new DoubleWritable(salary));
}
}
public class UserReducer extends TableReducer<Text, DoubleWritable, NullWritable> {
@Override
protected void reduce(Text key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException {
double totalSalary = 0.0;
int count = 0;
for (DoubleWritable value : values) {
totalSalary += value.get();
count++;
}
Put put = new Put(key.getBytes());
put.addColumn("cf1".getBytes(), "avgSalary".getBytes(), (totalSalary / count + "").getBytes());
context.write(NullWritable.get(), put);
}
}
搭建环境
环境
- Hadoop
HADOOP_HOME
- Zookeeper
单机
[root@HadoopNode00 ~]
[root@HadoopNode00 ~]
[root@HadoopNode00 ~]
export HBASE_HOME=/usr/hbase/hbase-1.2.4
export HBASE_MANAGES_ZK=false
export PATH=$PATH:$HBASE_HOME/bin
[root@HadoopNode00 ~]
<property>
<name>hbase.rootdir</name>
<value>hdfs://HadoopNode00:9000/hbase</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>HadoopNode00</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
[root@HadoopNode00 ~]
HadoopNode00
[root@HadoopNode00 zookeeper-3.4.6]
集群
HadoopNode01 | HadoopNode02 | HadoopNode03 |
---|
nn1 | nn2 | |
journal node | journal node | journal node |
zkfc | zkfc | |
datanode | datanode | datanode |
zk01 | zk02 | zk03 |
| rm1 | rm2 |
nodemanager | nodemanager | nodemanager |
hregionserver | hregionserver | hregionserver |
hmaster | hmaster | hmaster |
[root@ZK01 ~]
JMX enabled by default
Using config: /usr/zk/zookeeper-3.4.6/conf/zk.conf
Starting zookeeper ... STARTED
[root@ZK01 ~]
JMX enabled by default
Using config: /usr/zk/zookeeper-3.4.6/conf/zk.conf
Mode: follower
[root@HadoopNodeX ~]
[root@HadoopNodeX ~]
[root@HadoopNodeX ~]
export HBASE_HOME=/usr/hbase/hbase-1.2.4
export HBASE_MANAGES_ZK=false
export PATH=$PATH:$HBASE_HOME/bin
[root@HadoopNodeX ~]
[root@HadoopNodeX ~]
<property>
<name>hbase.rootdir</name>
<value>hdfs://mycluster/hbase</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>ZK01,ZK02,ZK03</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
[root@HadoopNodeX ~]
hadoopNode01
hadoopNode02
hadoopNode03
[root@HadoopNodeX ~]
[root@HadoopNodeX ~]