文章目录
一、zookeeper?
1. 搭建zk集群
<1>上传解压
tar -zxvf zookeeper-3.4.9.tar.gz
<2>.修改配置
cd /usr/local/zk/zookeeper-3.4.9/conf
cp zoo_sample.cfg zoo.cfg #复制配置模板
vi zoo.cfg
zoo.cfg
#配置zk数据目录,zkdatas是新创建的目录
dataDir=/usr/local/zk/zookeeper-3.4.9/zkdatas
# 保留多少个快照
autopurge.snapRetainCount=3
# 日志多少小时清理一次
autopurge.purgeInterval=1
# 集群中服务器地址
server.1=node01:2888:3888
server.2=node02:2888:3888
server.3=node03:2888:3888
<3>.添加myid文件
echo 1 > /usr/local/zk/zookeeper-3.4.9/zkdatas/myid
<4>.分发zk
测点
cd /usr/local/zk
scp -r zookeeper-3.4.9/ node02:$PWD
scp -r zookeeper-3.4.9/ node03:$PWD
<5>.修改myid
node02 改成2
node03 改成3
<6>.启动脚本 zk.sh
#!/bin/bash
case $1 in
"start"){
for i in node01 node02 node03
do
ssh $i "source /etc/profile; /usr/local/zk/zookeeper-3.4.9/bin/zkServer.sh start /usr/local/zk/zookeeper-3.4.9/conf/zoo.cfg"
done
};;
"stop"){
for i in node01 node02 node03
do
ssh $i "source /etc/profile; /usr/local/zk/zookeeper-3.4.9/bin/zkServer.sh stop"
done
};;
"status"){
for i in node01 node02 node03
do
ssh $i "source /etc/profile; /usr/local/zk/zookeeper-3.4.9/bin/zkServer.sh status"
done
};;
esac
二、Hadoop?
1.hadoop集群搭建
<1>.上传解压
tar -xzvf hadoop-3.1.1.tar.gz
<2>.修改配置文件
cd /usr/local/hadoop/hadoop-3.1.1/etc/hadoop
- 文件core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://node01:8020</value>
</property>
<!-- 临时文件存储目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/hadoop-3.1.1/datas/tmp</value>
</property>
<!-- 缓冲区大小,实际工作中根据服务器性能动态调整 -->
<property>
<name>io.file.buffer.size</name>
<value>8192</value>
</property>
<!-- 开启hdfs的垃圾桶机制,删除掉的数据可以从垃圾桶中回收,单位分钟 -->
<property>
<name>fs.trash.interval</name>
<value>10080</value>
</property>
</configuration>
- 文件hadoop-env.sh
export JAVA_HOME=/usr/local/jdk/jdk1.8
export HDFS_NAMENODE_USER="root"
export HDFS_DATANODE_USER="root"
export HDFS_SECONDARYNAMENODE_USER="root"
export YARN_RESOURCEMANAGER_USER="root"
export YARN_NODEMANAGER_USER="root"
- 文件hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/datas/namenode/namenodedatas</value>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>10</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/datas/datanode/datanodeDatas</value>
</property>
<property>
<name>dfs.namenode.http-address</name>
<value>node01:50070</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.namenode.checkpoint.edits.dir</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/datas/dfs/nn/snn/edits</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>node01.hadoop.com:50090</value>
</property>
<property>
<name>dfs.namenode.edits.dir</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/datas/dfs/nn/edits</value>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/datas/dfs/snn/name</value>
</property>
</configuration>
- mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.map.memory.mb</name>
<value>1024</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx512M</value>
</property>
<property>
<name>mapreduce.reduce.memory.mb</name>
<value>1024</value>
</property>
<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx512M</value>
</property>
<property>
<name>mapreduce.task.io.sort.mb</name>
<value>256</value>
</property>
<property>
<name>mapreduce.task.io.sort.factor</name>
<value>100</value>
</property>
<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>25</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>node01:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>node01:19888</value>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>/usr/local/hadoop/hadoop3.1.1/datas/jobhsitory/intermediateDoneDatas</value>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>/usr/local/hadoop/hadoop-3.1.1/datas/jobhsitory/DoneDatas</value>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=/usr/local/hadoop/hadoop-3.1.1</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=/usr/local/hadoop/hadoop-3.1.1/</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=/usr/local/hadoop/hadoop-3.1.1</value>
</property>
</configuration>
- yarn-site.xml
<configuration>
<property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>node01:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>node01:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>node01:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>node01:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>node01:8088</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>node01</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>1024</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>2048</value>
</property>
<property>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
<value>2.1</value>
</property>
<!-- 设置不检查虚拟内存的值,不然内存不够会报错 -->
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>1024</value>
</property>
<property>
<name>yarn.nodemanager.resource.detect-hardware-capabilities</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/nodemanager/nodemanagerDatas</value>
</property>
<property>
<name>yarn.nodemanager.log-dirs</name>
<value>file:///usr/local/hadoop/hadoop-3.1.1/datas/nodemanager/nodemanagerLogs</value>
</property>
<property>
<name>yarn.nodemanager.log.retain-seconds</name>
<value>10800</value>
</property>
<property>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/usr/local/hadoop/hadoop-3.1.1/remoteAppLog/remoteAppLogs</value>
</property>
<property>
<name>yarn.nodemanager.remote-app-log-dir-suffix</name>
<value>logs</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>18144000</value>
</property>
<property>
<name>yarn.log-aggregation.retain-check-interval-seconds</name>
<value>86400</value>
</property>
<!-- yarn上面运行一个任务,最少需要1.5G内存,虚拟机没有这么大的内存就调小这个值,不然会报错 -->
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>1024</value>
</property>
</configuration>
- worker
node01
node02
node03
<3>.创建文件
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/tmp
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/dfs/nn/snn/edits
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/namenode/namenodedatas
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/datanode/datanodeDatas
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/dfs/nn/edits
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/dfs/snn/name
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/jobhsitory/intermediateDoneDatas
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/jobhsitory/DoneDatas
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/nodemanager/nodemanagerDatas
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/nodemanager/nodemanagerLogs
mkdir -p /usr/local/hadoop/hadoop-3.1.1/datas/remoteAppLog/remoteAppLogs
<4>.分发
scp -r hadoop-3.1.1/ node02:$PWD
scp -r hadoop-3.1.1/ node03:$PWD
<5>.环境配置
export HADOOP_HOME=/usr/local/hadoop/hadoop-3.1.1/
export PATH=:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
<6>.格式化HDFS
bin/hdfs namenode -format
<7>.启动集群
/usr/local/hadoop/hadoop-3.1.1/sbin/start-dfs.sh #hdfs集群
/usr/local/hadoop/hadoop-3.1.1/sbin/start-yarn.sh #mapreduce yarn集群
三、Hbase?
1.Hbase集群搭建
<1>.上传解压
tar -xzvf hbase-1.2.7-bin.tar.gz
<2>.配置文件
hbase-env.sh
export JAVA_HOME=/usr/local/jdk/jdk1.8 #Java安装路径
export HBASE_MANAGES_ZK=false #不用Hbase自己的zk
hbase-site.xml
#依赖hdfs
<property>
<name>hbase.rootdir</name>
<value>hdfs://node01:8020/hbase</value>
</property>
#开启集群
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
#依赖zk,默认2181
<property>
<name>hbase.zookeeper.quorum</name>
<value>node01,node02,node03</value>
</property>
regionservers
node01
node02
node03
<3>.启动集群
cd sbin
./start-hbase.sh
<4>.访问hbase web页面 http://192.168.18.110:16010/
2.Hbase命令行操作
cd bin
hbase shell #进入命令行
#命名空间 相当于库
create_namespace "aa" #新增命名空间
list_namespace #查看命名空间
disable "aa" #disable状态才可以drop
drop_namespace "aa" #删除命名空间
#表
list #查用户表
describe "test01:student" #查询表的详情
create "aa" ,"info" #创建表 默认是default命名空间,表名+列簇
alter "aa",{NAME=>"data",VERSIONS=>"3"} #修改表结构
disable "aa"
drop "aa" #删除表
#数据
put "aa","1001","info:sex","male" #插入数据
scan "aa" #查看表数据
get "aa","1001" #获取行数据
get "aa","1001","info:sex" #获取行中某个列
scan "aa",{STARTROW=>'1001',STOPROW=>'1002'} #{) 左闭右开
deleteall 'aa','1001' #删除一行数据
delete "aa","1001","info:sex" #删除某一行某一列 删除所有版本
truncate "aa" #删除所有
3.Hbase API
package cn.zjw;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
/**
* @Description:
* @author: zjw
* @date: 2021年03月04日 13:21
*/
public class HbaseApiStart {
Configuration configuration = null;
Connection connection = null;
Admin admin = null;
@Before
public void initConfig() throws IOException {
configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.quorum", "node01,node02,node03");
connection = ConnectionFactory.createConnection(configuration);
admin = connection.getAdmin();
}
/**表操作*/
@Test
public void testApi() throws IOException {
//不指定命名空间,默认的default
Boolean b = isExit("test01:student");
System.out.println("student表存在吗: " + b);
//创建表
createTable("stu3","info","score");
//删除表
deleteTable("stu3");
//创建命名空间
createNameSpace("test02");
}
/**数据操作*/
@Test
public void testApi01() throws IOException {
//插入数据
// put("stu3","1001","info","name","zs");
// put("stu3","1001","score","Math","95");
// put("stu3","1001","score","chinese","95");
// put("stu3","1002","info","name","ls");
// put("stu3","1002","score","Math","77");
// put("stu3","1003","info","name","ww");
//get数据
// get("stu3","1001","info","name");
//scan数据
// scan("stu3",null,"score","Math");
//delete
delete("stu3","1008","info","name");
}
/**
* 判断表是否存在
*/
public Boolean isExit(String name) throws IOException {
return admin.tableExists(TableName.valueOf(name));
}
/**
* 创建表
*/
public void createTable(String name,String... columnFamily) throws IOException {
while (isExit(name)){
System.out.println("表已存在");
}
//表名
HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(name));
//列簇
for (String column : columnFamily) {
tableDescriptor.addFamily(new HColumnDescriptor(column));
}
admin.createTable(tableDescriptor);
System.out.println("表创建成功");
}
/**删除表*/
public void deleteTable(String name) throws IOException {
while (! isExit(name)){
System.out.println("表不存在");
}
//先下线
admin.disableTable(TableName.valueOf(name));
admin.deleteTable(TableName.valueOf(name));
System.out.println("表删除成功");
}
/**创建命名空间*/
public void createNameSpace(String name) {
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create(name).build();
try {
admin.createNamespace(namespaceDescriptor);
}
catch (NamespaceExistException e){
System.out.println("命名空间已存在");
}
catch (IOException e) {
e.printStackTrace();
}
}
/**插入数据*/
public void put(String name,String rk,String cf,String cn,String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
Put put = new Put(Bytes.toBytes(rk));
put.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn),Bytes.toBytes(value));
table.put(put);
table.close();
System.out.println("插入成功");
table.close();
}
/**获取数据get*/
public void get(String name,String rk,String cf,String cn) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
Get get = new Get(Bytes.toBytes(rk));
//加上列簇
// get.addFamily(Bytes.toBytes(cf));
//加上列
get.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn));
get.setMaxVersions(2);
Result result = table.get(get);
for (Cell cell : result.rawCells()) {
System.out.println("列簇:"+Bytes.toString(CellUtil.cloneFamily(cell))+",列:"+
Bytes.toString(CellUtil.cloneQualifier(cell))+",value:"+Bytes.toString(CellUtil.cloneValue(cell)));
}
table.close();
}
/**获取数据scan*/
public void scan(String name,String rk,String cf,String cn) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
Scan scan = new Scan();
// Scan scan = new Scan(Bytes.toBytes("1001"),Bytes.toBytes("1003"));
// scan.addFamily(Bytes.toBytes(cf));
scan.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn));
ResultScanner results = table.getScanner(scan);
for (Result result : results) {
for (Cell cell : result.rawCells()) {
System.out.println("rowKey:"+Bytes.toString(CellUtil.cloneRow(cell))+",列簇:"+Bytes.toString(CellUtil.cloneFamily(cell))+",列:"+
Bytes.toString(CellUtil.cloneQualifier(cell))+",value:"+Bytes.toString(CellUtil.cloneValue(cell)));
}
}
table.close();
}
/**删除*/
public void delete(String name,String rk,String cf,String cn) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
//rowkey删-->会deleteall命令,会将指定rowKey下的所有列族以及所有列的所有版本数据都删除,最终做的标记类型也是DeleteFamily
Delete delete = new Delete(Bytes.toBytes(rk));
// delete.addFamily(Bytes.toBytes(cf));
// delete.addColumns(Bytes.toBytes(cf),Bytes.toBytes(cn),1614844098039L);
delete.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn),1614844704715L);
table.delete(delete);
table.close();
}
@After
public void close() throws IOException {
admin.close();
connection.close();
}
}
4.删除分析
根据rowkey删,删除前
删除后 把1001索引列簇全部删除,打上DeleteFamily标记
根据rowkey+columnFamily删,删除前
删除后 把1004 info全部删除,打上deleteFamily标记
根据rk+cf+cn删除
=addColumns() 该方法的作用是删除指定列的所有版本的数据,同时它还有一个重载的方法,多了一个参数,这个参数是时间戳参数,作用是删除所有小于等于指定列的指定时间戳的所有版本的数据=
删除前
删除后,打上deleteColumn
加上时间戳 删除前 1614844098039L
删除后 比他大的版本没有删除掉
=addColumn() 该方法添加指定的列,然后执行删除操作,是存在较大的争议的,它的作用是删除指定列的最新版本的那一条数据而非全部=,同时也可以传入要删除的指定的时间戳。标记是delete 慎用
删除前
删除后 最大版本删除,原来版本没有删除 1007老版本顶替
=加时间戳,有对应的时间戳就删除指定版本,没有就删除最新版本=
5.hbase原理
<1> 高可用,实时,面向列,可伸缩,海量数据 k-v型的数据库
<2> 概念
Row Key: 行唯一标识,字典排序
Column Family:qualifier 列簇:列
Timestamp: 时间戳,版本
Cell:行和列确定单元 由{row key,column(= +),version}唯一确定的单元
<3>架构
zk:
监控regionserver
保证只有一个master
保存.mate表信息,能找到master节点
master:
分配region,挂掉的重新分配region
region server负载均衡
RegionServer:
处理region的IO
<4>Hlog文件
当memStore有数据,挂掉了.从hlog中回复.写入的时候,先写到Hlog,在写道缓存,就返回
<5>刷新和compact
当所有缓存达到设置的值时,刷新到storefile,等storefile越来越大时,进行inpact,合并过程中会进行版本合并和删除工作
四、HDFS?
1.架构
1.架构
2.分块副本
分块默认1块128M,例如一个文件 130M, 会被切分成 2 个 block 块, 保存在两个 block 块里面, 实际占用磁盘 130M 空间, 而不是占用256M的磁盘空间 hdfs-site.xml 中可以修改块大小
<property>
<name>dfs.block.size</name>
<value>块大小 以字节为单位</value>
</property>
副本机架感应,分开存
3.SecondaryNameNode
fsimage :元数据镜像,元数据存在内存,重启后恢复镜像
edits: 镜像之后元数据的变化操作,恢复镜像之前,先把edits执行一遍,形成最新的镜像
secondaryNameNode: 辅助namenode,当namenode一直不重启,edits文件会过大,当时间因素/edits文件大小因素触时,secondaryNameNode会将两文件合并成一个新的镜像,并刷新镜像,清空edits
2.写入过程
3.读取过程
Client 选取排序靠前的 DataNode 来读取 bloc(副本最近原则)
4.shell操作
hadoop目录下执行命令就行
5.web页面
web端口 50070
server端口 8020
6.API操作
=操作API之前 ,windows必须配置hadoop环境=
<1>解压windows环境的hadoop
<2>配置环境变量
<3>将hadoop bin文件下hadoop.dll复制到windows 目录下C:\Windows\System32
<4>重启电脑
package cn.zjw;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
/**
* @Description:
* @author: zjw
* @date: 2021年03月04日 13:21
*/
public class HbaseApiStart {
Configuration configuration = null;
Connection connection = null;
Admin admin = null;
@Before
public void initConfig() throws IOException {
configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.quorum", "node01,node02,node03");
connection = ConnectionFactory.createConnection(configuration);
admin = connection.getAdmin();
}
/**表操作*/
@Test
public void testApi() throws IOException {
//不指定命名空间,默认的default
Boolean b = isExit("test01:student");
System.out.println("student表存在吗: " + b);
//创建表
createTable("stu3","info","score");
//删除表
deleteTable("stu3");
//创建命名空间
createNameSpace("test02");
}
/**数据操作*/
@Test
public void testApi01() throws IOException {
//插入数据
// put("stu3","1001","info","name","zs");
// put("stu3","1001","score","Math","95");
// put("stu3","1001","score","chinese","95");
// put("stu3","1002","info","name","ls");
// put("stu3","1002","score","Math","77");
// put("stu3","1003","info","name","ww");
//get数据
// get("stu3","1001","info","name");
//scan数据
// scan("stu3",null,"score","Math");
//delete
delete("stu3","1008","info","name");
}
/**
* 判断表是否存在
*/
public Boolean isExit(String name) throws IOException {
return admin.tableExists(TableName.valueOf(name));
}
/**
* 创建表
*/
public void createTable(String name,String... columnFamily) throws IOException {
while (isExit(name)){
System.out.println("表已存在");
}
//表名
HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(name));
//列簇
for (String column : columnFamily) {
tableDescriptor.addFamily(new HColumnDescriptor(column));
}
admin.createTable(tableDescriptor);
System.out.println("表创建成功");
}
/**删除表*/
public void deleteTable(String name) throws IOException {
while (! isExit(name)){
System.out.println("表不存在");
}
//先下线
admin.disableTable(TableName.valueOf(name));
admin.deleteTable(TableName.valueOf(name));
System.out.println("表删除成功");
}
/**创建命名空间*/
public void createNameSpace(String name) {
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create(name).build();
try {
admin.createNamespace(namespaceDescriptor);
}
catch (NamespaceExistException e){
System.out.println("命名空间已存在");
}
catch (IOException e) {
e.printStackTrace();
}
}
/**插入数据*/
public void put(String name,String rk,String cf,String cn,String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
Put put = new Put(Bytes.toBytes(rk));
put.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn),Bytes.toBytes(value));
table.put(put);
table.close();
System.out.println("插入成功");
table.close();
}
/**获取数据get*/
public void get(String name,String rk,String cf,String cn) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
Get get = new Get(Bytes.toBytes(rk));
//加上列簇
// get.addFamily(Bytes.toBytes(cf));
//加上列
get.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn));
get.setMaxVersions(2);
Result result = table.get(get);
for (Cell cell : result.rawCells()) {
System.out.println("列簇:"+Bytes.toString(CellUtil.cloneFamily(cell))+",列:"+
Bytes.toString(CellUtil.cloneQualifier(cell))+",value:"+Bytes.toString(CellUtil.cloneValue(cell)));
}
table.close();
}
/**获取数据scan*/
public void scan(String name,String rk,String cf,String cn) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
Scan scan = new Scan();
// Scan scan = new Scan(Bytes.toBytes("1001"),Bytes.toBytes("1003"));
// scan.addFamily(Bytes.toBytes(cf));
scan.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn));
ResultScanner results = table.getScanner(scan);
for (Result result : results) {
for (Cell cell : result.rawCells()) {
System.out.println("rowKey:"+Bytes.toString(CellUtil.cloneRow(cell))+",列簇:"+Bytes.toString(CellUtil.cloneFamily(cell))+",列:"+
Bytes.toString(CellUtil.cloneQualifier(cell))+",value:"+Bytes.toString(CellUtil.cloneValue(cell)));
}
}
table.close();
}
/**删除*/
public void delete(String name,String rk,String cf,String cn) throws IOException {
Table table = connection.getTable(TableName.valueOf(name));
//rowkey删-->会deleteall命令,会将指定rowKey下的所有列族以及所有列的所有版本数据都删除,最终做的标记类型也是DeleteFamily
Delete delete = new Delete(Bytes.toBytes(rk));
// delete.addFamily(Bytes.toBytes(cf));
// delete.addColumns(Bytes.toBytes(cf),Bytes.toBytes(cn),1614844098039L);
delete.addColumn(Bytes.toBytes(cf),Bytes.toBytes(cn),1614844704715L);
table.delete(delete);
table.close();
}
@After
public void close() throws IOException {
admin.close();
connection.close();
}
}
五、MapReduce?
1.介绍 分而治之
Map负责“分”,即把复杂的任务分解为若干个“简单的任务”来并行处理。可以进行拆 分的前提是这些小任务可以并行计算,彼此间几乎没有依赖关系。
Reduce负责“合”,即对map阶段的结果进行全局汇总。