大数据环境搭建(测试)

安装包链接：https://pan.baidu.com/s/1YcIbXq2kwginm91YFm_XLw 密码：bsxu

centos6.5 + mysql5.1.73 + jdk1.7.0 + hadoop2.6.0 + hive2.0.0 + hbase1.2.2 + sqoop1.4.6 + flume1.6.0 + kafka0.8.2 + scala2.11.8 + spark1.5.0 + storm0.10.1 + maven3.2.5

 

准备：
/etc/sysconfig/network  
/etc/hosts   
/etc/sysconfig/selinux
service iptables stop
chkconfig iptables off  
rz 安装包 /usr/local/src


1.安装jdk、hadoop
rpm -ivh jdk-7u71-linux-x64.rpm 
tar -xvzf hadoop-2.6.0.tar.gz -C /usr/local/
export JAVA_HOME=/usr/java/jdk1.7.0_71
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib
export HADOOP_HOME=/usr/local/hadoop-2.6.0
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_HOME/bin


2.配置主机互信
ssh-keygen -t rsa
cat id_rsa.pub >> authorized_keys
chmod 700 ~/.ssh/
ssh master 
ssh localhost


3.修改如下文件内的JAVA_HOME
[root@master hadoop]# vi hadoop-env.sh
[root@master hadoop]# vi yarn-env.sh
[root@master hadoop]# vi mapred-env.sh


4.修改core-site.xml
<configuration>
    <property>
    <name>fs.defaultFS</name>
    <value>hdfs://master:9000</value>
    </property>
     <property>
    <name>hadoop.tmp.dir</name>
    <value>/usr/local/hadoop-2.6.0/tmp</value>
    </property>
</configuration>


5.修改hdfs-site.xml
<configuration>
  <property>
      <name>dfs.replication</name>
      <value>1</value>
  </property>
  <property>
      <name>dfs.namenode.name.dir</name>
      <value>/usr/local/hadoop-2.6.0/dfs/name</value>
  </property>
  <property>
      <name>dfs.datanode.data.dir</name>
      <value>/usr/local/hadoop-2.6.0/dfs/data</value>
  </property>
</configuration>


6.修改mapred-site.xml
<configuration>
  <property>
     <name>mapreduce.framework.name</name>
     <value>yarn</value>
     <final>true</final>
  </property>
</configuration>


7.修改yarn-site.xml
<configuration>
  <property>
     <name>yarn.resourcemanager.hostname</name>
     <value>master</value>
  </property>
  <property>
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce_shuffle</value>
  </property>
</configuration>


8.格式化hdfs
hadoop namenode -format


9.启动hadoop
start-dfs.sh
jps
http://master:50070/ 


10.启动yarn
start-yarn.sh
jps
http://master:8088/cluster


测试：
echo "1,zhangsan" > text
hadoop fs -mkdir -p /data/test
hadoop fs -put text /data/test
hadoop fs -text /data/test/text


1.安装hive
tar -xvzf apache-hive-2.0.0-bin.tar.gz -C /usr/local/
export HIVE_HOME=/usr/local/hive-2.0.0/
export PATH=$PATH:$HIVE_HOME/bin


2.创建hive-site.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/user/hive/warehouse</value>
    <description>location of default database for the warehouse</description>
</property>
<property>
   <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true</value>
   <description>JDBC connect string for a JDBC metastore</description>
</property>
<property>
  <name>javax.jdo.option.ConnectionDriverName</name>
  <value>com.mysql.jdbc.Driver</value>
  <description>Driver class name for a JDBC metastore</description>
</property>
<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>root</value>
  <description>username to use against metastore database</description>
</property>
<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>password</value>
  <description>password to use against metastore database</description>
</property>
</configuration>


3.拷贝mysql驱动到lib目录


4.元数据初始化
schematool -initSchema -dbType mysql


5.启动hive


测试：
create table test (id int,name string) row format delimited fields terminated by ',';
load data inpath '/data/test/text' overwrite into table test;
select * from test;
hadoop fs -ls /user/hive/warehouse
hadoop fs -cat /user/hive/warehouse/test/text


1.安装hbase
tar -xvzf hbase-1.2.2-bin.tar.gz -C /usr/local/
export HBASE_HOME=/usr/local/hbase-1.2.2
export PATH=$PATH:$HBASE_HOME/bin


2.修改hbase-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_71
export HBASE_MANAGES_ZK=true


3.修改hbase-site.xml
<property>
 <name>hbase.rootdir</name>
 <value>hdfs://master:9000/hbase</value>
</property>
<property>
 <name>hbase.cluster.distributed</name>
 <value>true</value>
</property>
<property>
 <name>hbase.zookeeper.quorum</name>
 <value>master</value>
</property>
<property>
 <name>dfs.replication</name>
 <value>1</value>
</property>


4.修改regionservers
localhost


5.启动hbase
start-hbase.sh
jps
http://master:16010


测试：
hbase shell
list


1.安装zookeeper
tar -xvzf zookeeper-3.4.6.tar.gz -C /usr/local/
export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.6
export PATH=$PATH:$ZOOKEEPER_HOME/bin


2.安装目录下创建data、datalog
mkdir data datalog


3.创建新zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/local/zookeeper-3.4.6/data
dataLogDir=/usr/local/zookeeper-3.4.6/datalog
clientPort=2186  
server.1=master:2287:3387
#server.2=master:2288:3388
#server.3=master:2289:3389


3.data下创建myid文件
vi myid  
1


4.启动zookeeper
zkServer.sh start


测试：
zkServer.sh status
zkCli.sh -server master:2186
help
quit


1.安装sqoop
tar -xvzf sqoop-1.4.6-cdh5.5.2.tar.gz -C /usr/local
export SQOOP_HOME=/usr/local/sqoop-1.4.6
export PATH=$PATH:$SQOOP_HOME/bin


2.拷贝mysql jdbc到到sqoop/lib目录下


3.修改sqoop-env.sh
mv sqoop-env-template.sh sqoop-env.sh
export HADOOP_COMMON_HOME=/usr/local/hadoop-2.6.0
export HADOOP_MAPRED_HOME=/usr/local/hadoop-2.6.0
export HBASE_HOME=/usr/local/hbase-1.2.2
export HIVE_HOME=/usr/local/hive-2.0.0
export ZOOCFGDIR=/usr/local/zookeeper-3.4.6


4.修改bin/configure-sqoop
注释掉HCatalog，Accumulo检查


测试：
sqoop version
sqoop list-databases -connect jdbc:mysql://localhost:3306/ --username root -password password


1.安装flume
tar -xvzf apache-flume-1.6.0-bin.tar.gz -C /usr/local/
mv apache-flume-1.6.0-bin/ flume-1.6.0/
export FLUME_HOME=/usr/local/flume-1.6.0
export FLUME_CONF_DIR=/usr/local/flume-1.6.0/conf
export PATH=$PATH:$FLUME_HOME/bin


2.修改配置文件
mv flume-conf.properties.template flume-conf.properties
agent.sources = r1
agent.sinks = s1
agent.channels = c1
agent.sources.r1.channels = c1
agent.sinks.s1.channel = c1
#Describe/configure the source
agent.sources.r1.type = exec
agent.sources.r1.command = tail -F /usr/local/hadoop-2.6.0/logs/hadoop-root-namenode-master.log
#Use a channel which buffers events in memory
agent.channels.c1.type = memory
agent.channels.c1.capacity = 1000
agent.channels.c1.transactionCapacity = 100
agent.sinks.s1.type = logger


mv flume-env.sh.template flume-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_71
export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote"


3.启动flume
cd $FLUME_HOME/bin
flume-ng  agent -c ../conf -f ../conf/flume-conf.properties -Dflume.root.logger=INFO,console -n agent


测试：
tail -F /usr/local/hadoop-2.6.0/logs/hadoop-root-namenode-master.log


1.安装kafka
tar -xvzf kafka_2.10-0.8.2.0.tgz -C /usr/local/
mv kafka_2.10-0.8.2.0/ kafka-0.8.2/
mkdir -p log/kafka
mkdir zookeeper
mkdir -p log/zookeeper


2.修改server.properties
host.name=master
port=9092
log.dirs=/usr/local/kafka-0.8.2/log/kafka
zookeeper.connect=localhost:2182  


3.修改zookeeper.properties
dataDir=/usr/local/kafka-0.8.2/zookeeper
dataLogDir=/usr/local/kafka-0.8.2/log/zookeeper
clientPort=2182
maxClientCnxns=0
tickTime=2000
initLimit=10


4.bin下创建kafka启动脚本
vi kafkastart.sh 
#!/bin/sh
/usr/local/kafka-0.8.2/bin/zookeeper-server-start.sh /usr/local/kafka-0.8.2/config/zookeeper.properties &
sleep 3
/usr/local/kafka-0.8.2/bin/kafka-server-start.sh /usr/local/kafka-0.8.2/config/server.properties &


vi kafkastop.sh
#!/bin/sh
/usr/local/kafka-0.8.2/bin/zookeeper-server-stop.sh /usr/local/kafka-0.8.2/config/zookeeper.properties &
sleep 3
/usr/local/kafka-0.8.2/bin/kafka-server-stop.sh /usr/local/kafka-0.8.2/config/server.properties &


chmod +x kafkastart.sh
chmod +x kafkastop.sh


5.启动kafka
zkServer.sh start
./kafkastart.sh


测试：
./kafka-topics.sh --create --zookeeper localhost:2182 --replication-factor 1 --partitions 1 --topic test &
./kafka-topics.sh --list --zookeeper localhost:2182
./kafka-console-producer.sh --broker-list 10.8.63.7:9092 --topic test    #使用ip，输入内容，回车
./kafka-console-consumer.sh --zookeeper master:2182 --topic test --from-beginning  #另外一个终端观测
./kafkastop.sh
ps -ef | grep kafka   
kill -9 pid


1.安装scala、spark
tar -xvzf scala-2.11.8.tgz -C /usr/local/
export SCALA_HOME=/usr/local/scala-2.11.8
export PATH=$PATH:$SCALA_HOME/bin


tar -xvzf spark-1.5.0-bin-hadoop2-without-hive.tgz -C /usr/local/
mv spark-1.5.0-bin-hadoop2-without-hive/ spark-1.5.0/
export SPARK_HOME=/usr/local/spark-1.5.0
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin


2.修改slaves
mv slaves.template slaves
vi slaves
master


3.修改spark-env.sh
mv spark-env.sh.template spark-env.sh
export HADOOP_HOME=/usr/local/hadoop-2.6.0
export HADOOP_CONF_DIR=/usr/local/hadoop-2.6.0/etc/hadoop
export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop-2.6.0/bin/hadoop classpath)
export JAVA_HOME=/usr/java/jdk1.7.0_71
export SCALA_HOME=/usr/local/scala-2.11.8
export SPARK_MASTER_IP=master
export SPARK_WORKER_MEMORY=512M


4.启动spark
cd sbin
./start-all.sh
jps
http://master:8080/
spark-shell
http://master:4040/


测试：
val testFile = sc.textFile("hdfs://master:9000/data/test/text")
testFile.collect
exit


1.安装storm
tar -xvzf apache-storm-0.10.1.tar.gz -C /usr/local/
mv apache-storm-0.10.1/ storm-0.10.1/
export STORM_HOME=/usr/local/storm-0.10.1
export PATH=$PATH:$STORM_HOME/bin


2.修改storm.yaml
storm.zookeeper.servers:
  - "127.0.0.1"
nimbus.host: "127.0.0.1"
storm.zookeeper.port: 2186
storm.local.dir: "/usr/local/storm-0.10.1"
supervisor.slots.ports:
  - 6700
  - 6701
  - 6702
  - 6703


3.启动strom
storm nimbus &
storm supervisor &
storm ui &
http://master:8080


1.安装maven
tar -xvzf apache-maven-3.2.5-bin.tar.gz -C /usr/local/
mv apache-maven-3.2.5/ maven-3.2.5/
export MAVEN_HOME=/usr/local/maven-3.2.5
export PATH=$PATH:$MAVEN_HOME/bin


测试：
mvn -v






附录：
hadoop端口整理：
50070：hdfs文件管理
8088：ResourceManager
8042：NodeManager
19888：JobHistory(使用“mr-jobhistory-daemon.sh”来启动JobHistory Server)


spark端口整理：
master端口是7077
master webui是8080
spark shell webui端口是4040


问题：
hadoop:
WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
用网盘里编译好的64进制库文件替换hadoop/lib/native目录下的文件


hive:
SLF4J: Class path contains multiple SLF4J bindings.
由jar包冲突引起：mv log4j-slf4j-impl-2.4.1.jar log4j-slf4j-impl-2.4.1.jar.bak  
mv hive-jdbc-2.0.0-standalone.jar hive-jdbc-2.0.0-standalone.jar.bak


spark:
Exception in thread "main" java.lang.NoClassDefFoundError: org/slf4j/Logger
在spark-env.sh里加入export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop-2.6.0/bin/hadoop classpath)即可