hadoop cdh 安装笔记

最新推荐文章于 2024-04-25 19:41:00 发布

okie-dokie

最新推荐文章于 2024-04-25 19:41:00 发布

阅读量116

点赞数

分类专栏： hadoop 文章标签：运维大数据数据库

hadoop 专栏收录该内容

13 篇文章 0 订阅

订阅专栏

https://ccp.cloudera.com/display/CDHDOC/CDH3+Installation

https://ccp.cloudera.com/display/CDHDOC/HBase+Installation

https://ccp.cloudera.com/display/CDHDOC/ZooKeeper+Installation

https://ccp.cloudera.com/display/CDHDOC/CDH3+Deployment+on+a+Cluster

-install

http://archive.cloudera.com/redhat/cdh/cdh3-repository-1.0-1.noarch.rpm

http://archive.cloudera.com/redhat/6/x86_64/cdh/cdh3-repository-1.0-1.noarch.rpm

rpm -ivh cdh3-repository-1.0-1.noarch.rpm

yum --nogpgcheck localinstall cdh3-repository-1.0-1.noarch.rpm

rpm --import http://archive.cloudera.com/redhat/cdh/RPM-GPG-KEY-cloudera

--hadoop

yum search hadoop

yum install hadoop-0.20 hadoop-0.20-native

yum install hadoop-0.20-<daemon type> #namenode|datanode|secondarynamenode|jobtracker|tasktracker

--hbase

yum install hadoop-hbase

yum install hadoop-hbase-master

yum install hadoop-hbase-regionserver

yum install hadoop-hbase-thrift

yum install hadoop-hbase-rest

--zookeeper

yum install hadoop-zookeeper

yum install hadoop-zookeeper-server

-config

--hadoop

alternatives --display hadoop-0.20-conf

cp -r /etc/hadoop-0.20/conf.empty /etc/hadoop-0.20/conf.cluster

alternatives --install /etc/hadoop-0.20/conf hadoop-0.20-conf /etc/hadoop-0.20/conf.cluster 50

alternatives --set hadoop-0.20-conf /etc/hadoop-0.20/conf.cluster

cd /data/hadoop/

mkdir dfs && chown hdfs:hadoop dfs

mkdir mapred && chown mapred:hadoop mapred

chmod 755 dfs mapred

hadoop fs -mkdir /data/hadoop/temp

hadoop fs -mkdir /mapred/system

hadoop fs -chown mapred:hadoop /mapred/system

core-site.xml:

<name>fs.default.name</name>

<value>hdfs://namenode:9000</value>

</property>

<name>hadoop.tmp.dir</name>

<value>/data/hadoop-${user.name}/</value>

</property>

hdfs-site.xml:

<value>${hadoop.tmp.dir}/dfs/data</value>

</property>

<name>dfs.block.size</name>

</property>

mapred-site.xml:

<name>mapred.job.tracker</name>

<value>jobtracker:9001</value>

</property>

<name>mapred.child.java.opts</name>

<value>-Dfile.encoding=utf-8 -Duser.language=zh -Xmx512m</value>

</property>

<name>mapred.system.dir</name>

<value>/mapred/system</value>

</property>

--zookeeper

#!/bin/sh

ZOO=/usr/lib/zookeeper

java -cp $ZOO/zookeeper.jar:$ZOO/lib/log4j-1.2.15.jar:$ZOO/conf:$ZOO/lib/jline-0.9.94.jar org.apache.zookeeper.ZooKeeperMain -server $1:2181

crontab -e

15 * * * * java -cp $classpath:/usr/lib/zookeeper/lib/log4j-1.2.15.jar:/usr/lib/zookeeper/lib/jline-0.9.94.jar:/usr/lib/zookeeper/zookeeper.jar:/usr/lib/zookeeper/conf org.apache.zookeeper.server.PurgeTxnLog /var/zookeeper/ -n 5

--hbase

hadoop fs -mkdir /hbase

hadoop fs -chown hbase:hbase /hbase

crontab -e

* 10 * * * rm -rf `ls /usr/lib/hbase/logs/ | grep -P 'hbase\-hbase\-.+\.log\.[0-9]{4}\-[0-9]{2}\-[0-9]{2}' | sed -r 's/^(.+)$/\/usr\/lib\/hbase\/logs\/\1/g'` >> /dev/null &

<name>hbase.cluster.distributed</name>

</property>

<name>hbase.rootdir</name>

<value>hdfs://node1:9000/hbase</value>

</property>

<name>hbase.tmp.dir</name>

<value>/data0/hbase</value>

</property>

<name>hbase.zookeeper.quorum</name>

</property>

如果需要限制ip

iptables -F
iptables -A INPUT -i lo -j ACCEPT 
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A INPUT -p icmp -m icmp --icmp-type any -j ACCEPT
iptables -A INPUT -i eth0 -p tcp --dport 22 -j ACCEPT;
iptables -A INPUT -i eth0 -p tcp -s 192.168.1.1 -j ACCEPT
iptables -A INPUT -i eth0 -p tcp -s 192.168.1.2 -j ACCEPT
iptables -A INPUT -i eth0 -p tcp -s 192.168.1.3 -j ACCEPT
iptables -A INPUT -i eth0 -p tcp -s 192.168.1.4 -j ACCEPT
iptables -A INPUT -i eth0 -p tcp -s 192.168.1.5 -j ACCEPT
iptables -A INPUT -j DROP

各服务的端口占用：

4.1.2

-core

<name>fs.defaultFS</name>

</property>

-hdfs

<name>dfs.namenode.name.dir</name> <value>/data0/hadoop/dfs/name,/data1/hadoop/dfs/name,/data2/hadoop/dfs/name,/data3/hadoop/dfs/name,/data4/hadoop/dfs/name,/data5/hadoop/dfs/name</value>

</property>

<name>dfs.datanode.data.dir</name>

<value>/data0/hadoop/dfs/data,/data1/hadoop/dfs/data,/data2/hadoop/dfs/data,/data3/hadoop/dfs/data,/data4/hadoop/dfs/data,/data5/hadoop/dfs/data</value>

</property>

<name>dfs.namenode.checkpoint.dir</name>

<value>/data0/hadoop/dfs/namesecondary,/data1/hadoop/dfs/namesecondary,/data2/hadoop/dfs/namesecondary,/data3/hadoop/dfs/namesecondary,/data4/hadoop/dfs/namesecondary,/data5/hadoop/dfs/namesecondary</value>

</property>

-mapred

<name>mapreduce.framework.name</name>

</property>

-yarn

<name>yarn.nodemanager.aux-services</name>

<value>mapreduce.shuffle</value>

</property>

<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

<value>org.apache.hadoop.mapred.ShuffleHandler</value>

</property>

<name>yarn.log-aggregation-enable</name>

</property>

<description>Classpath for typical applications.</description>

<name>yarn.application.classpath</name>

<value>

$HADOOP_CONF_DIR,

$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,

$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,

$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,

$YARN_HOME/*,$YARN_HOME/lib/*

</value>

</property>

<name>yarn.resourcemanager.resource-tracker.address</name>

</property>

<name>yarn.resourcemanager.address</name>

</property>

<name>yarn.resourcemanager.scheduler.address</name>

</property>

<name>yarn.resourcemanager.admin.address</name>

</property>

<name>yarn.resourcemanager.webapp.address</name>

</property>

<name>yarn.nodemanager.local-dirs</name>

<value>/data0/hadoop/yarn/local,/data1/hadoop/yarn/local</value>

</property>

<name>yarn.nodemanager.log-dirs</name>

<value>/data0/hadoop/yarn/logs,/data1/hadoop/yarn/logs</value>

</property>

<name>yarn.nodemanager.remote-app-log-dir</name>

<value>/var/log/hadoop-yarn/apps</value>

</property>

<name>yarn.app.mapreduce.am.staging-dir</name>

</property>

mkdir /data/hadoop

mkdir /data/hadoop/dfs; chown hdfs:hdfs /data/hadoop/dfs

mkdir /data/hadoop/yarn; chown yarn:yarn /data/hadoop/yarn

hadoop namenode -format

service hadoop-hdfs-namenode start

service hadoop-hdfs-secondarynamenode start

service hadoop-hdfs-datanode start

hadoop fs -mkdir /user/history /var/log/hadoop-yarn /tmp

hadoop fs -chmod 1777 /user/history /tmp

hadoop fs -chown yarn /user/history

hadoop fs -chown yarn:mapred /var/log/hadoop-yarn

service hadoop-yarn-resourcemanager start

service hadoop-yarn-nodemanager start

service hadoop-mapreduce-historyserver start

<name>yarn.web-proxy.address</name>

</property>

service hadoop-yarn-proxyserver start

tickTime=2000

dataDir=/data1/zookeeper

clientPort=2181

server.1=node1:2888:3888

server.2=node2:2888:3888

server.3=node3:2888:3888

service zookeeper-server init --myid=n

service zookeeper-server start

<name>hbase.cluster.distributed</name>

</property>

<name>hbase.rootdir</name>

<value>hdfs://node0:9000/hbase</value>

</property>

<name>hbase.zookeeper.quorum</name>

</property>

hadoop fs -mkdir /hbase

hadoop fs -chmod hbase /hbase

service hbase-master start

service hbase-regionserver start

<name>javax.jdo.option.ConnectionURL</name>

<value>jdbc:mysql://node0/metastore</value>

</property>

<name>javax.jdo.option.ConnectionDriverName</name>

<value>com.mysql.jdbc.Driver</value>

</property>

<name>javax.jdo.option.ConnectionUserName</name>

</property>

<name>javax.jdo.option.ConnectionPassword</name>

<value>mypassword</value>

</property>

<name>datanucleus.autoCreateSchema</name>

<value>false</value>

</property>

<name>datanucleus.fixedDatastore</name>

</property>

<name>hive.metastore.uris</name>

<value>thrift://node0:9083</value>

</property>

<name>hive.support.concurrency</name>

</property>

<name>hive.zookeeper.quorum</name>

</property>

/etc/default/hive-server2

export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce

mysql -u root

CREATE DATABASE metastore;

USE metastore;

SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.9.0.mysql.sql;

CREATE USER 'hive'@'metastorehost' IDENTIFIED BY 'mypassword';

REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'metastorehost';

GRANT SELECT,INSERT,UPDATE,DELETE,LOCK TABLES,EXECUTE ON metastore.* TO 'hive'@'metastorehost';

FLUSH PRIVILEGES;

sudo –u postgres psql

CREATE USER hiveuser WITH PASSWORD 'mypassword';

CREATE DATABASE metastore;

\c metastore

\i /usr/lib/hive/scripts/metastore/upgrade/postgres/hive-schema-0.9.0.postgres.sql

service hive-metastore start

service hive-server2 start

beeline

!connect jdbc:hive2://localhost:10000 metastore mypassword org.apache.hive.jdbc.HiveDriver

okie-dokie

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
hadoop cdh 安装笔记

3https://ccp.cloudera.com/display/CDHDOC/CDH3+Installationhttps://ccp.cloudera.com/display/CDHDOC/HBase+Installationhttps://ccp.cloudera.com/display/CDHDOC/ZooKeeper+Installationhttps://c...
复制链接

扫一扫