使用Python在Ubuntu14.04上部署Hadoop2.7.7

使用Python在Ubuntu14.04上部署Hadoop

coding：utf-8

import os
import time

hadoop_environment_set = ‘’’#HADOOP VARIABLES START
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME=$HADOO{P}_{H}OMEexportHADOO{P}_{C}OMMO{N}_{H}OME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOO{P}_{H}OMEexportYAR{N}_{H}OME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOO{P}_{H}OME/lib/nativeexportHADOO{P}_{O}PTS="-Djava.library.path=$HADOOP_HOME/lib"
export JAVA_LIBRARY_PATH=$HADOO{P}_{H}OME/lib/native:$JAVA_LIBRARY_PATH
#HADOOP VARIABLES END’’’
username = ‘hadoop’

class install_and_set_hadoop():
“”“安装配置Hadoop”""

def update_linux(self):
    '''更新Linux'''
    os.system('sudo apt-get update')

def create_hadoop_user(self):
    # 创建Hadoop用户
    command_list = ['sudo useradd %s' % username,
                    'sudo chown %s /home/%s' % (username, username),
                    'sudo passwd %s' % username,
                    'aaa123',
                    'aaa123',
                    'sudo adduser %s sudo' % username,
                    ]

    for i in command_list:
        os.system(i)

def install_ssh(self):
    command_list = ['sudo apt-get install openssh-server',
                    'Y',
                    "ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa",
                    'cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys'
                    ]
    for i in command_list:
        os.system(i)

def install_JDK(self):
    if os.path.exists("/usr/lib/jvm/java-7-openjdk-amd64"):
        print('The JAVA is exist.')
    else:
        command_list = ['sudo apt-get install openjdk-7-jdk',

                        ]

def download_and_install_hadoop(self):

    with open('./temporary.txt', 'w', encoding='utf-8')as f:
        f.write(hadoop_environment_set)

    command_list = ['wget http://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz'
                    'sudo tar -zxvf hadoop-2.7.7.tar.gz',  # 解压
                    'sudo mv hadoop-2.7.7 /usr/local/hadoop',
                    'sudo chown -R %s:%s /usr/local/hadoop' % (username,username),
                    '/usr/local/hadoop/bin/hadoop',
                    'sudo cat temporary.txt >> ~/.bashrc',
                    'source ~/.bashrc'
                    ]

    for i in command_list:
        os.system(i)

def set_hadoop(self):
    # 1.core-site.xml
    if os.path.exists('/usr/local/hadoop/etc/hadoop/core-site.xml'):
        with open('/usr/local/hadoop/etc/hadoop/core-site.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>',
                      '''<configuration>\n<property>\n<name>fs.default.name</name>\n<value>hdfs://localhost:9000</value>\n</property>\n''')

        with open('/usr/local/hadoop/etc/hadoop/core-site.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    # 2.hdfs-site.xml
    os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/namenode')
    os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/datanode')
    if os.path.exists('/usr/local/hadoop/etc/hadoop/hdfs-site.xml'):
        with open('/usr/local/hadoop/etc/hadoop/hdfs-site.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>',
                      '''<configuration>\n<property>\n<name>dfs.namenode.name.dir</name>\n<value>file:/usr/local/hadoop/hadoop_data/hdfs/namenode</value>\n</property>\n<property>\n<name>dfs.datanode.data.dir</name>\n<value>file:/usr/local/hadoop/hadoop_data/hdfs/datanode</value>\n</property>\n<property>\n<name>dfs.replication</name>\n<value>2</value>\n</property> ''')
        with open('/usr/local/hadoop/etc/hadoop/hdfs-site.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    # 3.mapred-site.xml
    os.system('sudo cp etc/hadoop/mapred-site.xml.template  etc/hadoop/mapred-site.xml')
    time.sleep(0.1)
    if os.path.exists('/usr/local/hadoop/etc/hadoop/mapred-site.xml'):
        with open('/usr/local/hadoop/etc/hadoop/mapred-site.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>',
                      '''<configuration>\n<property>\n<name>mapreduce.framework.name</name>     \n	<value>yarn</value>\n</property>''')

        with open('/usr/local/hadoop/etc/hadoop/mapred-site.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    # 4.hdfs-yarn.xml
    if os.path.exists('/usr/local/hadoop/etc/hadoop/hdfs-yarn.xml'):
        with open('/usr/local/hadoop/etc/hadoop/hdfs-yarn.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>', '''<configuration>\n<property>

yarn.nodemanager.aux-services
mapreduce_shuffle


yarn.nodemanager.aux-services.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler
‘’’)

        with open('/usr/local/hadoop/etc/hadoop/hdfs-yarn.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    os.system('chown -R %s:%s /usr/local/hadoop/hadoop_data/hdfs/*'%(username))
    os.system('sudo chown 777 /usr/local/hadoop/hadoop_data/hdfs/datanode')
    # # 最后：创建NameNode、DataNode目录
    # os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/namenode')
    # os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/datanode')
    # sudo chown 777 /usr/local/hadoop/hadoop_data/hdfs/datanode
    # chown -R hadoop:hadoop /usr/local/hadoop/hadoop_data/hdfs/*

def install(self):
    self.update_linux()
    self.create_hadoop_user()
    self.install_ssh()
    self.install_JDK()
    self.download_and_install_hadoop()
    self.set_hadoop()

install_and_set_hadoop().install()

‘’’
1.要输密码说明权限不够，/usr/local/hadoop已经设置过权限
好像是这两个文件夹的权限设置问题
sudo chown 777 /usr/local/hadoop/hadoop_data/hdfs/datanode
chown -R hadoop:hadoop /usr/local/hadoop/hadoop_data/hdfs/*
然后执行/usr/local/hadoop/sbin/start-all.sh
‘’’