Flume及nginx的安装运用

Flume采集数据到hdfs：

1、启动HDFS

start-dfs.sh

2、启动YARN

start-yarn.sh

3、将flume上传到hdp-1的apps中，解压

tar -zxvf flume-1.6.0

4、配置文件
vi /etc/profile

export FLUME_HOME=/root/apps/flume-1.6.0
export PATH=$FLUME_HOME/bin:$PATH

soure /etc/profile

4、在flume-1.6.0/conf新建文本dome.loggeer.properties

# Name the components on this agent
a1.sources = r1   
a1.sinks = k1 
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = spooldir
# 根据需要修改监控日志目录： nginx为：/usr/local/nginx/logs
a1.sources.r1.spoolDir = /usr/local/nginx/logs
a1.sources.r1.fileHeader = true
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = timestamp

# Describe the sink
a1.sinks.k1.type = hdfs
# 收集到hdfs的地址
a1.sinks.k1.hdfs.path = hdfs://hdp-1:9000/output/flume
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.rollInterval = 10
a1.sinks.k1.hdfs.rollSize = 0
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.filePrefix = %Y-%m-%d-%H-%M-%S
a1.sinks.k1.hdfs.useLocalTimeStamp = true

# Use a channel which buffers events in file
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /home/hadoop/flume/checkpoint
a1.channels.c1.dataDirs = /home/hadoop/flume/data

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

5、根目录中新建flume包: flume下makeflume.sh

 [root@hdp-1 flume]#cd flume
[root@hdp-1 flume]# ll
总用量 4
-rwxr-xr-x. 1 root root 58 10月 10 11:52 makeflume.sh

makeflume.sh存储数据脚本:

while ture
do
echo 'date' >> access.flume
slpee  0.1
done

6、启动flume

$FLUME_HOME/bin/flume-ng agent \-c conf \-n a1 \-f $FLUME_HOME/conf/dome.logger.properties \-Dflume.root.logger=DEBUG,console

7、查看日志是否被收集

cd /usr/local/nginx/logs/

8、查看HDFS响应的目录

hadoop fs -ls /

nginx安装配置及运用

1、将web项目（作用：产生日志）重新编译（compile）成功后，打成jar包（package），上传到hdp-1，并运行jar包，

cd apps/

java -jar demo-0.0.1-SNAPSHOT.jar

2、安装make

yum -y install gcc automake autoconf libtool make

安装g++

yum install gcc gcc-c++

安装openssl

yum -y install openssl openssl-devel

安装PCRE库到apps

运行安装pcre-8.39

./configure
make
make install

安装zlib库到apps

运行安装zlib-1.2.11

./configure
make
make install

安装nginx到apps，与上同理，运行安装

3、启动nginx

cd /usr/local/nginx/sbin
./nginx

在浏览器中输入你的节点的ip地址，查看是否启动成功
停止

./nginx -s stop  
./nginx -s quit

./nginx -s quit:此方式停止步骤是待nginx进程处理任务完毕进行停止。
./nginx -s stop:此方式相当于先查出nginx进程id再使用kill命令强制杀掉进程。

重新加载

./nginx -s reload

4、修改配置文件
vi /apps/nginx-1.1.10/conf/nginx.conf

 
#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;
#将main放出
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;
    upstream frame-tomcat{
      server 127.0.0.1:1997;#修改
}
    server {
        listen       80;
        server_name  hdp-1;#修改

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
}
server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        access_log  logs/log.frame.access.log  main;

        location / {
            # root   html;
            # index  index.html index.htm;
            proxy_pass http://frame-tomcat;
        }

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;#修改
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443;
    #    server_name  localhost;

    #    ssl                  on;
    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_timeout  5m;

    #    ssl_protocols  SSLv2 SSLv3 TLSv1;
    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers   on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

在hdp-4上也启动运行jar包，安装nginx，浏览器搜索两个地址就会访问到同一个网页，nginx起的作用是转发作用，一个是tomcat服务器，一个是nginx服务器