hadoop HDFS命令

创建与查看hdfs目录

shm@master:~$ hadoop fs -mkdir /user
shm@master:~$ hadoop fs -mkdir /user/shm
shm@master:~$ hadoop fs -mkdir /user/shm/test
shm@master:~$ hadoop fs -ls
Found 1 items
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 test
shm@master:~$ hadoop fs -ls /
Found 1 items
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user
shm@master:~$ hadoop fs -ls -R
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 test
shm@master:~$ hadoop fs -ls -R /
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user/shm
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user/shm/test

shm@master:~$ hadoop fs -mkdir -p /dir1/dir2/dir3
shm@master:~$ hadoop fs -ls -R /
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:00 /dir1
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:00 /dir1/dir2
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:00 /dir1/dir2/dir3
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user/shm
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user/shm/test

从本地计算机复制文件到hdfs

shm@master:~$ hadoop fs -copyFromLocal /usr/local/hadoop/README.txt /user/shm/test
17/05/01 16:06:58 WARN hdfs.DFSClient: Slow waitForAckedSeqno took 86903ms (threshold=30000ms)
shm@master:~$ hadoop fs -copyFromLocal /usr/local/hadoop/README.txt /user/shm/test/test1.txt

shm@master:~$ hadoop fs -ls -R /
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:00 /dir1
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:00 /dir1/dir2
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:00 /dir1/dir2/dir3
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user
drwxr-xr-x   - shm supergroup          0 2017-05-01 15:52 /user/shm
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:07 /user/shm/test
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt

shm@master:~$ hadoop fs -ls /user/shm/test
Found 2 items
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt

//查看文件内容

shm@master:~$ hadoop fs -cat /user/shm/test/README.txt

//分页显示

shm@master:~$ hadoop fs -cat /user/shm/test/README.txt|more

//强制复制

shm@master:~$ hadoop fs -copyFromLocal /usr/local/hadoop/README.txt /user/shm/test
copyFromLocal: `/user/shm/test/README.txt': File exists
shm@master:~$ hadoop fs -copyFromLocal -f /usr/local/hadoop/README.txt /user/shm/test
shm@master:~$ hadoop fs -ls /user/shm/test
Found 2 items
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:16 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt

//一次复制多个本地文件到Hdfs

shm@master:~$ hadoop fs -copyFromLocal /usr/local/hadoop/NOTICE.txt /usr/local/hadoop/LICENSE.txt /user/shm/test
shm@master:~$ hadoop fs -ls /user/shm/test
Found 4 items
-rw-r--r--   3 shm supergroup      15429 2017-05-01 16:18 /user/shm/test/LICENSE.txt
-rw-r--r--   3 shm supergroup        101 2017-05-01 16:18 /user/shm/test/NOTICE.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:16 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt

//复制整个本地文件目录到hdfs

shm@master:~$ hadoop fs -copyFromLocal /usr/local/hadoop/etc /user/shm/test
shm@master:~$ hadoop fs -ls /user/shm/test
Found 5 items
-rw-r--r--   3 shm supergroup      15429 2017-05-01 16:18 /user/shm/test/LICENSE.txt
-rw-r--r--   3 shm supergroup        101 2017-05-01 16:18 /user/shm/test/NOTICE.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:16 /user/shm/test/README.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:21 /user/shm/test/etc
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt

shm@master:~$ hadoop fs -ls /user/shm/test/etc
Found 1 items
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:21 /user/shm/test/etc/hadoop
shm@master:~$ hadoop fs -ls -R /user/shm/test/etc
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:21 /user/shm/test/etc/hadoop
-rw-r--r--   3 shm supergroup       4436 2017-05-01 16:21 /user/shm/test/etc/hadoop/capacity-scheduler.xml
-rw-r--r--   3 shm supergroup       1335 2017-05-01 16:21 /user/shm/test/etc/hadoop/configuration.xsl
-rw-r--r--   3 shm supergroup        318 2017-05-01 16:21 /user/shm/test/etc/hadoop/container-executor.cfg
-rw-r--r--   3 shm supergroup        862 2017-05-01 16:21 /user/shm/test/etc/hadoop/core-site.xml
-rw-r--r--   3 shm supergroup       3670 2017-05-01 16:21 /user/shm/test/etc/hadoop/hadoop-env.cmd
....

//put复制文件，不管文件是否存在，会直接覆盖原文件，而不提示

shm@master:~$ hadoop fs -put /usr/local/hadoop/README.txt /user/shm/test/test2.txt
//将原本显示在屏幕上的内容存储到hdfs文件

shm@master:~$ echo hello hadoop | hadoop fs -put - /user/shm/test/echoin.txt
shm@master:~$ hadoop fs -cat /user/shm/test/echoin.txt
hello hadoop

//将本地目录列表存储到hdfs

shm@master:~$ ls /usr/local/hadoop | hadoop fs -put - /user/shm/test/hadooplist.txt
shm@master:~$ hadoop fs -cat /user/shm/test/hadooplist.txt
bin
etc
hadoop_data
include
lib
libexec
LICENSE.txt
logs
NOTICE.txt
README.txt
sbin
share

=====将hdfs文件复制到本地计算机==========

shm@master:~$ mkdir test
shm@master:~$ ls
Desktop    Downloads         hadoop-2.6.0.tar.gz  Pictures  Templates  Videos
Documents  examples.desktop  Music                Public    test
shm@master:~$ cd test

//复制文件

shm@master:~/test$ hadoop fs -copyToLocal /user/shm/test/hadooplist.txt
shm@master:~/test$ ll
total 12
drwxrwxr-x  2 shm shm 4096 May  1 16:35 ./
drwx------ 22 shm shm 4096 May  1 16:34 ../
-rw-r--r--  1 shm shm   90 May  1 16:35 hadooplist.txt
shm@master:~/test$
//复制目录

shm@master:~/test$ hadoop fs -copyToLocal /user/shm/test/etc
shm@master:~/test$ ll
total 16
drwxrwxr-x  3 shm shm 4096 May  1 16:37 ./
drwx------ 22 shm shm 4096 May  1 16:34 ../
drwxrwxr-x  3 shm shm 4096 May  1 16:37 etc/
-rw-r--r--  1 shm shm   90 May  1 16:35 hadooplist.txt

//get命令复制

shm@master:~/test$ hadoop fs -get /user/shm/test/README.txt localREADME.txt
shm@master:~/test$ ll
total 20
drwxrwxr-x  3 shm shm 4096 May  1 16:38 ./
drwx------ 22 shm shm 4096 May  1 16:34 ../
drwxrwxr-x  3 shm shm 4096 May  1 16:37 etc/
-rw-r--r--  1 shm shm   90 May  1 16:35 hadooplist.txt
-rw-r--r--  1 shm shm 1366 May  1 16:38 localREADME.txt
shm@master:~/test$

======复制与删除hdfs文件===============

shm@master:~$ hadoop fs -mkdir /user/shm/test/tmp

//复制文件到指定目录

shm@master:~$ hadoop fs -cp /user/shm/test/README.txt /user/shm/test/tmp
shm@master:~$ hadoop fs -ls -R /user/shm/test/tmp
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:42 /user/shm/test/tmp/README.txt

//删除文件
shm@master:~$ hadoop fs -ls /user/shm/test
Found 9 items
-rw-r--r--   3 shm supergroup      15429 2017-05-01 16:18 /user/shm/test/LICENSE.txt
-rw-r--r--   3 shm supergroup        101 2017-05-01 16:18 /user/shm/test/NOTICE.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:16 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup         13 2017-05-01 16:29 /user/shm/test/echoin.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:21 /user/shm/test/etc
-rw-r--r--   3 shm supergroup         90 2017-05-01 16:32 /user/shm/test/hadooplist.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:26 /user/shm/test/test2.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:42 /user/shm/test/tmp
shm@master:~$ hadoop fs -rm /user/shm/test/test2.txt
17/05/01 16:45:02 INFO fs.TrashPolicyDefault: Namenode trash configuration: Deletion interval = 0 minutes, Emptier interval = 0 minutes.
Deleted /user/shm/test/test2.txt
shm@master:~$ hadoop fs -ls /user/shm/test
Found 8 items
-rw-r--r--   3 shm supergroup      15429 2017-05-01 16:18 /user/shm/test/LICENSE.txt
-rw-r--r--   3 shm supergroup        101 2017-05-01 16:18 /user/shm/test/NOTICE.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:16 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup         13 2017-05-01 16:29 /user/shm/test/echoin.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:21 /user/shm/test/etc
-rw-r--r--   3 shm supergroup         90 2017-05-01 16:32 /user/shm/test/hadooplist.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:42 /user/shm/test/tmp
shm@master:~$

//删除目录

shm@master:~$ hadoop fs -ls /user/shm/test
Found 8 items
-rw-r--r--   3 shm supergroup      15429 2017-05-01 16:18 /user/shm/test/LICENSE.txt
-rw-r--r--   3 shm supergroup        101 2017-05-01 16:18 /user/shm/test/NOTICE.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:16 /user/shm/test/README.txt
-rw-r--r--   3 shm supergroup         13 2017-05-01 16:29 /user/shm/test/echoin.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:21 /user/shm/test/etc
-rw-r--r--   3 shm supergroup         90 2017-05-01 16:32 /user/shm/test/hadooplist.txt
-rw-r--r--   3 shm supergroup       1366 2017-05-01 16:07 /user/shm/test/test1.txt
drwxr-xr-x   - shm supergroup          0 2017-05-01 16:42 /user/shm/test/tmp

shm@master:~$ hadoop fs -rm -R /user/shm/test/etc
17/05/01 16:48:44 INFO fs.TrashPolicyDefault: Namenode trash configuration: Deletion interval = 0 minutes, Emptier interval = 0 minutes.
Deleted /user/shm/test/etc