如何使用python进行hdfs的操作,如何使用pysaprk进行hdfs操作?
python
from pywebhdfs.webhdfs import PyWebHdfsClient
#Create a PyWebHdfsClient object
hdfs = PyWebHdfsClient(host=‘<namenode_host>’, port=‘<namenode_port>’, user_name=‘<hdfs_username>’)
#Create a directory in HDFS
hdfs.make_dir(‘/path/to/new/directory’)
#List the contents of a directory in HDFS
files = hdfs.list_dir(‘/path/to/existing/directory’)
print(files)
#Upload a file to HDFS
hdfs.create_file(‘/path/to/upload/file.txt’, ‘Hello HDFS!’)
#Download a file from HDFS
data = hdfs.read_file(‘/path/to/existing/file.txt’)
print(data)
pyspark
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName(“Download HDFS File”).getOrCreate()
file_data = spark.read.text(“hdfs://<namenode_host>:/
#打印文件内容
file_data.show()