python 读取 hdfs 数据

 使用python hdfs库 进行访问hdfs

import pandas as pd
from hdfs import InsecureClient
import os

hdfs_ip写入自己的hdfs namenode主机的ip，端口号自己修改，用户名自己修改

client_hdfs = InsecureClient('http://hdfs_ip:50070', user='my_user')

读取hdfs数据

with client_hdfs.read('/user/hdfs/wiki/helloworld.csv', encoding = 'utf-8') as reader:
  df = pd.read_csv(reader,index_col=0)

写入hdfs数据

// ====write file
liste_hello = ['hello1','hello2']
liste_world = ['world1','world2']
df = pd.DataFrame(data = {'hello' : liste_hello, 'world': liste_world})

# To write a Dataframe to HDFS.
with client_hdfs.write('/user/hdfs/wiki/helloworld.csv', encoding = 'utf-8') as writer:
  df.to_csv(writer)