Avro通过schema写.avro文件

最新推荐文章于 2022-06-10 18:01:52 发布

地球人是我哈

最新推荐文章于 2022-06-10 18:01:52 发布

阅读量598

点赞数

分类专栏： hadoop 文章标签： hadoop avro

本文链接：https://blog.csdn.net/weixin_44865574/article/details/105419875

版权

hadoop 专栏收录该内容

7 篇文章

订阅专栏

导入依赖

<dependency>
	<groupId>org.apache.avro</groupId>
	<artifactId>avro</artifactId>
	<version>1.9.2</version>
</dependency>

构造scheam

{
    \"type\": \"record\",
    \"name\": \"hudi_rowkey_test_record\",
    \"namespace\": \"hoodie.hudi_rowkey_test\",
    \"fields\": [
        {
            \"name\": \"id\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"fk_id\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"lmdm\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"sldw\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"ugsa\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"kspc\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"siii\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"isdeleted\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"lastupdatedttm\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        },
        {
            \"name\": \"rowkey\",
            \"type\": [
                \"string\",
                \"null\"
            ]
        }
    ]
}";

代码写入

private static String schemaFile2 = "{\"type\":\"record\",\"name\":\"hudi_rowkey_test_record\",\"namespace\":\"hoodie.hudi_rowkey_test\",\"fields\":[{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"fk_id\",\"type\":[\"string\",\"null\"]},{\"name\":\"lmdm\",\"type\":[\"string\",\"null\"]},{\"name\":\"sldw\",\"type\":[\"string\",\"null\"]},{\"name\":\"ugsa\",\"type\":[\"string\",\"null\"]},{\"name\":\"kspc\",\"type\":[\"string\",\"null\"]},{\"name\":\"siii\",\"type\":[\"string\",\"null\"]},{\"name\":\"isdeleted\",\"type\":[\"string\",\"null\"]},{\"name\":\"lastupdatedttm\",\"type\":[\"string\",\"null\"]},{\"name\":\"rowkey\",\"type\":[\"string\",\"null\"]},{\"name\":\"compsiteKey\",\"type\":[\"string\",\"null\"]}]}";

    public static void main(String[] args) {
        Schema schema = new Schema.Parser().parse(schemaFile);

        int size = new Schema.Parser().parse(schemaFile).getFields().size();
        //2. scheam 新添加一列
        Schema type = Schema.createUnion(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.NULL));
        Schema.Field field = new Schema.Field("compsitekey", type, "", "");
        schema.getFields().add(size, field);

        //Schema.Field field = schema.getFields().get(10);
        GenericRecord my = new GenericData.Record(new Schema.Parser().parse(schema.toString()));
        my.put("id", "xxxx");
        my.put("fk_id", "xxxx");
        my.put("lmdm", "xxxx");
        my.put("sldw", "xxxx");
        my.put("ugsa", "xxxx");
        my.put("kspc", "xxxx");
        my.put("siii", "xxxx");
        my.put("isdeleted", "xxxx");
        my.put("lastupdatedttm", "xxxx");
        my.put("rowkey", "xxxx");
        my.put("compsitekey", "xxxx");

        System.out.println("---------------------------");


    }