Arrow 之 Java 之 Field Create

AvroToArrowUtils

arrow/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java

  private static Field avroSchemaToField(
      Schema schema,
      String name,
      AvroToArrowConfig config,
      Map<String, String> externalProps) {

    final Type type = schema.getType();
    final LogicalType logicalType = schema.getLogicalType();
    final List<Field> children = new ArrayList<>();
    final FieldType fieldType;

    switch (type) {
      case UNION:
        for (int i = 0; i < schema.getTypes().size(); i++) {
          Schema childSchema = schema.getTypes().get(i);
          // Union child vector should use default name
          children.add(avroSchemaToField(childSchema, null, config));
        }
        fieldType = createFieldType(new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps);
        break;
      case ARRAY:
        Schema elementSchema = schema.getElementType();
        children.add(avroSchemaToField(elementSchema, elementSchema.getName(), config));
        fieldType = createFieldType(new ArrowType.List(), schema, externalProps);
        break;
      case MAP:
        // MapVector internal struct field and key field should be non-nullable
        FieldType keyFieldType = new FieldType(/*nullable=*/false, new ArrowType.Utf8(), /*dictionary=*/null);
        Field keyField = new Field("key", keyFieldType, /*children=*/null);
        Field valueField = avroSchemaToField(schema.getValueType(), "value", config);

        FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), /*dictionary=*/null);
        Field structField = new Field("internal", structFieldType, Arrays.asList(keyField, valueField));
        children.add(structField);
        fieldType = createFieldType(new ArrowType.Map(/*keySorted=*/false), schema, externalProps);
        break;
      case RECORD:
        final Set<String> skipFieldNames = config.getSkipFieldNames();
        for (int i = 0; i < schema.getFields().size(); i++) {
          final Schema.Field field = schema.getFields().get(i);
          Schema childSchema = field.schema();
          String fullChildName = String.format("%s.%s", name, field.name());
          if (!skipFieldNames.contains(fullChildName)) {
            final Map<String, String> extProps = new HashMap<>();
            String doc = field.doc();
            Set<String> aliases = field.aliases();
            if (doc != null) {
              extProps.put("doc", doc);
            }
            if (aliases != null) {
              extProps.put("aliases", convertAliases(aliases));
            }
            children.add(avroSchemaToField(childSchema, fullChildName, config, extProps));
          }
        }
        fieldType = createFieldType(new ArrowType.Struct(), schema, externalProps);
        break;
      case ENUM:
        DictionaryProvider.MapDictionaryProvider provider = config.getProvider();
        int current = provider.getDictionaryIds().size();
        int enumCount = schema.getEnumSymbols().size();
        ArrowType.Int indexType = DictionaryEncoder.getIndexType(enumCount);

        fieldType = createFieldType(indexType, schema, externalProps,
            new DictionaryEncoding(current, /*ordered=*/false, /*indexType=*/indexType));
        break;

      case STRING:
        fieldType = createFieldType(new ArrowType.Utf8(), schema, externalProps);
        break;
      case FIXED:
        final ArrowType fixedArrowType;
        if (logicalType instanceof LogicalTypes.Decimal) {
          fixedArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType);
        } else {
          fixedArrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize());
        }
        fieldType = createFieldType(fixedArrowType, schema, externalProps);
        break;
      case INT:
        final ArrowType intArrowType;
        if (logicalType instanceof LogicalTypes.Date) {
          intArrowType = new ArrowType.Date(DateUnit.DAY);
        } else if (logicalType instanceof LogicalTypes.TimeMillis) {
          intArrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32);
        } else {
          intArrowType = new ArrowType.Int(32, /*signed=*/true);
        }
        fieldType = createFieldType(intArrowType, schema, externalProps);
        break;
      case BOOLEAN:
        fieldType = createFieldType(new ArrowType.Bool(), schema, externalProps);
        break;
      case LONG:
        final ArrowType longArrowType;
        if (logicalType instanceof LogicalTypes.TimeMicros) {
          longArrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64);
        } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
          longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
        } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
          longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null);
        } else {
          longArrowType = new ArrowType.Int(64, /*signed=*/true);
        }
        fieldType = createFieldType(longArrowType, schema, externalProps);
        break;
      case FLOAT:
        fieldType = createFieldType(new ArrowType.FloatingPoint(SINGLE), schema, externalProps);
        break;
      case DOUBLE:
        fieldType = createFieldType(new ArrowType.FloatingPoint(DOUBLE), schema, externalProps);
        break;
      case BYTES:
        final ArrowType bytesArrowType;
        if (logicalType instanceof LogicalTypes.Decimal) {
          bytesArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType);
        } else {
          bytesArrowType = new ArrowType.Binary();
        }
        fieldType = createFieldType(bytesArrowType, schema, externalProps);
        break;
      case NULL:
        fieldType = createFieldType(ArrowType.Null.INSTANCE, schema, externalProps);
        break;
      default:
        // no-op, shouldn't get here
        throw new UnsupportedOperationException();
    }

    if (name == null) {
      name = getDefaultFieldName(fieldType.getType());
    }
    return new Field(name, fieldType, children.size() == 0 ? null : children);
  }

FlightSqlProducer

arrow/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlProducer.java

/**
   * Default schema templates for the {@link FlightSqlProducer}.
   */
  final class Schemas {
    public static final Schema GET_TABLES_SCHEMA = new Schema(asList(
        Field.nullable("catalog_name", VARCHAR.getType()),
        Field.nullable("db_schema_name", VARCHAR.getType()),
        Field.notNullable("table_name", VARCHAR.getType()),
        Field.notNullable("table_type", VARCHAR.getType()),
        Field.notNullable("table_schema", MinorType.VARBINARY.getType())));
    public static final Schema GET_TABLES_SCHEMA_NO_SCHEMA = new Schema(asList(
        Field.nullable("catalog_name", VARCHAR.getType()),
        Field.nullable("db_schema_name", VARCHAR.getType()),
        Field.notNullable("table_name", VARCHAR.getType()),
        Field.notNullable("table_type", VARCHAR.getType())));
    public static final Schema GET_CATALOGS_SCHEMA = new Schema(
        singletonList(Field.notNullable("catalog_name", VARCHAR.getType())));
    public static final Schema GET_TABLE_TYPES_SCHEMA =
        new Schema(singletonList(Field.notNullable("table_type", VARCHAR.getType())));
    public static final Schema GET_SCHEMAS_SCHEMA =
        new Schema(asList(
            Field.nullable("catalog_name", VARCHAR.getType()),
            Field.notNullable("db_schema_name", VARCHAR.getType())));
    private static final Schema GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA =
        new Schema(asList(
            Field.nullable("pk_catalog_name", VARCHAR.getType()),
            Field.nullable("pk_db_schema_name", VARCHAR.getType()),
            Field.notNullable("pk_table_name", VARCHAR.getType()),
            Field.notNullable("pk_column_name", VARCHAR.getType()),
            Field.nullable("fk_catalog_name", VARCHAR.getType()),
            Field.nullable("fk_db_schema_name", VARCHAR.getType()),
            Field.notNullable("fk_table_name", VARCHAR.getType()),
            Field.notNullable("fk_column_name", VARCHAR.getType()),
            Field.notNullable("key_sequence", INT.getType()),
            Field.nullable("fk_key_name", VARCHAR.getType()),
            Field.nullable("pk_key_name", VARCHAR.getType()),
            Field.notNullable("update_rule", MinorType.UINT1.getType()),
            Field.notNullable("delete_rule", MinorType.UINT1.getType())));
    public static final Schema GET_IMPORTED_KEYS_SCHEMA = GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA;
    public static final Schema GET_EXPORTED_KEYS_SCHEMA = GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA;
    public static final Schema GET_CROSS_REFERENCE_SCHEMA = GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA;
    private static final List<Field> GET_SQL_INFO_DENSE_UNION_SCHEMA_FIELDS = asList(
        Field.notNullable("string_value", VARCHAR.getType()),
        Field.notNullable("bool_value", BIT.getType()),
        Field.notNullable("bigint_value", BIGINT.getType()),
        Field.notNullable("int32_bitmask", INT.getType()),
        new Field(
            "string_list", FieldType.notNullable(LIST.getType()),
            singletonList(Field.nullable("item", VARCHAR.getType()))),
        new Field(
            "int32_to_int32_list_map", FieldType.notNullable(new ArrowType.Map(false)),
            singletonList(new Field(DATA_VECTOR_NAME, new FieldType(false, STRUCT.getType(), null),
                ImmutableList.of(
                    Field.notNullable(KEY_NAME, INT.getType()),
                    new Field(
                        VALUE_NAME, FieldType.nullable(LIST.getType()),
                        singletonList(Field.nullable("item", INT.getType()))))))));
    public static final Schema GET_SQL_INFO_SCHEMA =
        new Schema(asList(
            Field.notNullable("info_name", UINT4.getType()),
            new Field("value",
                FieldType.notNullable(
                    new Union(UnionMode.Dense, range(0, GET_SQL_INFO_DENSE_UNION_SCHEMA_FIELDS.size()).toArray())),
                GET_SQL_INFO_DENSE_UNION_SCHEMA_FIELDS)));
    public static final Schema GET_PRIMARY_KEYS_SCHEMA =
        new Schema(asList(
            Field.nullable("catalog_name", VARCHAR.getType()),
            Field.nullable("db_schema_name", VARCHAR.getType()),
            Field.notNullable("table_name", VARCHAR.getType()),
            Field.notNullable("column_name", VARCHAR.getType()),
            Field.notNullable("key_sequence", INT.getType()),
            Field.nullable("key_name", VARCHAR.getType())));

    private Schemas() {
      // Prevent instantiation.
    }
  }