AvroToArrowUtils
arrow/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java
private static Field avroSchemaToField(
Schema schema,
String name,
AvroToArrowConfig config,
Map<String, String> externalProps) {
final Type type = schema.getType();
final LogicalType logicalType = schema.getLogicalType();
final List<Field> children = new ArrayList<>();
final FieldType fieldType;
switch (type) {
case UNION:
for (int i = 0; i < schema.getTypes().size(); i++) {
Schema childSchema = schema.getTypes().get(i);
// Union child vector should use default name
children.add(avroSchemaToField(childSchema, null, config));
}
fieldType = createFieldType(new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps);
break;
case ARRAY:
Schema elementSchema = schema.getElementType();
children.add(avroSchemaToField(elementSchema, elementSchema.getName(), config));
fieldType = createFieldType(new ArrowType.List(), schema, externalProps);
break;
case MAP:
// MapVector internal struct field and key field should be non-nullable
FieldType keyFieldType = new FieldType(/*nullable=*/false, new ArrowType.Utf8(), /*dictionary=*/null);
Field keyField = new Field("key", keyFieldType, /*children=*/null);
Field valueField = avroSchemaToField(schema.getValueType(), "value", config);
FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), /*dictionary=*/null);
Field structField = new Field("internal", structFieldType, Arrays.asList(keyField, valueField));
children.add(structField);
fieldType = createFieldType(new ArrowType.Map(/*keySorted=*/false), schema, externalProps);
break;
case RECORD:
final Set<String> skipFieldNames = config.getSkipFieldNames();
for (int i = 0; i < schema.getFields().size(); i++) {
final Schema.Field field = schema.getFields().get(i);
Schema childSchema = field.schema();
String fullChildName = String.format("%s.%s", name, field.name());
if (!skipFieldNames.contains(fullChildName)) {
final Map<String, String> extProps = new HashMap<>();
String doc = field.doc();
Set<String> aliases = field.aliases();
if (doc != null) {
extProps.put("doc", doc);
}
if (aliases != null) {
extProps.put("aliases", convertAliases(aliases));
}
children.add(avroSchemaToField(childSchema, fullChildName, config, extProps));
}
}
fieldType = createFieldType(new ArrowType.Struct(), schema, externalProps);
break;
case ENUM:
DictionaryProvider.MapDictionaryProvider provider = config.getProvider();
int current = provider.getDictionaryIds().size();
int enumCount = schema.getEnumSymbols().size();
ArrowType.Int indexType = DictionaryEncoder.getIndexType(enumCount);
fieldType = createFieldType(indexType, schema, externalProps,
new DictionaryEncoding(current, /*ordered=*/false, /*indexType=*/indexType));
break;
case STRING:
fieldType = createFieldType(new ArrowType.Utf8(), schema, externalProps);
break;
case FIXED:
final ArrowType fixedArrowType;
if (logicalType instanceof LogicalTypes.Decimal) {
fixedArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType);
} else {
fixedArrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize());
}
fieldType = createFieldType(fixedArrowType, schema, externalProps);
break;
case INT:
final ArrowType intArrowType;
if (logicalType instanceof LogicalTypes.Date) {
intArrowType = new ArrowType.Date(DateUnit.DAY);
} else if (logicalType instanceof LogicalTypes.TimeMillis) {
intArrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32);
} else {
intArrowType = new ArrowType.Int(32, /*signed=*/true);
}
fieldType = createFieldType(intArrowType, schema, externalProps);
break;
case BOOLEAN:
fieldType = createFieldType(new ArrowType.Bool(), schema, externalProps);
break;
case LONG:
final ArrowType longArrowType;
if (logicalType instanceof LogicalTypes.TimeMicros) {
longArrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64);
} else if (logicalType instanceof LogicalTypes.TimestampMillis) {
longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
} else if (logicalType instanceof LogicalTypes.TimestampMicros) {
longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null);
} else {
longArrowType = new ArrowType.Int(64, /*signed=*/true);
}
fieldType = createFieldType(longArrowType, schema, externalProps);
break;
case FLOAT:
fieldType = createFieldType(new ArrowType.FloatingPoint(SINGLE), schema, externalProps);
break;
case DOUBLE:
fieldType = createFieldType(new ArrowType.FloatingPoint(DOUBLE), schema, externalProps);
break;
case BYTES:
final ArrowType bytesArrowType;
if (logicalType instanceof LogicalTypes.Decimal) {
bytesArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType);
} else {
bytesArrowType = new ArrowType.Binary();
}
fieldType = createFieldType(bytesArrowType, schema, externalProps);
break;
case NULL:
fieldType = createFieldType(ArrowType.Null.INSTANCE, schema, externalProps);
break;
default:
// no-op, shouldn't get here
throw new UnsupportedOperationException();
}
if (name == null) {
name = getDefaultFieldName(fieldType.getType());
}
return new Field(name, fieldType, children.size() == 0 ? null : children);
}
FlightSqlProducer
arrow/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlProducer.java
/**
* Default schema templates for the {@link FlightSqlProducer}.
*/
final class Schemas {
public static final Schema GET_TABLES_SCHEMA = new Schema(asList(
Field.nullable("catalog_name", VARCHAR.getType()),
Field.nullable("db_schema_name", VARCHAR.getType()),
Field.notNullable("table_name", VARCHAR.getType()),
Field.notNullable("table_type", VARCHAR.getType()),
Field.notNullable("table_schema", MinorType.VARBINARY.getType())));
public static final Schema GET_TABLES_SCHEMA_NO_SCHEMA = new Schema(asList(
Field.nullable("catalog_name", VARCHAR.getType()),
Field.nullable("db_schema_name", VARCHAR.getType()),
Field.notNullable("table_name", VARCHAR.getType()),
Field.notNullable("table_type", VARCHAR.getType())));
public static final Schema GET_CATALOGS_SCHEMA = new Schema(
singletonList(Field.notNullable("catalog_name", VARCHAR.getType())));
public static final Schema GET_TABLE_TYPES_SCHEMA =
new Schema(singletonList(Field.notNullable("table_type", VARCHAR.getType())));
public static final Schema GET_SCHEMAS_SCHEMA =
new Schema(asList(
Field.nullable("catalog_name", VARCHAR.getType()),
Field.notNullable("db_schema_name", VARCHAR.getType())));
private static final Schema GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA =
new Schema(asList(
Field.nullable("pk_catalog_name", VARCHAR.getType()),
Field.nullable("pk_db_schema_name", VARCHAR.getType()),
Field.notNullable("pk_table_name", VARCHAR.getType()),
Field.notNullable("pk_column_name", VARCHAR.getType()),
Field.nullable("fk_catalog_name", VARCHAR.getType()),
Field.nullable("fk_db_schema_name", VARCHAR.getType()),
Field.notNullable("fk_table_name", VARCHAR.getType()),
Field.notNullable("fk_column_name", VARCHAR.getType()),
Field.notNullable("key_sequence", INT.getType()),
Field.nullable("fk_key_name", VARCHAR.getType()),
Field.nullable("pk_key_name", VARCHAR.getType()),
Field.notNullable("update_rule", MinorType.UINT1.getType()),
Field.notNullable("delete_rule", MinorType.UINT1.getType())));
public static final Schema GET_IMPORTED_KEYS_SCHEMA = GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA;
public static final Schema GET_EXPORTED_KEYS_SCHEMA = GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA;
public static final Schema GET_CROSS_REFERENCE_SCHEMA = GET_IMPORTED_EXPORTED_AND_CROSS_REFERENCE_KEYS_SCHEMA;
private static final List<Field> GET_SQL_INFO_DENSE_UNION_SCHEMA_FIELDS = asList(
Field.notNullable("string_value", VARCHAR.getType()),
Field.notNullable("bool_value", BIT.getType()),
Field.notNullable("bigint_value", BIGINT.getType()),
Field.notNullable("int32_bitmask", INT.getType()),
new Field(
"string_list", FieldType.notNullable(LIST.getType()),
singletonList(Field.nullable("item", VARCHAR.getType()))),
new Field(
"int32_to_int32_list_map", FieldType.notNullable(new ArrowType.Map(false)),
singletonList(new Field(DATA_VECTOR_NAME, new FieldType(false, STRUCT.getType(), null),
ImmutableList.of(
Field.notNullable(KEY_NAME, INT.getType()),
new Field(
VALUE_NAME, FieldType.nullable(LIST.getType()),
singletonList(Field.nullable("item", INT.getType()))))))));
public static final Schema GET_SQL_INFO_SCHEMA =
new Schema(asList(
Field.notNullable("info_name", UINT4.getType()),
new Field("value",
FieldType.notNullable(
new Union(UnionMode.Dense, range(0, GET_SQL_INFO_DENSE_UNION_SCHEMA_FIELDS.size()).toArray())),
GET_SQL_INFO_DENSE_UNION_SCHEMA_FIELDS)));
public static final Schema GET_PRIMARY_KEYS_SCHEMA =
new Schema(asList(
Field.nullable("catalog_name", VARCHAR.getType()),
Field.nullable("db_schema_name", VARCHAR.getType()),
Field.notNullable("table_name", VARCHAR.getType()),
Field.notNullable("column_name", VARCHAR.getType()),
Field.notNullable("key_sequence", INT.getType()),
Field.nullable("key_name", VARCHAR.getType())));
private Schemas() {
// Prevent instantiation.
}
}