Spark 之 DataType

TimestampType

spark-sql> create table if not exists test_emr_date(
         >     id      int,
         >     datetime1   timestamp
         > );

spark-sql> 
         > insert into test_emr_date values (1,timestamp'2021-11-29 00:01:00');
Time taken: 0.497 seconds

spark-sql> 
         > insert into test_emr_date values (1,timestamp'2021-11-29 00:01:00.100');
Time taken: 0.36 seconds

spark-sql> 
         > select * from test_emr_date;
1	2021-11-29 00:01:00.1
1	2021-11-29 00:01:00

DecimalType

/**
 * Extra factory methods and pattern matchers for Decimals.
 *
 * @since 1.3.0
 */
@Stable
object DecimalType extends AbstractDataType {
  import scala.math.min

  val MAX_PRECISION = 38
  val MAX_SCALE = 38
  val SYSTEM_DEFAULT: DecimalType = DecimalType(MAX_PRECISION, 18)
  val USER_DEFAULT: DecimalType = DecimalType(10, 0)
  val MINIMUM_ADJUSTED_SCALE = 6

  // The decimal types compatible with other numeric types
  private[sql] val BooleanDecimal = DecimalType(1, 0)
  private[sql] val ByteDecimal = DecimalType(3, 0)
  private[sql] val ShortDecimal = DecimalType(5, 0)
  private[sql] val IntDecimal = DecimalType(10, 0)
  private[sql] val LongDecimal = DecimalType(20, 0)
  private[sql] val FloatDecimal = DecimalType(14, 7)
  private[sql] val DoubleDecimal = DecimalType(30, 15)
  private[sql] val BigIntDecimal = DecimalType(38, 0)

spark functions

> select from_unixtime(123456789);
java.version：1.8.0_371
1973-11-30 05:33:09
Time taken: 0.603 seconds, Fetched 1 row(s)

 > select from_unixtime(0);
java.version：1.8.0_371
1970-01-01 08:00:00
Time taken: 0.246 seconds, Fetched 1 row(s)

> select datediff('2013-05-31 13:00:00', '2013-04-30 12:30:00');
java.version：1.8.0_371
31
Time taken: 0.072 seconds, Fetched 1 row(s)