java epoch times_Converting Timestamp to epoch in Spark (Java)

I think you are looking at using: unix_timestamp()

Which you can import from:

import static org.apache.spark.sql.functions.unix_timestamp;

And use like:

df = df.withColumn(

"epoch",

unix_timestamp(col("date")));

And here is a full example, where I tried to mimic your use-case:

package net.jgp.books.spark.ch12.lab990_others;

import static org.apache.spark.sql.functions.col;

import static org.apache.spark.sql.functions.from_unixtime;

import static org.apache.spark.sql.functions.unix_timestamp;

import java.util.ArrayList;

import java.util.List;

import java.util.Random;

import org.apache.spark.sql.Dataset;

import org.apache.spark.sql.Row;

import org.apache.spark.sql.RowFactory;

import org.apache.spark.sql.SparkSession;

import org.apache.spark.sql.types.DataTypes;

import org.apache.spark.sql.types.StructField;

import org.apache.spark.sql.types.StructType;

/**

* Use of from_unixtime() and unix_timestamp().

*

* @author jgp

*/

public class EpochTimestampConversionApp {

/**

* main() is your entry point to the application.

*

* @param args

* @throws InterruptedException

*/

public static void main(String[] args) {

EpochTimestampConversionApp app = new EpochTimestampConversionApp();

app.start();

}

/**

* The processing code.

*/

private void start() {

// Creates a session on a local master

SparkSession spark = SparkSession.builder()

.appName("expr()")

.master("local")

.getOrCreate();

StructType schema = DataTypes.createStructType(new StructField[] {

DataTypes.createStructField(

"event",

DataTypes.IntegerType,

false),

DataTypes.createStructField(

"original_ts",

DataTypes.StringType,

false) });

// Building a df with a sequence of chronological timestamps

List rows = new ArrayList<>();

long now = System.currentTimeMillis() / 1000;

for (int i = 0; i < 1000; i++) {

rows.add(RowFactory.create(i, String.valueOf(now)));

now += new Random().nextInt(3) + 1;

}

Dataset df = spark.createDataFrame(rows, schema);

df.show();

df.printSchema();

// Turning the timestamps to Timestamp datatype

df = df.withColumn(

"date",

from_unixtime(col("original_ts")).cast(DataTypes.TimestampType));

df.show();

df.printSchema();

// Turning back the timestamps to epoch

df = df.withColumn(

"epoch",

unix_timestamp(col("date")));

df.show();

df.printSchema();

// Collecting the result and printing ou

List timeRows = df.collectAsList();

for (Row r : timeRows) {

System.out.printf("[%d] : %s (%s)n",

r.getInt(0),

r.getAs("epoch"),

r.getAs("date"));

}

}

}

And the output is:

...

[994] : 1551997326 (2019-03-07 14:22:06)

[995] : 1551997329 (2019-03-07 14:22:09)

[996] : 1551997330 (2019-03-07 14:22:10)

[997] : 1551997332 (2019-03-07 14:22:12)

[998] : 1551997333 (2019-03-07 14:22:13)

[999] : 1551997335 (2019-03-07 14:22:15)

Hopefully this helps.