Java Monitor S3 File Update Time

Java Monitor S3 File Update Time

I have a JAVA Spring Boot Project. The requirement is to monitor the file update time in S3.

Here is the pom.xml requirement pom.xml
<!-- AWS -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.251</version>
</dependency>

Here is the implementation Class S3ServiceImpl.java

package com.sillycat.jobsmonitorapi.service;

import java.util.Date;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.sillycat.jobsmonitorapi.config.AWSConfig;

@Service
public class S3ServiceImpl implements S3Service {

private static final Logger logger = LoggerFactory.getLogger(S3ServiceImpl.class);

@Autowired
private AWSConfig awsConfig;

private AmazonS3 s3Client;

private void init() {
logger.info("start to init the s3Client--------");
this.s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(awsConfig.getKey(), this.awsConfig.getSecret())))
.withRegion(Regions.US_EAST_1).build();
logger.info("start to init the s3Client end--------");
}

public Date getFileLastUpdate(String bucketName, String path) {
if (this.s3Client == null) {
this.init();
}
Date result = null;
ObjectListing objectList = this.s3Client.listObjects(bucketName, path);
if (objectList != null) {
for (S3ObjectSummary fileSummary : objectList.getObjectSummaries()) {
String tmpKey = fileSummary.getKey();
logger.info("find files {}", tmpKey);
logger.info("file the file with last update {}", fileSummary.getLastModified());
result = fileSummary.getLastModified();
break;
}
}
return result;
}

}

Here is the config class under the config directory AWSConfig.java
package com.sillycat.jobsmonitorapi.config;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;

@Configuration
@EnableConfigurationProperties
@ConfigurationProperties(prefix = "aws")
public class AWSConfig {

private String key;

private String secret;

public String getKey() {
return key;
}

public void setKey(String key) {
this.key = key;
}

public String getSecret() {
return secret;
}

public void setSecret(String secret) {
this.secret = secret;
}

}

In side the application.yaml, we will have our configuration as follow:
aws:
key: xxxxxxx
secret: xxxxxxxxxxxx

Here is the UNIT test how it will pick up the configuration and work S3ServiceTest.java

package com.sillycat.jobsmonitorapi.service;

import java.util.Date;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;

@RunWith(SpringRunner.class)
@SpringBootTest
@ActiveProfiles("dev")
public class S3ServiceTest {

@Autowired
S3Service s3Service;

@Test
public void testDummy() {
Assert.assertTrue(true);
}

@Test
public void testLastModified() {
Date date = s3Service.getFileLastUpdate("j2c-production", "feed-2g/jobs_2g.txt.gz");
Assert.assertNotNull(date);
}

}

Here is the Util Class to deal with the time, I tried to use JAVA API 8 this time, no JODA Time
package com.sillycat.jobsmonitorapi.commons;

import java.time.LocalDateTime;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

public class DateTimeUtil {

private static final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");

/**
*
* @param timeString
* @return
*/
public static DateTime parseFullTimeString(String timeString) {
DateTime dt = formatter.parseDateTime(timeString);
return dt;
}

/**
*
* @param date2
* time from S3
* @param hours
* @return
*/
public static Boolean isUpdateDeleyHours(LocalDateTime updateTime, int hours) {
if (updateTime.isBefore(LocalDateTime.now().minusHours(hours))) {
// S3 is older than now - hours, something wrong
return true;
} else {
return false;
}
}

}

Here is the unit class to handle the DateTimeUtil.java

package com.sillycat.jobsmonitorapi.commons;

import java.time.LocalDateTime;
import java.time.Month;
import java.time.ZoneId;
import java.util.Date;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Test;

public class DateTimeUtilTest {

@Test
public void dummy() {
Assert.assertTrue(true);
}

@Test
public void parseFullTimeString() {
String timeString = "2017-11-02 10:55:37";
DateTime dt = DateTimeUtil.parseFullTimeString(timeString);
Assert.assertNotNull(dt);
}

@Test
public void isDeleyHours() {
LocalDateTime oldTime = LocalDateTime.of(2017, Month.JANUARY, 14, 19, 30);
Boolean result1 = DateTimeUtil.isUpdateDeleyHours(oldTime, 3);
Assert.assertTrue(result1);
Date now = new Date();
LocalDateTime nowDateTime = LocalDateTime.ofInstant(now.toInstant(), ZoneId.systemDefault());
Boolean result2 = DateTimeUtil.isUpdateDeleyHours(nowDateTime, 3);
Assert.assertFalse(result2);

}

}


References:
http://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingJava.html
http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/S3ObjectSummary.html
https://stackoverflow.com/questions/38578937/spring-boot-amazon-aws-s3-bucket-file-download-access-denied