import com.ZAMPSDK.AppTracking.common.ZPConstants;
import com.ZAMPSDK.AppTracking.mo.BootContent;
import com.ZAMPSDK.AppTracking.mo.UserProfile;
import com.google.protobuf.Message;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.json.JSONArray;
import org.json.JSONObject;
import xray.JarIntegration;
import xray.hadoop.input.PBWritable;
import zampdata.meta.GEOEntry;
import zampdata.meta.records.pb.protobuf;
import zampkb.utilities.IPUtil;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* Created by andy on 2015/5/14.
* 最终目标：生成app的用户历史表（每小时生成一次）
* 1、appboot日志和历史表用户合并
* 2、携程的渠道信息更新进入历史表中渠道字段
* 3、非携程的渠道信息更新进入历史表中渠道字段
* 4、将用户总时长设置进user字段
* 5、将event设置进user字段
*/
public class AppUserProfileJob extends JarIntegration {
enum JOINKEY {
FIELDSERROR,
FIELDUNDEFINE
}
enum VALUECONTENT {
FIELDSERROR,
FIELDUNDEFINE
}
public static class AppUserProfileJobMapper extends Mapper<Text, Writable, Text, Text> {
private String[] map_value_tmp;
private UserProfile up = new UserProfile();
private String[] map_key_tmp;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
}
@Override
protected void map(Text key, Writable value, Context context) throws IOException, InterruptedException {
if (value instanceof PBWritable) {
Message message = ((PBWritable) value).getMessage();
if (message instanceof protobuf.APPON) {
protobuf.APPON appon = ((PBWritable) value).getMessage();
String appkey = appon.getAppKey();
String idfa = appon.getIdfa();
String mac = appon.getMac();
String androidid = appon.getAndroidId();
String imei = appon.getImei();
long logtime = appon.getTimestamp();
String deviceType = appon.getDeviceType();
String os = appon.getOs();
String network = appon.getNetwork();
long clientIp = appon.getIp();
String channelId = appon.getChannelId();
String appVersion = appon.getAppVersion();
int isActivity = appon.getIsActivity() == true ? 1 : 0;
int screenHeight = appon.getScreenHeight();
int screenWidth = appon.getScreenWidth();
int screenDensity = appon.getScreenDensity();
String sp = appon.getSp();
String brand = appon.getBrand();
String countryId;
String provinceid;
String cityid;
/**
ZTGEOEntry ztgeoEntry = new ZTGEOEntry((int) clientIp);
int countryCode = ztgeoEntry.getCountryCode();
if (countryCode != 0) {
countryId = String.valueOf(ztgeoEntry.getCountryCode());
provinceid = String.valueOf(ztgeoEntry.getProvinceCode());
cityid = String.valueOf(ztgeoEntry.getCityCode());
} else {
GEOEntry geo = new GEOEntry((int) clientIp);
countryId = String.valueOf(geo.getCountryCode());
provinceid = String.valueOf(geo.getProvinceCode());
cityid = String.valueOf(geo.getCityCode());
}
**/
GEOEntry geo = new GEOEntry(IPUtil.getGEO(clientIp));
countryId = String.valueOf(geo.getCountryCode());
provinceid = String.valueOf(geo.getProvinceCode());
cityid = String.valueOf(geo.getCityCode());
BootContent bootContent = new BootContent();
bootContent.setMark(ZPConstants.BOOT_CONTENT_MARK);
bootContent.setLogtime(logtime * 1000);
bootContent.setDeviceType(deviceType);
bootContent.setOs(os);
bootContent.setNetwork(network);
bootContent.setCountryId(countryId);
bootContent.setProvinceid(provinceid);
bootContent.setCityid(cityid);
bootContent.setChannelId(channelId);
bootContent.setAppVersion(appVersion);
bootContent.setIsActivity(isActivity);
bootContent.setScreenHeight(screenHeight);
bootContent.setScreenWidth(screenWidth);
bootContent.setScreenDensity(screenDensity);
bootContent.setSp(sp);
bootContent.setBrand(brand);
String deviceId = appkey + ZPConstants.FIELDS_SPLIT + idfa + ZPConstants.FIELDS_SPLIT + mac + ZPConstants.FIELDS_SPLIT + androidid + ZPConstants.FIELDS_SPLIT + imei;
context.write(new Text(deviceId), new Text(bootContent.toString()));
}
//事件 add by caojian 2016-03-04
else if (message instanceof protobuf.APPEVENT) {
protobuf.APPEVENT appevent = ((PBWritable) value).getMessage();
String appkey = appevent.getAppKey();
String idfa = appevent.getIdfa();
String mac = appevent.getMac();
String androidid = appevent.getAndroidId();
String imei = appevent.getImei();
String eventId = appevent.getEventId();
String deviceId = appkey + ZPConstants.FIELDS_SPLIT + idfa + ZPConstants.FIELDS_SPLIT + mac + ZPConstants.FIELDS_SPLIT + androidid + ZPConstants.FIELDS_SPLIT + imei;
context.write(new Text(deviceId), new Text(ZPConstants.MARK_EVENTS_JOIN_USERPROFILE + ZPConstants.FIELDS_TAB + eventId));
}
} else if (value instanceof Text) {
map_value_tmp = StringUtils.splitPreserveAllTokens(value.toString(), ZPConstants.FIELDS_TAB);
map_key_tmp = StringUtils.splitPreserveAllTokens(key.toString(), ZPConstants.FIELDS_TAB);
//设置事件和总体使用时长 add by caojian 2016-03-04
if (NumberUtils.isNumber(value.toString())) {
//用户使用时长
context.write(key, new Text(ZPConstants.MARK_DURATION_JOIN_USERPROFILE + ZPConstants.FIELDS_TAB + value.toString()));
} else {
if (map_value_tmp[0].equals(ZPConstants.USER_PROFILE_MARK)) {
//历史表
up.parseToUserProfile(map_value_tmp);
if (up.isValidRecord()) {
String deviceId = up.getAppkey() + ZPConstants.FIELDS_SPLIT + up.getIdfa() + ZPConstants.FIELDS_SPLIT + up.getMac() + ZPConstants.FIELDS_SPLIT + up.getAndroidid() + ZPConstants.FIELDS_SPLIT + up.getImei();
context.write(new Text(deviceId), new Text(up.toString()));
}
}
/**
* 携程归因：
output key: appkey , idfa, days (归因天数)
output value: active_time , campaign_id,click_time
非携程归因：
output key: appkey, idfa, mac, imei, android, "14" (归因天数包括 1,7,14,30,n), "last"(归因类型 first,last,avg), "1.0" (归因权重)
output value:campaign_id, click_time, message_info, id_type, device_id, active_time, brand, device_type, os, province
*/
//1,携程的归因结果
else if (map_key_tmp.length == 3) {
String appkey = map_key_tmp[0];
String idfa = map_key_tmp[1];
if (map_value_tmp.length >= 2) {
String campaign_id = map_value_tmp[1];
String deviceId = appkey + ZPConstants.FIELDS_SPLIT + idfa + ZPConstants.FIELDS_SPLIT + "" + ZPConstants.FIELDS_SPLIT + "" + ZPConstants.FIELDS_SPLIT + "";
context.write(new Text(deviceId), new Text(ZPConstants.MARK_CTRIP_ACTIVE_RESULT + ZPConstants.FIELDS_TAB + campaign_id));
}
}
//2,非携程归因结果
else if (map_key_tmp.length == 8) {
String deviceId = map_key_tmp[0] + ZPConstants.FIELDS_SPLIT + map_key_tmp[1] + ZPConstants.FIELDS_SPLIT + map_key_tmp[2] + ZPConstants.FIELDS_SPLIT + map_key_tmp[3] + ZPConstants.FIELDS_SPLIT + map_key_tmp[4];
if (map_value_tmp.length > 0) {
String campaign_id = map_value_tmp[0];
context.write(new Text(deviceId), new Text(ZPConstants.MARK_OTHER_ACTIVE_RESULT + ZPConstants.FIELDS_TAB + campaign_id));
}
}
}
}
}
}
public static class AppUserProfileJobReduce extends Reducer<Text, Text, Text, Text> {
private String[] tmp;
private List<BootContent> lists;
private Date today;
Calendar cl;
private Set<String[]> campaignsJsonFromPHP = null;
private List<String> eventLists;
private MultipleOutputs<Text, Text> mos_;
String todayString;
String hdfs_phoenix;
private SimpleDateFormat simpleDateFormat;
private String runDayString;
//添加是否跨天
protected void updateStatus(UserProfile up, BootContent bc) {
if (!StringUtils.equals(up.getAppversion(), bc.getAppVersion()) && StringUtils.isNotBlank(bc.getAppVersion()) && StringUtils.isNotEmpty(bc.getAppVersion()) && !StringUtils.equals(bc.getAppVersion(), ZPConstants.DEFAULT_VALUE)) {
up.setAppversion(bc.getAppVersion());
}
if (!StringUtils.equals(up.getChannelid(), bc.getChannelId()) && StringUtils.isNotBlank(bc.getChannelId()) && StringUtils.isNotEmpty(bc.getChannelId()) && !StringUtils.equals(bc.getChannelId(), ZPConstants.DEFAULT_VALUE)) {
up.setChannelid(bc.getChannelId());
}
if (!StringUtils.equals(up.getCountryId(), bc.getCountryId()) && StringUtils.isNotBlank(bc.getCountryId()) && StringUtils.isNotEmpty(bc.getCountryId())) {
up.setCountryId(bc.getCountryId());
}
if (!StringUtils.equals(up.getProvinceid(), bc.getProvinceid()) && StringUtils.isNotBlank(bc.getProvinceid()) && StringUtils.isNotEmpty(bc.getProvinceid())) {
up.setProvinceid(bc.getProvinceid());
}
if (!StringUtils.equals(up.getCityid(), bc.getCityid()) && StringUtils.isNotBlank(bc.getCityid()) && StringUtils.isNotEmpty(bc.getCityid())) {
up.setCityid(bc.getCityid());
}
if (!StringUtils.equals(up.getDevicetype(), bc.getDeviceType()) && StringUtils.isNotBlank(bc.getDeviceType()) && StringUtils.isNotEmpty(bc.getDeviceType()) && !StringUtils.equals(bc.getDeviceType(), ZPConstants.DEFAULT_VALUE)) {
up.setDevicetype(bc.getDeviceType());
}
if (!StringUtils.equals(up.getScreen(), bc.getScreenWidth() + "*" + bc.getScreenHeight()) && bc.getScreenHeight() > 0 && bc.getScreenWidth() > 0) {
up.setScreen(bc.getScreenWidth() + "*" + bc.getScreenHeight());
}
if (!StringUtils.equals(up.getOs(), bc.getOs()) && StringUtils.isNotBlank(bc.getOs()) && StringUtils.isNotEmpty(bc.getOs()) && !StringUtils.equals(bc.getOs(), ZPConstants.DEFAULT_VALUE)) {
up.setOs(bc.getOs());
}
if (!StringUtils.equals(up.getSp(), bc.getSp()) && StringUtils.isNotBlank(bc.getSp()) && StringUtils.isNotEmpty(bc.getSp()) && !StringUtils.equals(bc.getSp(), ZPConstants.NULL_VALUE) && !StringUtils.equals(bc.getSp(), ZPConstants.DEFAULT_VALUE)) {
up.setSp(bc.getSp());
}
if (!StringUtils.equals(up.getNetwork(), bc.getNetwork()) && StringUtils.isNotBlank(bc.getNetwork()) && StringUtils.isNotEmpty(bc.getNetwork()) && !StringUtils.equals(bc.getNetwork(), ZPConstants.DEFAULT_VALUE)) {
up.setNetwork(bc.getNetwork());
}
// 如果品牌字段等于其他，devicetype字段是_NULL，则不更新品牌字段，该数据问题存在于20151112~20151121的ETL程序BUG中
if (StringUtils.equals(bc.getBrand(), "其他") && (StringUtils.equals(bc.getDeviceType(), ZPConstants.DEFAULT_VALUE) || StringUtils.equals(bc.getDeviceType(), ZPConstants.NULL_VALUE))) {
return;
}
//更新品牌
if (!StringUtils.equals(up.getBrand(), bc.getBrand()) && StringUtils.isNotBlank(bc.getBrand()) && StringUtils.isNotEmpty(bc.getBrand()) && !StringUtils.equals(bc.getBrand(), ZPConstants.DEFAULT_VALUE)) {
up.setBrand(bc.getBrand());
}
}
private String readFile(Path filePath) {
try{
BufferedReader bufferedReader = new BufferedReader(new FileReader(filePath.toString()));
StringBuffer sb = new StringBuffer();
String stopWord = null;
while((stopWord = bufferedReader.readLine()) != null) {
sb.append(stopWord);
}
return sb.toString().trim();
} catch(IOException ex) {
System.err.println("Exception while reading stop words file: " + ex.getMessage());
}
return "[]";
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
mos_ = new MultipleOutputs<Text, Text>(context);
lists = new ArrayList<BootContent>();
eventLists = new ArrayList<String>();
Configuration config = context.getConfiguration();
todayString = config.get("run.day", DateFormatUtils.format(DateUtils.toCalendar(DateUtils.addHours(new Date(), -1)), "yyyyMMddHH"));
hdfs_phoenix = config.get("hdfs_phoenix.path", "/bh/warehouse/mobile_tracking/AppUserProfileJob/hdfs_phoenix/" + todayString + "/");
String[] parsePatterns = {"yyyyMMdd", "yyyyMMddHH"};
campaignsJsonFromPHP = new HashSet<String[]>();
try {
today = DateUtils.parseDateStrictly(todayString, parsePatterns);
cl = DateUtils.toCalendar(today);
simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
runDayString = simpleDateFormat.format(today);
} catch (ParseException e) {
e.printStackTrace();
throw new RuntimeException("setup function parse data error!!!");
}
try{
Path[] stopWordsFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());
if(stopWordsFiles != null && stopWordsFiles.length > 0) {
for(Path stopWordFile : stopWordsFiles) {
String filename = stopWordFile.getName();
System.out.println("distirbutedcache="+filename);
if (filename.equals(todayString+".txt")) {
String tmp = readFile(stopWordFile);
JSONArray jsonArray = new JSONArray(tmp);
for( int i = 0; i < jsonArray.length(); i++ ) {
JSONObject campaignsJson = jsonArray.getJSONObject(i);
String[] campaignsArray = new String[3];
campaignsArray[0] = campaignsJson.get("campaign_id").toString();
campaignsArray[1] = campaignsJson.get("app_key").toString();
campaignsArray[2] = campaignsJson.get("campaign_channel").toString();
campaignsJsonFromPHP.add(campaignsArray);
}
return;
}
}
}
} catch(IOException ex) {
System.err.println("Exception in mapper setup: " + ex.getMessage());
}
}
public String getChannelByPHP(String appkey, Set<String[]> campaignsJsons, String campaign) {
String channel = null;
for (String[] campaignJson : campaignsJsons) {
// campaign_id
String campaign_id = campaignJson[0];
// appkey
String app_key = campaignJson[1];
// campaign_channel
String campaign_channel = campaignJson[2];
if (appkey.equals(app_key) && campaign.equals(campaign_id)) {
channel = campaign_channel;
break;
}
}
return channel;
}
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
UserProfile up = null;
String appkey = null;
String idfa = null;
String mac = null;
String androidId = null;
String imei = null;
String channel = null;
long userDuration = 0;
eventLists.clear();
lists.clear();
String keyString = key.toString();
String[] keySplits = StringUtils.splitPreserveAllTokens(keyString, ZPConstants.FIELDS_SPLIT);
if (keySplits.length == 5) {
appkey = keySplits[0];
idfa = keySplits[1];
mac = keySplits[2];
androidId = keySplits[3];
imei = keySplits[4];
} else {
context.getCounter(JOINKEY.FIELDSERROR).increment(1);
context.setStatus("JOINKEY.FIELDSERROR");
System.err.println("JOINKEY.FIELDSERROR:" + keyString);
return;
}
for (Text value : values) {
tmp = StringUtils.splitPreserveAllTokens(value.toString(), ZPConstants.FIELDS_TAB);
// 由于没有采用序列化与反序列化的方式，通过判断长度，扩展性不好，顾在UserProfile历史表中添加mark字段，其值为track_user_profile
if (StringUtils.equalsIgnoreCase(tmp[0], ZPConstants.USER_PROFILE_MARK) && tmp.length >= 27) {
up = UserProfile.getUserProfileInstance(tmp);
} else if (StringUtils.equalsIgnoreCase(tmp[0], ZPConstants.BOOT_CONTENT_MARK) && tmp.length >= 15) {
BootContent bc = BootContent.parseToBootContentInstance(tmp);
lists.add(bc);
}
//携程归因
else if (StringUtils.equalsIgnoreCase(tmp[0], ZPConstants.MARK_CTRIP_ACTIVE_RESULT) && tmp.length == 2) {
channel = getChannelByPHP(appkey, campaignsJsonFromPHP, tmp[1]);
}
//非携程归因
else if (StringUtils.equalsIgnoreCase(tmp[0], ZPConstants.MARK_OTHER_ACTIVE_RESULT) && tmp.length == 2) {
channel = getChannelByPHP(appkey, campaignsJsonFromPHP, tmp[1]);
}
//设置事件和总体使用时长 add by caojian 2016-03-04
else if (StringUtils.equalsIgnoreCase(tmp[0], ZPConstants.MARK_EVENTS_JOIN_USERPROFILE) && tmp.length == 2) {
eventLists.add(tmp[1]);
} else if (StringUtils.equalsIgnoreCase(tmp[0], ZPConstants.MARK_DURATION_JOIN_USERPROFILE) && tmp.length == 2) {
if (NumberUtils.isNumber(tmp[1])) {
userDuration = NumberUtils.toLong(tmp[1]);
} else {
System.err.println("时长错误:" + tmp[1]);
context.getCounter("duration error", "count").increment(1);
}
} else {
context.getCounter(VALUECONTENT.FIELDSERROR).increment(1);
context.setStatus("VALUECONTENT.FIELDSERROR");
System.err.println("VALUECONTENT.FIELDSERROR:" + tmp.toString());
}
}
int startNumber = lists.size();
//add by caojian 2016-03-04 ，如果历史用户不存在，appboot也不存在，但是有渠道，则不需要处理
if (up == null && startNumber <= 0 && channel != null && StringUtils.isNotEmpty(channel)) {
System.err.println("active_result user not match history user or not match appboot user,key=" + key.toString() + ",channel=" + channel);
context.getCounter("channel error", "count").increment(1);
return;
}
//add by caojian 2016-03-04 ,如果历史用户没有，appboot也不存在, 但是event有，则不需处理
if (up == null && startNumber <= 0 && eventLists != null && eventLists.size() > 0) {
System.err.println("event user not match history user or not match appboot user,key=" + key.toString() + ",eventLists=" + eventLists);
context.getCounter("events error", "count").increment(1);
return;
}
//add by caojian 2016-03-04 ,如果历史用户没有，appboot也不存在,但是使用时长有，则不需处理
if (up == null && startNumber <= 0 && userDuration != 0) {
System.err.println("userDuration not match history user or not match appboot user,key=" + key.toString() + ",userDuration=" + userDuration);
context.getCounter("userDuration error", "count").increment(1);
return;
}
if (startNumber > 0) {
Collections.sort(lists);
BootContent bc = lists.get(0);
BootContent lastBc;
if (startNumber > 1) {
lastBc = lists.get(startNumber - 1);
} else {
lastBc = bc;
}
String lastBcDateFormat = simpleDateFormat.format(new Date(lastBc.getLogtime()));
// userprofile历史表数据没有此次启动信息，则为首次激活的用户
if (up == null) {
up = new UserProfile();
up.setAppkey(appkey);
up.setIdfa(idfa);
up.setMac(mac);
up.setAndroidid(androidId);
up.setImei(imei);
up.setActive_timestamp(bc.getLogtime());
int aHour = (int) DateUtils.getFragmentInHours(new Date(bc.getLogtime()), Calendar.DAY_OF_YEAR);
if (aHour >= 0 && aHour <= 24) {
up.setActive_hour(aHour);
} else {
System.err.println("Active_hour_error:" + aHour);
}
up.setIs_update_appversion(1);
up.setAppversion(bc.getAppVersion());
//add by caojian on 2015/11/26 12:01.
up.setInsert_timestamp(today.getTime());
// 如果首条日志不是激活，则表示激活数据丢失，则通过收到的首条日志假设为激活日志,但记录状态
if (bc.getIsActivity() != 1) {
context.getCounter(VALUECONTENT.FIELDUNDEFINE).increment(1);
context.setStatus("VALUECONTENT.FIELDUNDEFINE");
System.err.println("VALUECONTENT.FIELDUNDEFINE:" + " FIRST IS NOT ACTIVITY LOG -- " + bc.toString());
}
}
//up第一种情况： if(up== null && startNumber <= 0){ }
//upActive第三种情况：if(up ！= null && startNumber <= 0){ }
//up第二种情况：第四种情况 add by caojian on 2016-05-20
Set<String> activeTimeSet = up.getActiveTimeSet();
activeTimeSet.add(lastBcDateFormat);
up.setActiveTimeSet(activeTimeSet);
//设置当天启动次数
if (DateUtils.isSameDay(new Date(up.getLast_start_timestamp()), new Date(bc.getLogtime()))) {
up.setCurrent_day_start_num(up.getCurrent_day_start_num() + startNumber);
} else {
up.setCurrent_day_start_num(startNumber);
}
//设置当小时启动次数
int current_hour_boot_count = 0;
for (BootContent bootContent : lists) {
if (DateUtils.isSameDay(new Date(bootContent.getLogtime()), today) && DateUtils.getFragmentInHours(new Date(bootContent.getLogtime()), Calendar.DATE) == cl.get(Calendar.HOUR_OF_DAY)) {
current_hour_boot_count++;
}
}
up.setCurrent_hour_start_num(current_hour_boot_count);
BootContent bct = null;
for (int t = 0; t < startNumber; t++) {
bct = lists.get(t);
updateStatus(up, bct);
}
bct = null;
if (up.getLast_start_timestamp() < lastBc.getLogtime()) {
//当天启动记录中数据和历史表中记录不一致，但是历史表记录的最后一次启动时间不等于当天，更新历史表，以启动记录数据为准。
if (!DateUtils.isSameDay(new Date(up.getLast_start_timestamp()), new Date(bc.getLogtime()))) {
updateStatus(up, lastBc);
//用户启动APP总天数(一天内多次启动算一次)
up.setHistory_start_days_num(up.getHistory_start_days_num() + 1);
}
//更新倒数第二次启动时间
if (up.getTbl_start_timestamp() != up.getLast_start_timestamp() && up.getLast_start_timestamp() != 0) {
up.setTbl_start_timestamp(up.getLast_start_timestamp());
up.setTbl_start_hour(up.getLast_start_hour());
}
// 更新最后一次启动时间
Date lastLogTime = new Date(lastBc.getLogtime());
up.setLast_start_timestamp(lastBc.getLogtime());
int startHour = (int) DateUtils.getFragmentInHours(lastLogTime, Calendar.DAY_OF_YEAR);
if (startHour >= 0 && startHour <= 24) {
up.setLast_start_hour(startHour);
} else {
System.err.println("last_start_hour_error:" + startHour);
}
}
up.setTotal_start_num(up.getTotal_start_num() + startNumber);
if (!StringUtils.equals(up.getMark(), ZPConstants.USER_PROFILE_MARK)) {
up.setMark(ZPConstants.USER_PROFILE_MARK);
}
}
//fixed by caojian on 2015/11/05 17:11.
else {
//该用户该小时启动次数为0，
// 如果是0点，则当天启动次数为0，
// 如果不是0点，则取前一个小时的当天启动次数
int hour = cl.get(Calendar.HOUR_OF_DAY);
if (hour == 0) {
up.setCurrent_day_start_num(0);
} else {
up.setCurrent_day_start_num(up.getCurrent_day_start_num());
}
//fixed by caojian on 2016/03/31 19:11. 如果一个用户当前小时没有启动，则当前小时启动设置为0
up.setCurrent_hour_start_num(0);
}
// modify by andy on 2016/5/10 16:00 如果应用是IOS则设置为推广渠道, 如果是安卓则设置为应用渠道
String upidfa = up.getIdfa();
if (up != null && StringUtils.isNotBlank(channel) && StringUtils.isNotEmpty(channel) && StringUtils.isNotBlank(upidfa) && StringUtils.isNotEmpty(upidfa)) {
up.setChannelid(channel);
}
String uchannel = up.getChannelid();
if (up != null && (StringUtils.isBlank(uchannel) || StringUtils.isEmpty(uchannel))){
up.setChannelid(channel);
}
//设置事件和总体使用时长 add by caojian 2016-03-04
if (up != null) {
Set<String> eventSet = up.getEventSet();
if (eventLists != null && eventLists.size() > 0) {
for (String event : eventLists) {
eventSet.add(event + ZPConstants.FIELDS_TAB +runDayString);
}
}
up.setEventSet(eventSet);
up.setTotal_visit_app_time(up.getTotal_visit_app_time() + (int) userDuration);
//add by andy to produce path of writing phoenix
StringBuilder sb = new StringBuilder();
long activetime = up.getActive_timestamp();
Date activeDate = new Date(activetime);
long lastStartTime = up.getLast_start_timestamp();
Date lastDate = new Date(lastStartTime);
String lastStr = DateFormatUtils.format(lastDate, "yyyyMMddHH");
if (lastStr != null && lastStr.equals(todayString)) {
String activeStr = DateFormatUtils.format(activeDate, "yyyyMMddHH");
if (activeStr != null && activeStr.equals(todayString)) {
sb.append(activeStr.substring(0, 8));
sb.append("\t");
sb.append(activeStr.substring(8, 10));
} else {
sb.append("0\t0");
}
sb.append("\t");
//etl中的地域没有进行非中国的处理、所以在这里进行非中国的处理：不是中国的省份为-1
if (!up.getCountryId().equals("1156000000")) {
sb.append("-1");
} else {
sb.append(up.getProvinceid());
}
sb.append("\t");
sb.append(up.getBrand());
sb.append("\t");
sb.append(up.getDevicetype());
sb.append("\t");
sb.append(up.getOs());
sb.append("\t");
sb.append(up.getNetwork());
sb.append("\t");
sb.append(up.getSp());
sb.append("\t");
sb.append(up.getLast_start_timestamp() / 1000);
sb.append("\t");
sb.append(up.getLast_start_hour());
sb.append("\t");
sb.append(up.getCurrent_hour_start_num());
sb.append("\t");
sb.append(userDuration);
//add channel, screeen, appversion 三个字段
sb.append("\t");
sb.append(up.getChannelid());
sb.append("\t");
sb.append(up.getScreen());
sb.append("\t");
sb.append(up.getAppversion());
mos_.write("AppUserProfileJob", key, new Text(sb.toString()), hdfs_phoenix);
}
context.write(key, new Text(up.toString()));
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
mos_.close();
}
}
@Override
protected void execute() throws Exception {
Job job = new Job(getConfiguration(), getJobName());
job.setMapperClass(AppUserProfileJobMapper.class);
job.setReducerClass(AppUserProfileJobReduce.class);
job.setInputFormatClass(SequenceFileInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
MultipleOutputs.addNamedOutput(job, "AppUserProfileJob",
SequenceFileOutputFormat.class, Text.class, Text.class);
Configuration cf = getConfiguration();
String log_date = cf.get("run.day");
DistributedCache.addCacheFile(new Path("/bh/warehouse/mobile_tracking/caches_app_analytics/"+log_date+".txt").toUri(), job.getConfiguration());
job.setReduceSpeculativeExecution(false);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
716
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
