嵌入式ETL工具的例子

程序代码

消息bean

import java.util.HashMap;

 

import javax.ejb.ActivationConfigProperty;

import javax.ejb.MessageDriven;

import javax.ejb.TransactionAttribute;

import javax.ejb.TransactionAttributeType;

 

import javax.jms.Message;

import javax.jms.MessageListener;

import javax.jms.ObjectMessage;

import com.yaxon.etl.core.JobUtil;

import com.yaxon.etl.pojo.JobPara;

 

@MessageDriven(activationConfig = {

 

@ActivationConfigProperty(propertyName = "destinationType",

 

propertyValue = "javax.jms.Queue"),

 

@ActivationConfigProperty(propertyName = "destination",

 

propertyValue = "queue/DoJobMB") })

public class DoJobMB implements MessageListener {

 

    @Override

    @TransactionAttribute(TransactionAttributeType.NEVER)

    public void onMessage(Message msg) {

       ObjectMessage oMsg = (ObjectMessage) msg;

       try {

           JobPara para = (JobPara) oMsg.getObject();

           JobUtil.execute(para);

       } catch (Exception e) {       

           e.printStackTrace();

       }     

 

    }

 

}

由于etl操作数据量大，操作时间比较长，一般采用异步方式，不建议采用容器的事务。

ETL嵌入式代码

import java.text.SimpleDateFormat;

import java.util.Calendar;

import java.util.Date;

 

 

import org.pentaho.di.core.Result;

import org.pentaho.di.core.database.Database;

import org.pentaho.di.core.exception.KettleException;

import org.pentaho.di.core.exception.KettleJobException;

import org.pentaho.di.core.logging.LogWriter;

 

import org.pentaho.di.core.util.EnvUtil;

import org.pentaho.di.job.Job;

import org.pentaho.di.job.JobEntryLoader;

import org.pentaho.di.job.JobMeta;

import org.pentaho.di.kitchen.Messages;

 

import org.pentaho.di.repository.RepositoriesMeta;

import org.pentaho.di.repository.Repository;

import org.pentaho.di.repository.RepositoryDirectory;

import org.pentaho.di.repository.RepositoryMeta;

import org.pentaho.di.repository.UserInfo;

import org.pentaho.di.trans.StepLoader;

import com.yaxon.etl.pojo.JobPara;

 

public  class JobUtil {

   

    public static final String STRING_StepLoader = "StepLoader";

    public static final String STRING_Repository = "Repository";

    private static final String rensName = "kettle"; 

   

    private static StepLoader stepLoader;

    private static Repository repository ;

  

    static{

       

//     System.setProperty("KETTLE_HOME", JobUtil.class.getResource("")+"/META-INF");

//     System.out.println(JobUtil.class.getResource("")+"/META-INF");

       init();

    }

    public static void init() {

       String STRING_KITCHEN = "YaxonETL init";

       LogWriter log = LogWriter.getInstance(LogWriter.LOG_LEVEL_BASIC);

      

       EnvUtil.environmentInit();

       try {

           StepLoader.init();

       } catch (KettleException e) {

           log.logError(STRING_KITCHEN, Messages

                  .getString("Kitchen.Error.LoadingSteps"), e);

 

           return ;

       }

       stepLoader = StepLoader.getInstance();

       /* Load the plugins etc. */

       try {

           JobEntryLoader.init();

       } catch (KettleException e) {

           log.logError(STRING_KITCHEN, Messages

                  .getString("Kitchen.Error.LoadingJobEntries"), e);

           return ;

       }

       if (log.isDebug())

           log.logDebug(STRING_KITCHEN, Messages

                  .getString("Kitchen.Log.AllocateNewJob"));

 

       UserInfo userinfo = null;

      

       RepositoryMeta repinfo = null;

 

       try {

           RepositoriesMeta repsinfo = new RepositoriesMeta(log);

           repsinfo.readData();

           repinfo = repsinfo.findRepository(rensName);

           repository = new Repository(log, repinfo, userinfo);

           repository.connect("YaxonETL");

 

       } catch (Exception e) {

           System.out.println(Messages.getString("Kitchen.Error.StopProcess",

                  e.getMessage()));

           return ;

       }

      

    }

 

    public static void execute(JobPara para) throws KettleException {

       String STRING_KITCHEN="YaxonETL do job ["+para.getJob()+"]"+",Task id= "+para.getTask()+"Thread id="+Thread.currentThread().getId();

      

       LogWriter log = LogWriter.getInstance(LogWriter.LOG_LEVEL_BASIC);

       RepositoryDirectory directory = null;

       JobMeta jobMeta = new JobMeta(log);

       Job job = null;

       Date start, stop;

       Calendar cal;

       SimpleDateFormat df = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS");

       cal = Calendar.getInstance();

       start = cal.getTime();

       Result result = null;

       try {

           if (para.getDir() == null)

              directory = repository.getDirectoryTree(); // Default

           else

              directory = repository.getDirectoryTree().findDirectory(

                     para.getDir());

           jobMeta = new JobMeta(log, repository, para.getJob(), directory);

           job = new Job(log, stepLoader, repository, jobMeta);

           if (para.getPara() == null || para.getPara().length == 0) {

              job.getJobMeta().setArguments(null);

           } else {

              job.getJobMeta().setArguments(para.getPara());

           }

           job.initializeVariablesFrom(null);

           job.getJobMeta().setInternalKettleVariables(job);

           job.copyParametersFrom(job.getJobMeta());

           if (para.getOptionParams() != null) {

              String[] jobParams = job.listParameters();

              for (String param : jobParams) {

                  Object value = para.getOptionParams().get(param);

                  if (value != null) {

                     job.setParameterValue(param, value.toString());

                  }

              }

           }

           job.activateParameters();

           result = job.execute(); // Execute the selected job.

           job.endProcessing(Database.LOG_STATUS_END, result); // The

           // bookkeeping...

       } catch (KettleJobException je) {

           if (result == null) {

              result = new Result();

           }

           result.setNrErrors(1L);

 

           try {

              job.endProcessing("error", result);

           } catch (KettleJobException je2) {

              log.logError(job.getJobname(), Messages.getString(

                     "Kitchen.Error.SeriousError", je2.getMessage()));

              log.logError(job.getJobname(), Messages.getString(

                     "Kitchen.Error.SeriousError", je.getMessage()));

 

           }

       }

       log.logMinimal(STRING_KITCHEN, Messages

              .getString("Kitchen.Log.Finished"));

 

       if (result != null && result.getNrErrors() != 0) {

           log.logError(STRING_KITCHEN, Messages

                  .getString("Kitchen.Error.FinishedWithErrors"));

 

       }

       cal = Calendar.getInstance();

       stop = cal.getTime();

       String begin = df.format(start).toString();

       String end = df.format(stop).toString();

 

       log.logMinimal(STRING_KITCHEN, Messages.getString(

              "Kitchen.Log.StartStop", begin, end));

 

       long seconds = (stop.getTime() - start.getTime()) / 1000;

       if (seconds <= 60) {

           log.logMinimal(STRING_KITCHEN, Messages.getString(

                  "Kitchen.Log.ProcessEndAfter", String.valueOf(seconds)));

       } else if (seconds <= 60 * 60) {

           int min = (int) (seconds / 60);

           int rem = (int) (seconds % 60);

           log.logMinimal(STRING_KITCHEN, Messages.getString(

                  "Kitchen.Log.ProcessEndAfterLong", String.valueOf(min),

                  String.valueOf(rem), String.valueOf(seconds)));

       } else if (seconds <= 60 * 60 * 24) {

           int rem;

           int hour = (int) (seconds / (60 * 60));

           rem = (int) (seconds % (60 * 60));

           int min = rem / 60;

           rem = rem % 60;

           log.logMinimal(STRING_KITCHEN, Messages.getString(

                  "Kitchen.Log.ProcessEndAfterLonger", String.valueOf(hour),

                  String.valueOf(min), String.valueOf(rem), String

                         .valueOf(seconds)));

       } else {

           int rem;

           int days = (int) (seconds / (60 * 60 * 24));

           rem = (int) (seconds % (60 * 60 * 24));

           int hour = rem / (60 * 60);

           rem = rem % (60 * 60);

           int min = rem / 60;

           rem = rem % 60;

           log.logMinimal(STRING_KITCHEN, Messages.getString(

                  "Kitchen.Log.ProcessEndAfterLongest", String.valueOf(days),

                  String.valueOf(hour), String.valueOf(min), String

                         .valueOf(rem), String.valueOf(seconds)));

       }

 

    }

   public static void disconnectRepository(){

      repository.disconnect();

   }