效果示例
意图:实现一个使用String字段哈希值进行分区的分区器
处理过程展示:
配置schema分两个分区
原始数据:
处理流程:
输出文件:
步骤执行过程:
实现配置对话框
编写实现类继承BaseStepDialog,实现StepDialogInterface接口
对话框效果:
private static final Class<?> PKG = HashPartitionerDialog.class;
private StepPartitioningMeta partitioningMeta;
private StepMeta stepMeta;
private HashPartitioner partitioner;
private String fieldName;
private Label wlFieldname;
private CCombo wFieldname;
private FormData fdlFieldname, fdFieldname;
public HashPartitionerDialog(Shell parent, StepMeta stepMeta, StepPartitioningMeta partitioningMeta, TransMeta transMeta ) {
super( parent, (BaseStepMeta) stepMeta.getStepMetaInterface(), transMeta, stepMeta.getName());//partitioningMeta.getPartitioner().getDescription() );
this.stepMeta = stepMeta;
this.partitioningMeta = partitioningMeta;
partitioner = (HashPartitioner) partitioningMeta.getPartitioner();
fieldName = partitioner.getFieldName();
}
public String open() {
Shell parent = getParent();
Display display = parent.getDisplay();
shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX );
props.setLook( shell );
setShellImage( shell, stepMeta.getStepMetaInterface() );
ModifyListener lsMod = new ModifyListener() {
public void modifyText( ModifyEvent e ) {
partitioningMeta.hasChanged( true );
}
};
changed = partitioningMeta.hasChanged();
FormLayout formLayout = new FormLayout();
formLayout.marginWidth = Const.FORM_MARGIN;
formLayout.marginHeight = Const.FORM_MARGIN;
shell.setLayout( formLayout );
shell.setText( partitioner.getDescription() );
int margin = Const.MARGIN;
int middle = props.getMiddlePct();
wlFieldname = new Label( shell, SWT.RIGHT );
wlFieldname.setText( "Fieldname" );
props.setLook( wlFieldname );
fdlFieldname = new FormData();
fdlFieldname.left = new FormAttachment( 0, 0 );
fdlFieldname.right = new FormAttachment( middle, -margin );
fdlFieldname.top = new FormAttachment( 0, margin );
wlFieldname.setLayoutData( fdlFieldname );
wFieldname = new CCombo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
wFieldname.setText( fieldName == null ? "" : fieldName );
props.setLook( wFieldname );
wFieldname.addModifyListener( lsMod );
fdFieldname = new FormData();
fdFieldname.left = new FormAttachment( middle, 0 );
fdFieldname.top = new FormAttachment( 0, margin );
fdFieldname.right = new FormAttachment( 100, 0 );
wFieldname.setLayoutData( fdFieldname );
try {
RowMetaInterface inputFields = transMeta.getPrevStepFields( stepMeta );
if ( inputFields != null ) {
String[] fieldNames = inputFields.getFieldNames();
Arrays.sort( fieldNames );
wFieldname.setItems( fieldNames );
}
} catch ( KettleStepException e ) {
new ErrorDialog( shell, "Error", "Error obtaining list of input fields:", e );
}
// Some buttons
wOK = new Button( shell, SWT.PUSH );
wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) ); //$NON-NLS-1$
wCancel = new Button( shell, SWT.PUSH );
wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) ); //$NON-NLS-1$
fdOK = new FormData();
setButtonPositions( new Button[] { wOK, wCancel }, margin, null );
// Add listeners
lsCancel = new Listener() {
public void handleEvent( Event e ) {
cancel();
}
};
lsOK = new Listener() {
public void handleEvent( Event e ) {
ok();
}
};
wCancel.addListener( SWT.Selection, lsCancel );
wOK.addListener( SWT.Selection, lsOK );
lsDef = new SelectionAdapter() {
public void widgetDefaultSelected( SelectionEvent e ) {
ok();
}
};
// Detect X or ALT-F4 or something that kills this window...
shell.addShellListener(
new ShellAdapter() {
public void shellClosed( ShellEvent e ) {
cancel();
}
}
);
// Set the shell size, based upon previous time...
setSize();
getData();
partitioningMeta.hasChanged( changed );
setSize();
shell.open();
while ( !shell.isDisposed() ) {
if ( !display.readAndDispatch() ) {
display.sleep();
}
}
return stepname;
}
public void getData() {
wFieldname.setText( fieldName == null ? "" : fieldName );
}
private void cancel() {
stepname = null;
partitioningMeta.hasChanged( changed );
dispose();
}
private void ok() {
fieldName = wFieldname.getText();
partitioner.setFieldName( fieldName );
dispose();
}
实现Partitioner
编写实现类继承BasePartitioner,实现Partitioner接口
再使用PartitionerPlugin 注解这个实现类
@PartitionerPlugin (
id = "HashPartitioner",
name = "Hash Partitioner",
description = "Laozhang's Partition by Hash Code of String field"
)
private static final Class<?> PKG = HashPartitionerDialog.class;
private String fieldName;
protected int partitionColumnIndex = -1;
public HashPartitioner() {
super();
}
public Partitioner getInstance() {
Partitioner partitioner = new HashPartitioner();
partitioner.setId( getId() );
partitioner.setDescription( getDescription() );
return partitioner;
}
public HashPartitioner clone() {
HashPartitioner hashPartitioner = (HashPartitioner) super.clone();
hashPartitioner.fieldName = fieldName;
return hashPartitioner;
}
public String getDialogClassName() {
return HashPartitionerDialog.class.getName();
}
public int getPartition( RowMetaInterface rowMeta, Object[] row ) throws KettleException {
// 第一步应该调用init(),父类BasePartitioner中的属性才能被初始化
init( rowMeta );
// 确定分区号
if ( partitionColumnIndex < 0 ) {
partitionColumnIndex = rowMeta.indexOfValue( fieldName );
if ( partitionColumnIndex < 0 ) {
throw new KettleStepException( "Unable to find partitioning field name [" + fieldName + "] in the output row..." + rowMeta );
}
}
// 获取被分区的字段值
String partitionColumnValue = rowMeta.getString( row, partitionColumnIndex );
int hash = partitionColumnValue.hashCode();
//根据字段值的hash code 计算出归属分区号
return (hash & 0x7FFFFFFF) % nrPartitions;
}
public String getDescription() {
String description = "Laozhang's Partition by Hash Code of String field";
if ( !Const.isEmpty( fieldName ) ) {
description += " (" + fieldName + ")";
}
return description;
}
public String getXML() {
StringBuilder xml = new StringBuilder();
xml.append( " " ).append( XMLHandler.addTagValue( "field_name", fieldName ) );
return xml.toString();
}
public void loadXML( Node partitioningMethodNode ) throws KettleXMLException {
fieldName = XMLHandler.getTagValue( partitioningMethodNode, "field_name" );
}
public void saveRep( Repository rep, ObjectId id_transformation, ObjectId id_step ) throws KettleException {
rep.saveStepAttribute( id_transformation, id_step, "PARTITIONING_FIELDNAME", fieldName );
}
public void loadRep( Repository rep, ObjectId id_step ) throws KettleException {
fieldName = rep.getStepAttributeString( id_step, "PARTITIONING_FIELDNAME" );
}
public String getFieldName() {
return fieldName;
}
public void setFieldName( String fieldName ) {
this.fieldName = fieldName;
}