自定义分区器插件-案例

效果示例

意图：实现一个使用String字段哈希值进行分区的分区器

处理过程展示：
配置schema分两个分区

原始数据：

处理流程：

输出文件：

步骤执行过程：

实现配置对话框

编写实现类继承BaseStepDialog，实现StepDialogInterface接口
对话框效果：

  private static final Class<?> PKG = HashPartitionerDialog.class;

  private StepPartitioningMeta partitioningMeta;
  private StepMeta stepMeta;
  private HashPartitioner partitioner;
  private String fieldName;

  private Label wlFieldname;
  private CCombo wFieldname;
  private FormData fdlFieldname, fdFieldname;

  public HashPartitionerDialog(Shell parent, StepMeta stepMeta, StepPartitioningMeta partitioningMeta, TransMeta transMeta ) {
    super( parent, (BaseStepMeta) stepMeta.getStepMetaInterface(), transMeta, stepMeta.getName());//partitioningMeta.getPartitioner().getDescription() );
    this.stepMeta = stepMeta;
    this.partitioningMeta = partitioningMeta;
    partitioner = (HashPartitioner) partitioningMeta.getPartitioner();
    fieldName = partitioner.getFieldName();
  }

  public String open() {
    Shell parent = getParent();
    Display display = parent.getDisplay();

    shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX );
    props.setLook( shell );
    setShellImage( shell, stepMeta.getStepMetaInterface() );

    ModifyListener lsMod = new ModifyListener() {
      public void modifyText( ModifyEvent e ) {
        partitioningMeta.hasChanged( true );
      }
    };
    changed = partitioningMeta.hasChanged();

    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;

    shell.setLayout( formLayout );
    shell.setText( partitioner.getDescription() );

    int margin = Const.MARGIN;

    int middle = props.getMiddlePct();

    wlFieldname = new Label( shell, SWT.RIGHT );
    wlFieldname.setText( "Fieldname" );
    props.setLook( wlFieldname );
    fdlFieldname = new FormData();
    fdlFieldname.left = new FormAttachment( 0, 0 );
    fdlFieldname.right = new FormAttachment( middle, -margin );
    fdlFieldname.top = new FormAttachment( 0, margin );
    wlFieldname.setLayoutData( fdlFieldname );
    wFieldname = new CCombo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    wFieldname.setText( fieldName == null ? "" : fieldName );
    props.setLook( wFieldname );
    wFieldname.addModifyListener( lsMod );
    fdFieldname = new FormData();
    fdFieldname.left = new FormAttachment( middle, 0 );
    fdFieldname.top = new FormAttachment( 0, margin );
    fdFieldname.right = new FormAttachment( 100, 0 );
    wFieldname.setLayoutData( fdFieldname );

    try {
      RowMetaInterface inputFields = transMeta.getPrevStepFields( stepMeta );
      if ( inputFields != null ) {
        String[] fieldNames = inputFields.getFieldNames();
        Arrays.sort( fieldNames );
        wFieldname.setItems( fieldNames );
      }
    } catch ( KettleStepException e ) {
      new ErrorDialog( shell, "Error", "Error obtaining list of input fields:", e );
    }

    // Some buttons
    wOK = new Button( shell, SWT.PUSH );
    wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) ); //$NON-NLS-1$
    wCancel = new Button( shell, SWT.PUSH );
    wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) ); //$NON-NLS-1$
    fdOK = new FormData();

    setButtonPositions(  new Button[] { wOK, wCancel }, margin, null );

    // Add listeners
    lsCancel = new Listener() {
      public void handleEvent( Event e ) {
        cancel();
      }
    };
    lsOK = new Listener() {
      public void handleEvent( Event e ) {
        ok();
      }
    };

    wCancel.addListener( SWT.Selection, lsCancel );
    wOK.addListener( SWT.Selection, lsOK );

    lsDef = new SelectionAdapter() {
      public void widgetDefaultSelected( SelectionEvent e ) {
        ok();
      }
    };

    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener(
      new ShellAdapter() {
        public void shellClosed( ShellEvent e ) {
          cancel();
        }
      }
    );

    // Set the shell size, based upon previous time...
    setSize();
    getData();
    partitioningMeta.hasChanged( changed );

    setSize();

    shell.open();
    while ( !shell.isDisposed() ) {
      if ( !display.readAndDispatch() ) {
        display.sleep();
      }
    }
    return stepname;
  }

  public void getData() {
    wFieldname.setText( fieldName == null ? "" : fieldName );
  }

  private void cancel() {
    stepname = null;
    partitioningMeta.hasChanged( changed );
    dispose();
  }

  private void ok() {
    fieldName = wFieldname.getText();
    partitioner.setFieldName( fieldName );
    dispose();
  }

实现Partitioner

编写实现类继承BasePartitioner，实现Partitioner接口
再使用PartitionerPlugin 注解这个实现类

@PartitionerPlugin (
    id = "HashPartitioner",
    name = "Hash Partitioner",
    description = "Laozhang's Partition by Hash Code of String field"
  )

  private static final Class<?> PKG = HashPartitionerDialog.class;
  
  private String fieldName;
  
  protected int partitionColumnIndex = -1;

  public HashPartitioner() {
    super();
  }

  public Partitioner getInstance() {
    Partitioner partitioner = new HashPartitioner();
    partitioner.setId( getId() );
    partitioner.setDescription( getDescription() );
    return partitioner;
  }

  public HashPartitioner clone() {
    HashPartitioner hashPartitioner = (HashPartitioner) super.clone();
    hashPartitioner.fieldName = fieldName;

    return hashPartitioner;
  }

  public String getDialogClassName() {
    return HashPartitionerDialog.class.getName();
  }

  public int getPartition( RowMetaInterface rowMeta, Object[] row ) throws KettleException {

    // 第一步应该调用init(),父类BasePartitioner中的属性才能被初始化
    init( rowMeta );

    // 确定分区号
    if ( partitionColumnIndex < 0 ) {
      partitionColumnIndex = rowMeta.indexOfValue( fieldName );
      if ( partitionColumnIndex < 0 ) {
        throw new KettleStepException( "Unable to find partitioning field name [" + fieldName + "] in the output row..." + rowMeta );
      }
    }

    // 获取被分区的字段值
    String partitionColumnValue = rowMeta.getString( row, partitionColumnIndex );
    int hash = partitionColumnValue.hashCode();
    //根据字段值的hash code 计算出归属分区号
    return (hash & 0x7FFFFFFF) % nrPartitions;
  }

  public String getDescription() {
    String description = "Laozhang's Partition by Hash Code of String field";
    if ( !Const.isEmpty( fieldName ) ) {
      description += " (" + fieldName + ")";
    }
    return description;
  }

  public String getXML() {
    StringBuilder xml = new StringBuilder();
    xml.append( "           " ).append( XMLHandler.addTagValue( "field_name", fieldName ) );
    return xml.toString();
  }

  public void loadXML( Node partitioningMethodNode ) throws KettleXMLException {
    fieldName = XMLHandler.getTagValue( partitioningMethodNode, "field_name" );
  }

  public void saveRep( Repository rep, ObjectId id_transformation, ObjectId id_step ) throws KettleException {
    rep.saveStepAttribute( id_transformation, id_step, "PARTITIONING_FIELDNAME", fieldName );
  }

  public void loadRep( Repository rep, ObjectId id_step ) throws KettleException {
    fieldName = rep.getStepAttributeString( id_step, "PARTITIONING_FIELDNAME" );
  }

  public String getFieldName() {
    return fieldName;
  }

  public void setFieldName( String fieldName ) {
    this.fieldName = fieldName;
  }