使用RawComparator加速Hadoop程序

 

http://yoyzhou.github.io/blog/2013/05/13/hadoop-write-ur-own-rawcomparator/

 

	static class MyComparator extends WritableComparator {
		static {
			WritableComparator.define(MyWritable.class, new MyComparator());
		}

		protected MyComparator() {
			super(MyWritable.class);
		}

		public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {

			int cmp = 1;
			//determine how many bytes the first VLong takes
			int n1 = WritableUtils.decodeVIntSize(b1[s1]);
			int n2 = WritableUtils.decodeVIntSize(b2[s2]);

			try {
				//read value from VLongWritable byte array
				long l11 = readVLong(b1, s1);
				long l21 = readVLong(b2, s2);

				cmp = l11 > l21 ? 1 : (l11 == l21 ? 0 : -1);
				if (cmp != 0) {

					return cmp;

				} else {

					long l12 = readVLong(b1, s1 + n1);
					long l22 = readVLong(b2, s2 + n2);
					return cmp = l12 > l22 ? 1 : (l12 == l22 ? 0 : -1);
				}
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		}

	}

	static class MyWritable extends BinaryComparable implements WritableComparable<BinaryComparable> {

		@Override
		public void write(DataOutput out) throws IOException {
			out.write("hello".getBytes());
		}

		@Override
		public void readFields(DataInput in) throws IOException {
			String str = WritableUtils.readString(in);
			System.out.println(str);
		}

		@Override
		public int getLength() {
			return 0;
		}

		@Override
		public byte[] getBytes() {
			return null;
		}

	}