zookeeper Unreasonable length错误导致无法启动

现象： 启动报错

Exception in thread "main" java.io.IOException: Unreasonable length = 1860320
	at org.apache.jute.BinaryInputArchive.checkLength(BinaryInputArchive.java:127)
	at org.apache.jute.BinaryInputArchive.readBuffer(BinaryInputArchive.java:92)
	at com.sf.boot.code.Zk4lTest.testLog(Zk4lTest.java:241)
	at com.sf.boot.code.Zk4lTest.main(Zk4lTest.java:392)

原因：

• zookeeper启动的时候会加载datalog(快照日志)进行内存数据恢复，zookeeper的快照日志是按照行记录的，在读取的时候会判断record的大小是否大于jute.maxbuffer(默认1M).
• 在写入数据的时候也是会判断节点数据是否大于jute.maxbuffer，那么这样的数据是如何写入快照日志的？答案是closeSession的时候
• 当一个客户端建立了很多的临时节点时，在进行closeSession的时候会把所有的临时节点的path集中在一起写到快照日志的record(因为closeSession时是需要删除所有临时节点的)。
  这时候如果path很长很多，那么就有可能超过jute.maxbuffer。例如:dubbo的场景，dubbo是把所有注册数据都当成path,这就导致path很长

解决方案：

• zookeeper启动的时候设置 jvm参数大小 -Djute.maxbuffer
• 删除zookeeper有问题的快照日志文件，不过这样做会导致还没有持久化的数据丢失。zookeeper是写快照日志，当快照日志达到一定条数才会落盘持久化

解析快照日志文件
可以使用org.apache.zookeeper.server.LogFormatter 解析快照日志。
下面的方法是copy LogFormatter的代码修改了一下

/**
     * 测试快照日志是否正常
     * @param filePath
     * @param errorFilePath
     * @param byteLength
     * @throws Exception
     */
    public static void testLog(String filePath,String errorFilePath,long byteLength) throws Exception {
        FileInputStream fis = new FileInputStream(filePath);
        BinaryInputArchive logStream = BinaryInputArchive.getArchive(fis);
        FileHeader fhdr = new FileHeader();
        fhdr.deserialize(logStream, "fileheader");

        int count = 0;
        int length = 0;
        while (true) {
            long crcValue = 0;
            byte[] bytes;
            try {
                crcValue = logStream.readLong("crcvalue");

                bytes = logStream.readBuffer("txnEntry");
                length += bytes.length;
            } catch (Exception e) {
                System.out.println("11EOF reached after " + count + " txns. crcValue:"+crcValue);
                throw e;
            }
            if (bytes.length > byteLength){
                File errorFile = new File(errorFilePath+"."+count);
                FileUtils.write(errorFile,new String(bytes));
            }
//            System.out.println(crcValue);
            if (bytes.length == 0) {
                // Since we preallocate, we define EOF to be an
                // empty transaction
                System.out.println("22EOF reached after " + count + " txns.");
                System.out.println(length);
                return;
            }
            Checksum crc = new Adler32();
            crc.update(bytes, 0, bytes.length);
            if (crcValue != crc.getValue()) {
                throw new IOException("CRC doesn't match " + crcValue +
                        " vs " + crc.getValue());
            }
            TxnHeader hdr = new TxnHeader();
            Record txn = SerializeUtils.deserializeTxn(bytes, hdr);
            if (hdr.getType() == -11){
                System.out.println();
                System.out.println("count:"+count+"byte:"+bytes.length);
                System.out.println("原始数据:"+new String(bytes));
                System.out.println(DateFormat.getDateTimeInstance(DateFormat.SHORT,
                        DateFormat.LONG).format(new Date(hdr.getTime()))
                        + " session 0x"
                        + Long.toHexString(hdr.getClientId())
                        + " cxid 0x"
                        + Long.toHexString(hdr.getCxid())
                        + " zxid 0x"
                        + Long.toHexString(hdr.getZxid())
                        + " type:" + hdr.getType() + " " + JSON.toJSONString(txn));
            }

            if (logStream.readByte("EOR") != 'B') {
//                LOG.error("Last transaction was partial.");
                throw new EOFException("Last transaction was partial.");
            }
            count++;
        }
    }

测试该错误的代码

    public static String getPath(String path){
        Random random = new Random();
        return path+random.nextLong();
    }

    public static CuratorFramework testLocalZk() throws Exception {
        CuratorFramework curatorFramework = getInstance("127.0.0.1:2181");
//        createNode(curatorFramework,CreateMode.PERSISTENT,"/a","a");
        String path = "/dubbo/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService/consumers/consumer192.168.255.10/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService?application=sgs-kafka-gw&category=consumers&check=false&dubbo=sf.2.0.4&group=ENV3_2&interface=com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService&methods=deliveryQuickRepairDataProcess,fvpUnpackMatchRespMsgProcess,deliverySdsDynamicReimburseProcess,deliveryTaskMsgProcess,deliveryOrderMsgProcess,wantedDeliveryTask,deliveryOrderScstmsMsgProcess,saveWaybillRemark2Sisp,imageGatewayUploadInfoMsgProcess,deliveryMatchEmployeeNotify,barCountDownProcess,deliveryCXSignConfirm,deliveryExtraTaskStatusProcess,storeOneKeyAcceptProcess,queryWaybillInfoMsgProcess,deliveryDetainPredictionProcess,deliveryChangeOmsMsgProcess,deliveryCusChgResProcess,paySuccessCallback,batchDeliveryExpressDeliveredNotify,deliveryOrderOmsOperationMsgProcess,deliverySpecialConfigMsgProcess,processPisLimitTimeMsg,deliveryDestDeptUnloadMsgProcess,deliveryAppointRespProcess,deliveryOrderOmsMsgProcess,processBigDataWaybillLabel,deliveryDdsHandoverNotify,deliveryAppointSelfPickupProcess,deliverySdsSortDynamicTypeProcess,hhtDeliveryBarRecordMsgProcess,deliveryExtraTaskInfoProcess,deliveryOrderEosfmstmsMsgProcess,expressStatusMsgProcess,handoverAsyncOperation,deliveryWiInstructionsMsgProcess,uploadImageInfoToWbepAsyncMsgProcess,deliveryChangeMsgProcess,hwDeliveryTimeMsgProcess,deliveryPackageHandoverAsyncMsgProcess,deliveryOrderUniPayStatusMsgProcess,deliveryOrderPayStatusMsgProcess,deliveryDelayWaybillDetailMsgProcess,deliveryPackage,transferSorterProcess,sfNewCardPayStatusMsgProcess,deliveryInsCallOutInfoMsgProcess,deliveryUploadImageAsyncMsgProcess,waybillExceptionMsgProcess&owner=morly&pid=11844&revision=19.2-20210407.102748-18&side=consumer&timeout=10000&timestamp=1617898909739";
//        String getPath(path) = "/dubbo/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService/consumers/consumer192.168.255.10/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService?application=sgs-kafka-gw&category=consumers&check=false&dubbo=sf.2.0.4&group=ENV3_2&interface=com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService&methods=deliveryQuickRepairDataProcess,fvpUnpackMatchRespMsgProcess,deliverySdsDynamicReimburseProcess,deliveryTaskMsgProcess,deliveryOrderMsgProcess,wantedDeliveryTask,deliveryOrderScstmsMsgProcess,saveWaybillRemark2Sisp,imageGatewayUploadInfoMsgProcess,deliveryMatchEmployeeNotify,barCountDownProcess,deliveryCXSignConfirm,deliveryExtraTaskStatusProcess,storeOneKeyAcceptProcess,queryWaybillInfoMsgProcess,deliveryDetainPredictionProcess,deliveryChangeOmsMsgProcess,deliveryCusChgResProcess,paySuccessCallback,batchDeliveryExpressDeliveredNotify,deliveryOrderOmsOperationMsgProcess,deliverySpecialConfigMsgProcess,processPisLimitTimeMsg,deliveryDestDeptUnloadMsgProcess,deliveryAppointRespProcess,deliveryOrderOmsMsgProcess,processBigDataWaybillLabel,deliveryDdsHandoverNotify,deliveryAppointSelfPickupProcess,deliverySdsSortDynamicTypeProcess,hhtDeliveryBarRecordMsgProcess,deliveryExtraTaskInfoProcess,deliveryOrderEosfmstmsMsgProcess,expressStatusMsgProcess,handoverAsyncOperation,deliveryWiInstructionsMsgProcess,uploadImageInfoToWbepAsyncMsgProcess,deliveryChangeMsgProcess,hwDeliveryTimeMsgProcess,deliveryPackageHandoverAsyncMsgProcess,deliveryOrderUniPayStatusMsgProcess,deliveryOrderPayStatusMsgProcess,deliveryDelayWaybillDetailMsgProcess,deliveryPackage,transferSorterProcess,sfNewCardPayStatusMsgProcess,deliveryInsCallOutInfoMsgProcess,deliveryUploadImageAsyncMsgProcess,waybillExceptionMsgProcess&owner=morly&pid=11844&revision=19.2-20210407.102748-18&side=consumer&timeout=10000&timestamp=1617900833708";
//        createNode(curatorFramework,CreateMode.PERSISTENT,"/dubbo/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService/consumers/consumer127.0.0.1");
        for (int i=0;i<1000;i++){
            createNode(curatorFramework,CreateMode.EPHEMERAL,getPath(path)+"-"+i,path);
        }
        return curatorFramework;
    }