limax线程池与锁-Zdb

ZDB包括事物，锁，线程池等多个模块

1.zdb随着进程开启而启动

public final synchronized void start(limax.xmlgen.Zdb meta) {
    if (DBC.isRunning())
        throw new IllegalAccessError("DBC is in use.");
    this.meta = testMeta(meta);
    Set<String> unusedTables = MetaUtils.testAndTrySaveToDbAndReturnUnusedTables(meta);
    if (!isStartedOnce) {
        Runtime.getRuntime().addShutdownHook(new Thread(Zdb.this::stop, "limax.zdb.ShutdownHook"));
        isStartedOnce = true;
    }
    Trace.fatal("zdb start begin");
    mbeans = MBeans.register(MBeans.root(), this, "limax.zdb:type=Zdb,name=Zdb");
    try {
        //开启多种类型的线程池
        ConcurrentEnvironment env = ConcurrentEnvironment.getInstance();
        if (useFixedThreadPool) {
            env.newFixedThreadPool("limax.zdb.core", meta.getCorePoolSize());
            env.newFixedThreadPool("limax.zdb.procedure", meta.getProcPoolSize());
        } else {
            env.newThreadPool("limax.zdb.core", meta.getCorePoolSize());
            env.newThreadPool("limax.zdb.procedure", meta.getProcPoolSize());
        }
        scheduledExecutorService = env.newScheduledThreadPool("limax.zdb.scheduler", meta.getSchedPoolSize());
        coreExecutor = env.newTimeoutExecutor("limax.zdb.core", meta.getProcedure().getMaxExecutionTime(),
                TimeUnit.MILLISECONDS);
        lockEnvironment = new LockEnvironment(() -> meta.getDeadlockDetectPeriod());
        ScheduledThreadPoolExecutor checkpointScheduler = env.newScheduledThreadPool("limax.zdb.checkpoint", 1);
        procedureExecutors = new ConcurrentHashMap<>();
        //Lockeys管理锁
        Class.forName("limax.zdb.Lockeys");
        //Xbean，_Tables_ 初始化
        Class.forName("limax.zdb.XBean");
        tableClass = Class.forName("table._Tables_");
        Constructor<?> constructor = tableClass.getDeclaredConstructor();
        constructor.setAccessible(true);
        table = constructor.newInstance();
        tables = new Tables();
        Field[] fields = tableClass.getDeclaredFields();
        AccessibleObject.setAccessible(fields, true);
        //初始化tables
        for (Field field : fields) {
            Object fieldValue = field.get(table);
            if (fieldValue instanceof AbstractTable)
                tables.add((AbstractTable) fieldValue);
        }
        tables.open(meta, unusedTables);
        tables.getLogger().dropTables(unusedTables.toArray(new String[unusedTables.size()]));
        //checkpoint初始化，用于数据定时刷新存储
        checkpoint = new Checkpoint(checkpointScheduler, tables);
        String trnHome = meta.getTrnHome();
        if (!trnHome.isEmpty()) {
            Path path = Paths.get(trnHome);
            Files.createDirectories(path);
            duration = new Duration(path, tables, () -> checkpoint.checkpoint(meta));
        }
        ManagementFactory.setCheckpointMBean(checkpoint);
        Trace.fatal("zdb start end");
        running = true;
    } catch (Throwable e) {
        Trace.fatal("Zdb start error", e);
        close();
        throw e instanceof XError ? (XError) e : new XError(e);
    }
}

2.Tables 持有所有表的引用（存储storages）

void open(limax.xmlgen.Zdb meta, Set<String> unusedTables) {
    if (null != logger)
        throw new XError("tables opened");
    switch (meta.getEngineType()) {
    case MYSQL:
        logger = new LoggerMysql(meta);
        break;
    case EDB:
        logger = new LoggerEdb(meta);
        break;
    }
    storages.add(tableSys.open(null, logger));
    tableSys.getAutoKeys().remove(unusedTables);
    Map<String, Integer> map = new HashMap<>();
    AtomicInteger idAlloc = new AtomicInteger();
    for (AbstractTable table : tables.values()) {
        StorageInterface storage = table.open(meta.getTable(table.getName()), logger);
        if (storage != null)
            storages.add(storage);
        TTable<?, ?> ttable = (TTable<?, ?>) table;
        ttable.setLockId(map.computeIfAbsent(ttable.getLockName(), k -> idAlloc.incrementAndGet()));
    }
    add(tableSys);
}

3.TStorage 引用edb和mysql两种存储引擎，是增删改查的中间层

private final StorageEngine engine;

TStorage(TTable<K, V> table, LoggerEngine logger) {
    this.table = table;
    switch (Zdb.meta().getEngineType()) {
    case MYSQL:
        engine = new StorageMysql((LoggerMysql) logger, table.getName());
        break;
    case EDB:
        engine = new StorageEdb((LoggerEdb) logger, table.getName());
        break;
    default:
        throw new XError("unknown engine type");
    }
}

增删改查

final boolean exist(K key, TTable<K, V> table) {
    Lock lock = snapshotLock.readLock();
    lock.lock();
    try {
        TRecord<K, V> r = snapshot.get(key);
        if (r != null)
            return r.exist();
    } finally {
        lock.unlock();
    }
    return engine.exist(table.marshalKey(key));
}

V find(K key, TTable<K, V> table) {
    OctetsStream value;
    Lock lock = snapshotLock.readLock();
    lock.lock();
    try {
        TRecord<K, V> r = snapshot.get(key);
        if (r != null) {
            value = r.find();
            if (value == null)
                return null;
        } else {
            value = null;
        }
    } finally {
        lock.unlock();
    }
    try {
        if (value == null) {
            Octets _value = engine.find(table.marshalKey(key));
            if (_value == null)
                return null;
            value = OctetsStream.wrap(_value);
        }
        return table.unmarshalValue(value);
    } catch (MarshalException e) {
        throw new RuntimeException(e);
    }
}

4.StorageEdb edb存储引擎

private final LoggerEdb edb;
private final String tableName;

public StorageEdb(LoggerEdb edb, String tableName) {
    this.edb = edb;
    this.tableName = tableName;
    try {
        this.edb.getDatabase().addTable(new String[] { tableName });
    } catch (Exception e) {
        throw new XError(e);
    }
}

@Override
public boolean exist(Octets key) {
    byte[] tkey = key.capacity() == key.size() ? key.array() : Arrays.copyOf(key.array(), key.size());
    try {
        //下面是edb表操作逻辑
        return edb.getDatabase().exist(tableName, tkey);
    } catch (Exception e) {
        throw new XError(e);
    }
}

5.TTable 数据表key-value上层操作封装，专注数据部分

public abstract class TTable<K, V> extends AbstractTable implements TTableMBean {
    private String lockname;
    private int lockId;
    private limax.xmlgen.Table meta;
    private AutoKeys.AutoKey autoKey;
    private Resource mbean;

    protected TTable() {
    }

    protected final Long nextKey() {
        return autoKey.next();
    }

    void recoverKey(long key) {
        autoKey.recover(key);
    }

    protected final boolean add(K key, V value) {
        Objects.requireNonNull(key, "key is null");
        Objects.requireNonNull(value, "value is null");
        Lockey lockey = Lockeys.getLockey(lockId, key);
        //包括在事务中
        Transaction.current().wAddLockey(lockey);
        if (null != autoKey)
            autoKey.accept((Long) key);
        countAdd.incrementAndGet();
        TRecord<K, V> r = cache.get(key);
        if (null != r)
            return r.add(value);
        countAddMiss.incrementAndGet();
        if (_exist(key)) {
            countAddStorageMiss.incrementAndGet();
            return false;
        }
        //将操作封装成一个TRecord
        cache.add(key, new TRecord<K, V>(this, value, lockey, TRecord.State.ADD));
        return true;
    }

    protected final boolean remove(K key) {
        Objects.requireNonNull(key, "key is null");
        Transaction current = Transaction.current();
        Lockey lockey = Lockeys.getLockey(lockId, key);
        current.wAddLockey(lockey);
        current.removeCachedTRecord(this, key);
        countRemove.incrementAndGet();
        TRecord<K, V> r = cache.get(key);
        if (null != r)
            return r.remove();
        countRemoveMiss.incrementAndGet();
        boolean exists = _exist(key);
        if (!exists)
            countRemoveStorageMiss.incrementAndGet();
        cache.add(key, new TRecord<K, V>(this, null, lockey, exists ? State.INDB_REMOVE : State.REMOVE));
        return exists;
    }
}

6.TRecord记录，zdb对表的操作都抽象成一条数据

final class TRecord<K, V> extends XBean {
    private final TTable<K, V> table;
    private final Lockey lockey;

    TRecord(TTable<K, V> table, V value, Lockey lockey, State state) {
        super(null, RECORD_VARNAME);
        this.table = table;
        if (null != value)
            Logs.link(value, this, RECORD_VARNAME, State.INDB_GET != state);
        this.value = value;
        this.lockey = lockey;
        this.state = state;
    }

    private void _remove() {
        Logs.link(value, null, null);
        Transaction.currentSavepoint().addIfAbsent(getLogKey(), new LogAddRemove());
        value = null;
    }

    boolean remove() {
        switch (state) {
        case INDB_GET:
            _remove();
            state = State.INDB_REMOVE;
            return true;
        case INDB_ADD:
            _remove();
            state = State.INDB_REMOVE;
            return true;
        case ADD:
            _remove();
            state = State.REMOVE;
            return true;
        default:
            return false;
        }
    }

    private void _add(V value) {
        Logs.link(value, this, RECORD_VARNAME);
        Transaction.currentSavepoint().addIfAbsent(getLogKey(), new LogAddRemove());
        this.value = value;
    }
}

7.Procedure 线程处理，线程被封装成ProcedureImpl

public ProcedureFuture(ProcedureImpl<P> p) {
    this(p, null);
}

private void launch() {
    this.future = Zdb.procedureExecutor(p.maxExecutionTime()).submit(this, p);
}

public ProcedureFuture(ProcedureImpl<P> p, Procedure.Done<P> done) {
    this.p = p;
    this.done = done;
    launch();
}

private void done() {
    if (done != null)
        try {
            done.doDone(p.process, p);
        } catch (Throwable e) {
            if (Trace.isErrorEnabled())
                Trace.error("doDone", e);
        }
}

ProcedureImpl

执行process方法

boolean call() {
    //事务环境
    int savepoint = Transaction.savepoint();
    try {
        //执行process方法，事务环境被Table.add，remove等操作处理
        if (process.process()) {
            setSuccess(true);
            setException(null);
            return true;
        }
    } catch (Exception e) {
        setException(e);
        Trace.log(Zdb.pmeta().getTrace(), "Procedure execute", e);
    }
    //失败时回滚
    Transaction.rollback(savepoint);
    setSuccess(false);
    return false;
}

private static boolean perform(ProcedureImpl<?> p) {
    try {
        Transaction.current().perform(p);
    } catch (XDeadlock e) {
        return false;
    } catch (Throwable e) {
    }
    return true;
}

static <P extends Procedure> Procedure.Result call(P p) {
    ProcedureImpl<P> impl = new ProcedureImpl<>(p);
    if (Transaction.current() == null) {
        try {
            Transaction.create();
            for (int retry = 0;;) {
                if (perform(impl))
                    break;
                retry++;
                if (retry > impl.retryTimes())
                    break;
                if (retry == impl.retryTimes() && impl.retrySerial())
                    impl.isolation = Transaction.Isolation.LEVEL3;
                try {
                    Thread.sleep(impl.delay());
                } catch (InterruptedException e) {
                }
            }
        } finally {
            Transaction.destroy();
        }
    } else {
        impl.call();
    }
    return impl;
}

8.表的操作引入锁和事务，再次分析TTable

protected final boolean add(K key, V value) {
    Objects.requireNonNull(key, "key is null");
    Objects.requireNonNull(value, "value is null");
    //当前事务中锁操作，不同表代表不同的锁
    Lockey lockey = Lockeys.getLockey(lockId, key);
    Transaction.current().wAddLockey(lockey);
    if (null != autoKey)
        autoKey.accept((Long) key);
    countAdd.incrementAndGet();
    //从缓存中取数据，缓存策略包括：TTableCacheConcurrentMap，TTableCacheLRU
    TRecord<K, V> r = cache.get(key);
    if (null != r)
        return r.add(value);
    countAddMiss.incrementAndGet();
    if (_exist(key)) {
        countAddStorageMiss.incrementAndGet();
        return false;
    }
    //将变化数据放入cache缓存，存入的是一条TRecord
    cache.add(key, new TRecord<K, V>(this, value, lockey, TRecord.State.ADD));
    return true;
}

9.缓存cache中变化表数据刷新 CheckPoint

synchronized void checkpoint(limax.xmlgen.Zdb meta) {
    if (Trace.isDebugEnabled())
        Trace.debug("---------------- begin ----------------");
    //表的存储引用
    final List<StorageInterface> storages = this.tables.getStorages();

    if (meta.getMarshalN() < 1)
        if (Trace.isWarnEnabled())
            Trace.warn("marshalN disabled");

    elapse.reset();
    for (int i = 1; i <= meta.getMarshalN(); ++i) {
        int countMarshalN = storages.stream().mapToInt(StorageInterface::marshalN).sum();
        this.marshalNCount += countMarshalN;
        if (Trace.isDebugEnabled())
            Trace.debug("marshalN=" + i + "/" + countMarshalN);
    }
    this.marshalNTotalTime += elapse.elapsed();
    Runnable cleanupDuration;
    {
        int countSnapshot;
        int countMarshal0;
        Lock lock = tables.flushWriteLock();
        lock.lock();
        elapse.reset();
        try {
            countMarshal0 = storages.stream().mapToInt(StorageInterface::marshal0).sum();
            countSnapshot = storages.stream().mapToInt(StorageInterface::snapshot).sum();
            cleanupDuration = Zdb.checkpointDuration();
        } finally {
            lock.unlock();
        }
        long snapshotTime = elapse.elapsedAndReset();
        if (snapshotTime / 1000000 > meta.getSnapshotFatalTime())
            Trace.fatal(
                    "snapshot time=" + snapshotTime + " snapshot=" + countSnapshot + " marshal0=" + countMarshal0);
        this.marshal0Count += countMarshal0;
        this.snapshotTotalTime += snapshotTime;
        this.snapshotCount += countSnapshot;
        if (Trace.isDebugEnabled())
            Trace.debug("snapshot=" + countSnapshot + " marshal0=" + countMarshal0);
    }
    int countFlush = storages.stream().mapToInt(StorageInterface::flush).sum();
    this.flushTotalTime += elapse.elapsedAndReset();
    if (Trace.isDebugEnabled())
        Trace.debug("flush=" + countFlush);
    if (countFlush > 0)
        this.tables.getLogger().checkpoint();
    if (countFlush > 0) {
        this.flushCount += countFlush;
        this.checkpointTotalTime += elapse.elapsedAndReset();
        if (Trace.isDebugEnabled())
            Trace.debug("checkpoint");
    }
    ++checkpointCount;
    if (cleanupDuration != null)
        cleanupDuration.run();
    if (Trace.isDebugEnabled())
        Trace.debug("----------------- end -----------------");
}

DataBase的flush，将数据存储到文本中

private synchronized void checkpoint(boolean rotate) {
    try {
        Collection<PageCache> tableCollection = tables.values();
        tableCollection.forEach(cache -> cache.writeFreeze());
        for (PageCache cache : tableCollection)
            cache.waitWriteFrozen();
        Map<PageCache, List<PageLayout>> snapshots = labeledHash.createSnapshots();
        tableCollection.forEach(cache -> {
            cache.setSnapshots(snapshots.get(cache));
            cache.writeUnfreeze();
        });
        if (!snapshots.isEmpty() || rotate) {
            logger.prepare(rotate);
            LabeledHash.releaseSnapshots(snapshots);
            logger.commit();
        }
        labeledHash.permitWash();
    } catch (IOException e) {
        Trace.fatal("limax.edb.checkpoint", e);
        System.exit(-1);
    }
}

10.总结

（1）缓存LRU策略，当内存表超过最大数量后悔主动删除

（2）Lockey hash锁调度，死锁是zdb经常出现的问题，对开发造成一定的难度

（3）procedure回滚是zdb优势