关于游戏中cache的后写入回写机制

最新推荐文章于 2022-11-28 09:50:34 发布

hw_ritsky

最新推荐文章于 2022-11-28 09:50:34 发布

阅读量655

点赞数

文章标签： cache 游戏 exception structure null insert

本文链接：https://blog.csdn.net/hw_ritsky/article/details/5929151

版权

这几天把引擎中关于cache回写部分的操作进行一次整理，并把原来分散到由个模块自已负责的cache回写操作集中到一起来进行回写操作，我这里暂时称他为后写入管理器。

TEWriterBehindsManager.java

package com.te.server.engine.writerbehind; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import org.jdom.Element; /** * * 后写入方式管理器，负责管理所有角色对应的后写入方式 * * 后写入方式，不负责管理cache，但他负责所有注册cache类型数据的IO操作，从而把cache操作与io操作分离 * * 在游戏中有一些cache是全局的，比如说帮会，拍买行的交易，等，这些在后写入管理器都有对应的全局后写入的映射表 * 我们大部分是每个角色有一个后写入器，后写入器中有针对在游戏中的所有Cache后写入队例，每个角色的后写入器都会 * 启动一个定时器，按给定时间的间隔进行后写入，后写入中双分为insert,update,delete等操作，这些操作在定时器到 * 达时会按insert,update,delete等次序进行操作，为更有效的节省开销，在每个操作放入到后写入器时都会检查，并且 * 合并已有的操作，比如放入一个update操作，后写入器会检查是否有相同对象的insert操作，如果有就不把update操作 * 放入队列 * * 注意: * 1.由于后写入器，并不知道各自cache中的数据对象的结构，但为了区分是否为同一个数据，所以一般情况下都是用Object中的 * equals方法，所以我们最好是让需要认为是相同对象的实例实现hashCode,与 equals方法 * * 2.对于要操作的cache中的数据对象，最好是同一对象，比如我们插入一个数据对象，之后再更新一个数据对象，我们不仅要做到 * 他们在数据库中的key是相同的，而且最好是同一对象，如果万不得以不能是同一对象，最好是在相同键不同对象的情况下，重 * 新实现hashCode()与equals()方法，让他同的判断是正确的。 * * * * * @author 胡玮 * */ public class TEWriterBehindsManager { /** * 全局Behind，给游戏中一些不在角色之下管理的Cache使用的 */ private final HashMap<Integer, TEWriterBehind> globalWriterBehindMap = new HashMap<Integer, TEWriterBehind>(); /** * 角色对应的Behind */ private ConcurrentHashMap<Long, TEWriterBehind> writerBehindMap = null; /**定时器*/ ScheduledExecutorService timer; /**系统CacheWriter工厂*/ TECacheWriterFactory systemFactory = null; /**自定义CacheWriter工厂*/ TECacheWriterFactory customFactory = null; /**cache类型映射表*/ HashMap<Integer, TECacheType> cacheTypeMap = new HashMap<Integer, TECacheType>(); /**写间隔时间*/ long writeDelayMs = 0; /**每次进行插入批量处理的数量*/ int insertBathingSize = 0; /**每次进行更新批量处理的数量*/ int updateBathingSize = 0; /**每次进行删除批量处理的数量*/ int deleteBathingSize = 0; /**初次回写失败后，失败重写的次数*/ int retryAttempts = 0; /**失败重写的间隔时间*/ long retryAttemptDelay = 1; public TEWriterBehindsManager(ScheduledExecutorService timer){ this.timer = timer; } /** * * 启动管理器 * * * @param configElement * @throws Exception */ public void start(Element configElement) throws Exception { fillCacheTypeMap(); int initialCapacity = Integer.parseInt(configElement.getChild("ConcurrentMap_structure").getChildTextTrim("initialCapacity")); float loadFactor = Float.parseFloat(configElement.getChild("ConcurrentMap_structure").getChildTextTrim("loadFactor")); int concurrencyLevel = Integer.parseInt(configElement.getChild("ConcurrentMap_structure").getChildTextTrim("concurrencyLevel")); writerBehindMap = new ConcurrentHashMap<Long, TEWriterBehind>(initialCapacity, loadFactor, concurrencyLevel); systemFactory = (TECacheWriterFactory) Class.forName(configElement.getChild("systemWriterFactory").getAttributeValue("class")).newInstance(); writeDelayMs = Long.parseLong(configElement.getChild("config").getAttributeValue("writeDelayMs")); insertBathingSize = Integer.parseInt(configElement.getChild("config").getAttributeValue("insertBathingSize")); updateBathingSize = Integer.parseInt(configElement.getChild("config").getAttributeValue("updateBathingSize")); deleteBathingSize = Integer.parseInt(configElement.getChild("config").getAttributeValue("deleteBathingSize")); retryAttempts = Integer.parseInt(configElement.getChild("config").getAttributeValue("retryAttempts")); retryAttemptDelay = Integer.parseInt(configElement.getChild("config").getAttributeValue("retryAttemptDelay")); Element customCacheTypes = configElement.getChild("customCacheTypes"); if(customCacheTypes != null) { customFactory = (TECacheWriterFactory) Class.forName(customCacheTypes.getAttributeValue("writerFactory")).newInstance(); List<Element> elist = customCacheTypes.getChildren("cacheType"); for(Iterator<Element> iter = elist.iterator(); iter.hasNext();) { Element e = iter.next(); boolean isGlobal = Boolean.parseBoolean(e.getAttributeValue("global")); int id = Integer.parseInt(e.getAttributeValue("id")); String name = e.getAttributeValue("name"); if(cacheTypeMap.containsKey(id)) { throw new Exception("Cache类型有重复的，不能声名重复的"); } TECacheType ct = new TECacheType(id, isGlobal, name); cacheTypeMap.put(ct.getId(), ct); } } createGlobalBehind(); //创建全局的Behind } /** * 以系统的cache类型填充cacheTypeMap; */ private void fillCacheTypeMap() { cacheTypeMap.put(TECacheType.CACHE_GANG.getId(), TECacheType.CACHE_GANG); cacheTypeMap.put(TECacheType.CACHE_ITEM.getId(), TECacheType.CACHE_ITEM); cacheTypeMap.put(TECacheType.CACHE_MAIL.getId(), TECacheType.CACHE_MAIL); cacheTypeMap.put(TECacheType.CACHE_MISSION.getId(), TECacheType.CACHE_MISSION); cacheTypeMap.put(TECacheType.CACHE_ROLE.getId(), TECacheType.CACHE_ROLE); cacheTypeMap.put(TECacheType.CACHE_SKILL.getId(), TECacheType.CACHE_SKILL); cacheTypeMap.put(TECacheType.CACHE_TRANSACTION.getId(), TECacheType.CACHE_TRANSACTION); } /** * 创建全局的WriterBehind */ private void createGlobalBehind() { Collection<TECacheType> values = cacheTypeMap.values(); for(Iterator<TECacheType> iter = values.iterator(); iter.hasNext();) { TECacheType ct = iter.next(); if(!ct.isGlobal()) { continue; } TEWriterBehind behind = new TEWriterBehind(); behind.start(this, ct); globalWriterBehindMap.put(ct.getId(), behind); } } /** * * 放入一个角色的WriterBehind，并启动 * * * @param roleId * @param behind */ public void putBehind(long roleId, TEWriterBehind behind) { if(behind != null) { behind.start(this); writerBehindMap.put(roleId, behind); } } /** * * 移除一个角色的WriterBehind，并停止 * * * @param roleId */ public void removeBehind(long roleId) { TEWriterBehind behind = writerBehindMap.remove(roleId); if(behind != null) { behind.stop(); } } /** * * 获取一个角色的WriterBehind * * * @param roleId * @return */ public TEWriterBehind getBehind(long roleId) { return writerBehindMap.get(roleId); } /** * * 获取一个全局的WriterBehind * * * @param cacheType * @return */ public TEWriterBehind getGlobalBehind(TECacheType cacheType) { if(!cacheType.isGlobal()) { return null; } return globalWriterBehindMap.get(cacheType.getId()); } public void shutdown() { //TODO:停止一切活动，并把所有需要回写的cache全部进行回写 Collection<TEWriterBehind> values = writerBehindMap.values(); for(Iterator<TEWriterBehind> iter = values.iterator(); iter.hasNext();) { TEWriterBehind behind = iter.next(); behind.stop(); } values = globalWriterBehindMap.values(); for(Iterator<TEWriterBehind> iter = values.iterator(); iter.hasNext();) { TEWriterBehind behind = iter.next(); behind.stop(); } } }

TEWriterBehind.java

package com.te.server.engine.writerbehind; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; /** * * 后写入器，每个角色与全局cache会对应一个后写入器，每个后写器将会启动一个定时器，定时器的间隔时间由配置文件决定 * 后写器提供api负责操作的写入，一个后写入器中可能对应N多个后写入队列，每个队列实际上对应一个cache类型。 * * * @author 胡玮 * */ public class TEWriterBehind implements Runnable{ private final HashMap<Integer, TEWriterBehindQueue> queueMap = new HashMap<Integer, TEWriterBehindQueue>(); TEWriterBehindQueue[] queues; private ScheduledFuture future; private boolean isGlobal; private TEWriterBehindsManager manager; protected final void start(TEWriterBehindsManager manager) { this.manager = manager; isGlobal = false; Collection<TECacheType> values = manager.cacheTypeMap.values(); for(Iterator<TECacheType> iter = values.iterator(); iter.hasNext();) { TECacheType ct = iter.next(); if(ct.isGlobal()) { continue; } TEWriterBehindQueue queue = new TEWriterBehindQueue(); TECacheWriter cacheWriter = null; if(manager.systemFactory != null) { cacheWriter = manager.systemFactory.createCacheWriter(ct); } if(cacheWriter == null && manager.customFactory != null) { cacheWriter = manager.customFactory.createCacheWriter(ct); } queue.cacheWriter = cacheWriter; queue.queueId = ct.getId(); queue.cacheName = ct.getName(); queueMap.put(ct.getId(), queue); } queues = new TEWriterBehindQueue[queueMap.size()]; int index = 0; Collection<TEWriterBehindQueue> qValues = queueMap.values(); for(Iterator<TEWriterBehindQueue> iter = qValues.iterator(); iter.hasNext();) { queues[index++] = iter.next(); } future = manager.timer.scheduleAtFixedRate(this, manager.writeDelayMs, manager.writeDelayMs, TimeUnit.SECONDS); } /** * 启动全局Behind * @param manager * @param type */ protected final void start(TEWriterBehindsManager manager, TECacheType type) { this.manager = manager; this.isGlobal = true; TEWriterBehindQueue queue = new TEWriterBehindQueue(); TECacheWriter cacheWriter = null; if(manager.systemFactory != null) { cacheWriter = manager.systemFactory.createCacheWriter(type); } if(cacheWriter == null && manager.customFactory != null) { cacheWriter = manager.customFactory.createCacheWriter(type); } queue.cacheWriter = cacheWriter; queue.queueId = type.getId(); queue.cacheName = type.getName(); queueMap.put(type.getId(), queue); queues = new TEWriterBehindQueue[1]; queues[0] = queue; future = manager.timer.scheduleAtFixedRate(this, manager.writeDelayMs, manager.writeDelayMs, TimeUnit.SECONDS); } protected final void stop() { //TODO:把所有存在的操作全部回写到数据库中 future.cancel(true); for(int i=0; i<queues.length; i++) { TEWriterBehindQueue behind = queues[i]; behind.operation( 0, 0, 0, manager.retryAttempts, manager.retryAttemptDelay); } } public void write(Object cacheObj, TECacheType cacheType, TEOperationType operationType) { if(cacheType.isGlobal() != isGlobal) { throw new RuntimeException("提交的cacheType类型不正确"); } TEWriterBehindQueue queue = queueMap.get(cacheType.getId()); queue.putOperation(cacheObj, operationType); } @Override public void run() { for(int i=0; i<queues.length; i++) { TEWriterBehindQueue behind = queues[i]; behind.operation( manager.insertBathingSize, manager.updateBathingSize, manager.deleteBathingSize, manager.retryAttempts, manager.retryAttemptDelay); } } }

TEWriterBehindQueue.java

package com.te.server.engine.writerbehind; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.log4j.Logger; /** * * 后写入队列，负责将对应cahce类型的操作保存，等待定时器时间到达时调用提供的API进行相应的IO操作 * * * @author 胡玮 * */ public class TEWriterBehindQueue { private final static Logger LOGGER = Logger.getRootLogger(); //private static final int MS_IN_SEC = 1000; int queueId; String cacheName=""; private final ReentrantReadWriteLock queueLock = new ReentrantReadWriteLock(); private final ReentrantReadWriteLock.ReadLock queueReadLock = queueLock.readLock(); private final ReentrantReadWriteLock.WriteLock queueWriteLock = queueLock.writeLock(); //private final Condition queueIsEmpty = queueWriteLock.newCondition(); //private final Condition queueIsStopped = queueWriteLock.newCondition(); //private final AtomicLong lastProcessing = new AtomicLong(System.currentTimeMillis()); //private final AtomicLong lastWorkDone = new AtomicLong(System.currentTimeMillis()); private final AtomicBoolean busyProcessing = new AtomicBoolean(false); Set<Object> insertWaiting = new HashSet<Object>(); Set<Object> updateWaiting = new HashSet<Object>(); Set<Object> deleteWaiting = new HashSet<Object>(); TECacheWriter cacheWriter; public void putOperation(Object cacheObj, TEOperationType type) { queueWriteLock.lock(); try { if(type.equals(TEOperationType.INSERT)) { //插入 insertWaiting.add(cacheObj); } else if(type.equals(TEOperationType.UPDATE)) { //更新 if(!insertWaiting.contains(cacheObj)) { updateWaiting.add(cacheObj); } } else { //删除 insertWaiting.remove(cacheObj); updateWaiting.remove(cacheObj); deleteWaiting.add(cacheObj); } } finally { queueWriteLock.unlock(); } } void operation(int insertBathingSize, int updateBathingSize, int deleteBathingSize, int retryAttempts, long retryAttemptDelay) { if(cacheWriter == null) { LOGGER.error(cacheName+"[cacheId="+queueId+"] cacheWriter is not exist"); return; } if(busyProcessing.get()) { return; } else { busyProcessing.set(true); } ArrayList<Object> tempInserts = new ArrayList<Object>(); ArrayList<Object> tempUpdates = new ArrayList<Object>(); ArrayList<Object> tempDeletes = new ArrayList<Object>(); queueReadLock.lock(); try { _operation(tempInserts, insertWaiting, insertBathingSize); _operation(tempUpdates, updateWaiting, updateBathingSize); _operation(tempDeletes, deleteWaiting, deleteBathingSize); } finally { queueReadLock.unlock(); } int num = retryAttempts + 1; if(tempInserts.size() > 0) { //插入 for(int i=0; i<num; i++) { try { cacheWriter.insertAll(tempInserts); break; } catch (Exception e) { LOGGER.error(e.getMessage(), e); try { Thread.sleep(retryAttemptDelay * 1000); } catch (InterruptedException e1) { e1.printStackTrace(); } } } } if(tempUpdates.size() > 0) { //更新 for(int i=0; i<num; i++) { try { cacheWriter.updateAll(tempUpdates); break; } catch (Exception e) { LOGGER.error(e.getMessage(), e); try { Thread.sleep(retryAttemptDelay * 1000); } catch (InterruptedException e1) { e1.printStackTrace(); } } } } if(tempDeletes.size() > 0) { //删除 for(int i=0; i<num; i++) { try { cacheWriter.deleteAll(tempDeletes); break; } catch (Exception e) { LOGGER.error(e.getMessage(), e); try { Thread.sleep(retryAttemptDelay * 1000); } catch (InterruptedException e1) { e1.printStackTrace(); } } } } busyProcessing.set(false); } private void _operation(ArrayList<Object> _list, Set<Object> _waiting, int _bathingSize) { int index = 0; for(Iterator<Object> iter = _waiting.iterator(); iter.hasNext();) { if(_bathingSize >0 && index >= _bathingSize) { break; } Object cacheobj = iter.next(); iter.remove(); _list.add(cacheobj); index ++; } } }

TECacheType.java

package com.te.server.engine.writerbehind; /** * * cache类型，标识游戏中cache数据的类型，通过配置文件注册的cache类型分为全局与非全局的，非全局的则为每个角色中存在的 * cache类型，全局的则是在整个游戏系统中唯一存在的 * * * * @author 胡玮 * */ public class TECacheType { public static final TECacheType CACHE_ITEM = new TECacheType(1, false, "item"); public static final TECacheType CACHE_MISSION = new TECacheType(2, false, "mission"); public static final TECacheType CACHE_ROLE = new TECacheType(3, false, "role"); public static final TECacheType CACHE_SKILL = new TECacheType(4, false, "skill"); public static final TECacheType CACHE_MAIL = new TECacheType(5, false, "mail"); public static final TECacheType CACHE_GANG = new TECacheType(6, true, "gang"); public static final TECacheType CACHE_TRANSACTION = new TECacheType(7, true, "transaction"); private int id; private boolean isGlobal; private String name; public TECacheType(int id, boolean isGlobal, String name) { this.id = id; this.isGlobal = isGlobal; this.name = name; } /** * @return the id */ public int getId() { return id; } /** * @return the isGlobal */ public boolean isGlobal() { return isGlobal; } /** * @return the name */ public String getName() { return name; } }

TECacheWriter.java

package com.te.server.engine.writerbehind; import java.sql.SQLException; import java.util.List; /** * * cache写操作器，这个接口由使用者实现，因为系统并不知道使用者要怎么样操作IO，操作器分为三个方法，分别对应数据库中的 * 插入，更新，删除 * * 注意： * 每个方法传入的是一个cache数据列表，传入列表的原因是希望使用者在实现时尽量一次进行IO操作，不要分散进行，这样能最大限度的 * 节省开销 * * * @author 胡玮 * */ public interface TECacheWriter { public void insertAll(List<Object> elements) throws SQLException; public void updateAll(List<Object> elements) throws SQLException; public void deleteAll(List<Object> elements) throws SQLException; }

TECacheWriterFactory.java

package com.te.server.engine.writerbehind; /** * * CacheWriter的工厂接口，负责创建CacheWriter,原则上每一个cache类型对应一个CacheWriter，工厂分为系统工厂与自定义工厂 * 这二者都可以由配置文件定义工厂的具体实现，但是系统工厂是必须实现的，否则不能启动后写入管理器，系统工厂所做的事情与自 * 定义工厂是一样的，区别在于，我们知道游戏中一些我们耳熟能详的cache类型，比如道具，任务等等，这些是属于游戏引擎必然存 * 在，所以这里为他们定义了一个系统工厂，为这些cache类型创建对应的CacheWriter，而自定义的工厂，我们是为了方便扩展未知的 * cache类型而准备的。 * * * @author 胡玮 * */ public interface TECacheWriterFactory { public TECacheWriter createCacheWriter(TECacheType cacheType); }

TEOperationType.java

package com.te.server.engine.writerbehind; /** * * 操作类型，需要通知后写入器进行怎样的操作，它们分别对应数据库的插入，更新，删除操作 * * @author 胡玮 * */ public enum TEOperationType { INSERT, UPDATE, DELETE }

config.xml

<writer-behind>
 <ConcurrentMap_structure>
 <initialCapacity>100</initialCapacity>
 <loadFactor>0.8</loadFactor>
 <concurrencyLevel>20</concurrencyLevel>
 </ConcurrentMap_structure>

 <systemWriterFactory class="com.te.server.engine.writerbehind.SystemCacheWriterFactory"/>

 <config writeDelayMs="20"
 insertBathingSize="10"
 updateBathingSize="10"
 deleteBathingSize="10"
 retryAttempts="2"
 retryAttemptDelay="3"/>
 
</writer-behind>