Distributed Lock

1 Overview

      在分布式系统中，通常会避免使用分布式锁。然而在某些场景下，还是存在对分布式锁的需求。跟普通锁相比，分布式锁面需要对的问题更多，例如怎样保证某个进程在持有锁时意外终止之后，其它进程也能够正常地获得锁等等。笔者认为一个比较好的分布式锁实现是Terracotta，但是这不是本文的重点，感兴趣的读者可以参考笔者的Terracotta in Action 系列文章（http://whitesock.iteye.com/blog/351780 ,http://whitesock.iteye.com/blog/352876 , http://whitesock.iteye.com/blog/354587 ）。

      除了Terracotta，不少其它开源项目也声称支持分布式锁，例如ZooKeeper，JGroups和Hazelcast等。在这些项目中，笔者倾向于使用ZooKeeper。ZooKeeper在其官方文档的ZooKeeper Recipes and Solutions章节中介绍了一个分布式锁的实现，本文主要对该版本进行了改良。关于Hazelcast，笔者不得不说，其官方文档文字不少但却苍白，很多内容介绍的都是浅尝辄止，难道是强迫开发人员去仔细地阅读源码，或者参加其价格不菲的培训？

 

2 Implementation

     首先，笔者希望分布式锁能够支持Java并发包中的Lock接口，并且最好是可重入的。此外，在某个进程持有分布式锁的过程中，如果不能保证该锁不会被其它进程同时持有（例如网络故障），那么至少应该能够通知锁的持有者，以便其采取相应的应对措施。以下是笔者对分布式锁的定义：

Java代码  

1. import java.util.concurrent.locks.Lock;  
2.   
3. public interface DistributedLock extends Lock {  
4.       
5.     Listener getListener();  
6.       
7.     void setListener(Listener listener);  
8.       
9.     /** 
10.      *  
11.      */  
12.     interface Listener {  
13.           
14.         void onAbort(DistributedLock lock, Exception e);  
15.     }  
16. }  

    其中Listener接口的作用是，在无法排它独占该锁时进行回调。接下来是笔者的两个实现的共通父类。

Java代码  

1. import java.util.concurrent.TimeUnit;  
2. import java.util.concurrent.locks.Condition;  
3. import java.util.concurrent.locks.ReentrantLock;  
4.   
5. public abstract class AbstractDistributedLock implements DistributedLock {  
6.     //  
7.     protected volatile boolean verbose;  
8.     protected volatile Listener listener;  
9.     protected final ReentrantLock lock = new ReentrantLock();  
10.   
11.     //  
12.     protected abstract void doLock();  
13.     protected abstract void doUnlock();  
14.     protected abstract boolean doTryLock();  
15.     protected abstract void doLockInterruptibly() throws InterruptedException;  
16.     protected abstract boolean doTryLock(long timeout, TimeUnit unit) throws InterruptedException;  
17.       
18.     /** 
19.      *  
20.      */  
21.     public boolean isVerbose() {  
22.         return verbose;  
23.     }  
24.   
25.     public void setVerbose(boolean verbose) {  
26.         this.verbose = verbose;  
27.     }  
28.       
29.     public boolean isLocked() {  
30.         return this.lock.isLocked();  
31.     }  
32.       
33.     public boolean isHeldByCurrentThread() {  
34.         return this.lock.isHeldByCurrentThread();  
35.     }  
36.       
37.     /** 
38.      *  
39.      */  
40.     @Override  
41.     public Listener getListener() {  
42.         return this.listener;  
43.     }  
44.   
45.     @Override  
46.     public void setListener(Listener listener) {  
47.         this.listener = listener;  
48.     }  
49.       
50.     /** 
51.      *  
52.      */  
53.     @Override  
54.     public void lock() {  
55.         //  
56.         this.lock.lock();  
57.         if(this.lock.getHoldCount() > 1) return;  
58.           
59.         //  
60.         boolean succeed = false;  
61.         try {  
62.             doLock();  
63.             succeed = true;  
64.         } finally {  
65.             if(!succeed) {  
66.                 this.lock.unlock();  
67.             }  
68.         }  
69.     }  
70.   
71.     @Override  
72.     public void lockInterruptibly() throws InterruptedException {  
73.         //  
74.         this.lock.lockInterruptibly();  
75.         if(this.lock.getHoldCount() > 1) return;  
76.           
77.         //  
78.         boolean succeed = false;  
79.         try {  
80.             doLockInterruptibly();  
81.             succeed = true;  
82.         } finally {  
83.             if(!succeed) {  
84.                 this.lock.unlock();  
85.             }  
86.         }  
87.     }  
88.       
89.     @Override  
90.     public boolean tryLock() {  
91.         //  
92.         if(!this.lock.tryLock()) return false;  
93.         if(this.lock.getHoldCount() > 1) return true;  
94.           
95.         //  
96.         boolean succeed = false;  
97.         try {  
98.             succeed = doTryLock();  
99.         } finally {  
100.             if(!succeed) {  
101.                 this.lock.unlock();  
102.             }  
103.         }  
104.         return succeed;  
105.     }  
106.   
107.     @Override  
108.     public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException {  
109.         //  
110.         final long mark = System.nanoTime();  
111.         if(!this.lock.tryLock(timeout, unit)) return false;  
112.         if(this.lock.getHoldCount() > 1) return true;  
113.           
114.         //  
115.         boolean succeed = false;  
116.         try {  
117.             timeout = TimeUnit.NANOSECONDS.convert(timeout, unit) - (System.nanoTime() - mark);  
118.             if(timeout >= 0) {  
119.                 succeed = doTryLock(timeout, TimeUnit.NANOSECONDS);  
120.             }  
121.         } finally {  
122.             if(!succeed) {  
123.                 this.lock.unlock();  
124.             }  
125.         }  
126.         return succeed;  
127.     }  
128.   
129.     @Override  
130.     public void unlock() {  
131.         //  
132.         if(!this.lock.isHeldByCurrentThread()) return;  
133.         if(this.lock.getHoldCount() > 1) return;  
134.           
135.         //  
136.         try {  
137.             doUnlock();  
138.         } finally {  
139.             this.lock.unlock();  
140.         }  
141.     }  
142.       
143.     @Override  
144.     public Condition newCondition() {  
145.         throw new UnsupportedOperationException();  
146.     }  
147. }  

 

2.1 MySQL Named Lock

    在讨论ZooKeeper的分布式锁实现之前，先介绍一下笔者基于MySQL Named Lock的一个实现。

Java代码  

1. import java.sql.Connection;  
2. import java.sql.PreparedStatement;  
3. import java.sql.ResultSet;  
4. import java.util.concurrent.ScheduledExecutorService;  
5. import java.util.concurrent.ScheduledFuture;  
6. import java.util.concurrent.TimeUnit;  
7. import java.util.concurrent.atomic.AtomicReference;  
8.   
9. import javax.sql.DataSource;  
10.   
11. import org.apache.commons.lang.builder.ToStringBuilder;  
12. import org.apache.commons.lang.builder.ToStringStyle;  
13. import org.apache.commons.lang.exception.NestableRuntimeException;  
14. import org.slf4j.Logger;  
15. import org.slf4j.LoggerFactory;  
16.   
17.   
18. public final class MySQLNamedLock extends AbstractDistributedLock {  
19.     //  
20.     private static final Logger LOGGER = LoggerFactory.getLogger(MySQLNamedLock.class);  
21.       
22.     //  
23.     private String name;  
24.     private DataSource dataSource;  
25.     private long validationInterval = 1000L;  
26.     private ScheduledExecutorService scheduler;  
27.     private final AtomicReference<Connection> connection;  
28.     private final AtomicReference<ScheduledFuture<?>> future;  
29.   
30.     /** 
31.      *  
32.      */  
33.     public MySQLNamedLock() {  
34.         this(null, null, null);  
35.     }  
36.       
37.     public MySQLNamedLock(String name, DataSource dataSource, ScheduledExecutorService scheduler) {  
38.         this.name = name;  
39.         this.scheduler = scheduler;  
40.         this.dataSource = dataSource;  
41.         this.connection = new AtomicReference<Connection>();  
42.         this.future = new AtomicReference<ScheduledFuture<?>>();  
43.     }  
44.       
45.     /** 
46.      *  
47.      */  
48.     @Override  
49.     public String toString() {  
50.         return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)  
51.         .append("name", this.name).toString();  
52.     }  
53.       
54.     /** 
55.      *  
56.      */  
57.     public String getName() {  
58.         return name;  
59.     }  
60.   
61.     public void setName(String name) {  
62.         this.name = name;  
63.     }  
64.   
65.     public long getValidationInterval() {  
66.         return validationInterval;  
67.     }  
68.   
69.     public void setValidationInterval(long interval) {  
70.         this.validationInterval = interval;  
71.     }  
72.       
73.     public DataSource getDataSource() {  
74.         return dataSource;  
75.     }  
76.   
77.     public void setDataSource(DataSource dataSource) {  
78.         this.dataSource = dataSource;  
79.     }  
80.       
81.     public ScheduledExecutorService getScheduler() {  
82.         return scheduler;  
83.     }  
84.   
85.     public void setScheduler(ScheduledExecutorService scheduler) {  
86.         this.scheduler = scheduler;  
87.     }  
88.       
89.     /** 
90.      *  
91.      */  
92.     @Override  
93.     protected void doLock() {  
94.         doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);  
95.     }  
96.   
97.     @Override  
98.     protected void doLockInterruptibly() {  
99.         doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);  
100.     }  
101.   
102.     @Override  
103.     protected boolean doTryLock() {  
104.         return doTryLock(0, TimeUnit.SECONDS);  
105.     }  
106.       
107.     @Override  
108.     protected boolean doTryLock(long timeout, TimeUnit unit) {  
109.         //  
110.         Integer r = null;  
111.         ResultSet rs = null;  
112.         PreparedStatement ps = null;  
113.         try {  
114.             this.connection.set(this.dataSource.getConnection());  
115.             ps = this.connection.get().prepareStatement("SELECT GET_LOCK(?, ?)");  
116.             ps.setString(1, this.name);  
117.             ps.setInt(2, (int)TimeUnit.SECONDS.convert(timeout, unit));  
118.             rs = ps.executeQuery();  
119.             if(rs.next()) {  
120.                 r = rs.getInt(1);  
121.                 if(rs.wasNull()) r = null;  
122.             }  
123.         } catch(Exception e) {  
124.             throw new NestableRuntimeException("failed to lock, name: " + this.name, e);  
125.         } finally {  
126.             JdbcUtils.closeQuietly(rs);  
127.             JdbcUtils.closeQuietly(ps);  
128.         }  
129.           
130.         //  
131.         final boolean succeed = (r != null && r == 1);  
132.         if(succeed && this.listener != null) {  
133.             final long interval = this.validationInterval;  
134.             this.future.set(this.scheduler.scheduleWithFixedDelay(new ValidationTask(), interval, interval, TimeUnit.MILLISECONDS));  
135.         }  
136.           
137.         //  
138.         return succeed;  
139.     }  
140.   
141.     @Override  
142.     protected void doUnlock() {  
143.         //  
144.         final ScheduledFuture<?> f = this.future.getAndSet(null);  
145.         if(f != null) f.cancel(true);  
146.           
147.         //  
148.         Integer r = null;  
149.         ResultSet rs = null;  
150.         PreparedStatement ps = null;  
151.         try {  
152.             //  
153.             ps = this.connection.get().prepareStatement("SELECT RELEASE_LOCK(?)");  
154.             ps.setString(1, this.name);  
155.             rs = ps.executeQuery();  
156.             if(rs.next()) {  
157.                 r = rs.getInt(1);  
158.                 if(rs.wasNull()) r = null;  
159.             }  
160.               
161.             //  
162.             if(r == null) {  
163.                 LOGGER.warn("lock does NOT exist, name: {}", this.name);  
164.             } else if(r == 0) {  
165.                 LOGGER.warn("lock was NOT accquired by current thread, name: {}", this.name);  
166.             } else {  
167.                 LOGGER.warn("failed to unlock, name: {}, result: {}", this.name, r);  
168.             }  
169.         } catch(Exception e) {  
170.             throw new NestableRuntimeException("failed to unlock, name: " + this.name, e);  
171.         } finally {  
172.             JdbcUtils.closeQuietly(rs);  
173.             JdbcUtils.closeQuietly(ps);  
174.             JdbcUtils.closeQuietly(this.connection.getAndSet(null));  
175.         }  
176.     }  
177.       
178.     /** 
179.      *  
180.      */  
181.     private class ValidationTask implements Runnable {  
182.   
183.         @Override  
184.         public void run() {  
185.             try {  
186.                 ((com.mysql.jdbc.Connection)connection.get()).ping();  
187.             } catch(Exception e) {  
188.                 //  
189.                 if(isLocked() && listener != null && connection.get() != null) {  
190.                     listener.onAbort(MySQLNamedLock.this, e);  
191.                 }  
192.                   
193.                 //  
194.                 throw new NestableRuntimeException(e); // Note: suppress subsequent executions   
195.             }  
196.         }  
197.     }  
198. }  

    需要注意的是，如果在该锁上注册了Listener，并且Connection在持有锁的过程中失效，那么该Listener会被回调。

2.2 ZooKeeper Lock

    以下代码是笔者对ZooKeeper官方版本的改良：

Java代码  

1. import java.lang.management.ManagementFactory;  
2. import java.util.Collections;  
3. import java.util.Comparator;  
4. import java.util.List;  
5. import java.util.concurrent.CountDownLatch;  
6. import java.util.concurrent.TimeUnit;  
7. import java.util.concurrent.atomic.AtomicReference;  
8.   
9. import org.apache.commons.lang.builder.ToStringBuilder;  
10. import org.apache.commons.lang.builder.ToStringStyle;  
11. import org.apache.commons.lang.exception.NestableRuntimeException;  
12. import org.apache.zookeeper.CreateMode;  
13. import org.apache.zookeeper.KeeperException;  
14. import org.apache.zookeeper.WatchedEvent;  
15. import org.apache.zookeeper.Watcher;  
16. import org.apache.zookeeper.ZooDefs;  
17. import org.apache.zookeeper.ZooKeeper;  
18. import org.apache.zookeeper.data.Stat;  
19. import org.slf4j.Logger;  
20. import org.slf4j.LoggerFactory;  
21.   
22.   
23. public final class ZooKeeperLock extends AbstractDistributedLock {  
24.     //  
25.     private static final Logger LOGGER = LoggerFactory.getLogger(ZooKeeperLock.class);  
26.   
27.     //  
28.     private String directory;  
29.     private ZooKeeper zookeeper;  
30.     private final String processName;  
31.     private final AtomicReference<ZooKeeperLocker> locker;  
32.   
33.     /** 
34.      *  
35.      */  
36.     public ZooKeeperLock() {  
37.         this(null, null);  
38.     }  
39.   
40.     public ZooKeeperLock(ZooKeeper zookeeper, String directory) {  
41.         this.zookeeper = zookeeper;  
42.         this.directory = directory;  
43.         this.locker = new AtomicReference<ZooKeeperLocker>();  
44.         this.processName = ManagementFactory.getRuntimeMXBean().getName();  
45.     }  
46.   
47.     /** 
48.      *  
49.      */  
50.     @Override  
51.     public String toString() {  
52.         return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)  
53.         .append("directory", this.directory).toString();  
54.     }  
55.       
56.     /** 
57.      *  
58.      */  
59.     public String getDirectory() {  
60.         return directory;  
61.     }  
62.   
63.     public void setDirectory(String directory) {  
64.         this.directory = directory;  
65.     }  
66.       
67.     public ZooKeeper getZookeeper() {  
68.         return zookeeper;  
69.     }  
70.   
71.     public void setZookeeper(ZooKeeper zookeeper) {  
72.         this.zookeeper = zookeeper;  
73.     }  
74.   
75.     /** 
76.      *  
77.      */  
78.     @Override  
79.     protected void doLock() {  
80.         doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);  
81.     }  
82.   
83.     @Override  
84.     protected void doLockInterruptibly() {  
85.         doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);  
86.     }  
87.       
88.     @Override  
89.     protected boolean doTryLock() {  
90.         return doTryLock(0, TimeUnit.SECONDS);  
91.     }  
92.   
93.     @Override  
94.     protected boolean doTryLock(long timeout, TimeUnit unit) {  
95.         try {  
96.             this.locker.set(new ZooKeeperLocker());  
97.             return this.locker.get().lock(timeout, unit);  
98.         } catch(Exception e) {  
99.             throw new NestableRuntimeException("failed to lock, directory: " + this.directory, e);  
100.         }  
101.     }  
102.   
103.     @Override  
104.     protected void doUnlock() {  
105.         try {  
106.             this.locker.get().unlock();  
107.         } catch(Exception e) {  
108.             throw new NestableRuntimeException("failed to unlock, directory: " + this.directory, e);  
109.         } finally {  
110.             this.locker.set(null);  
111.         }  
112.     }  
113.   
114.     /** 
115.      *  
116.      */  
117.     private class ZooKeeperLocker implements Watcher {  
118.         //  
119.         private volatile String name;  
120.         private volatile CountDownLatch latch;  
121.   
122.         /** 
123.          *  
124.          */  
125.         @Override  
126.         public void process(WatchedEvent event) {  
127.             //  
128.             if(this.latch != null) {  
129.                 this.latch.countDown();  
130.             }  
131.               
132.             //  
133.             if(isVerbose() && LOGGER.isInfoEnabled()) {  
134.                 LOGGER.info("received an event: {}", event);  
135.             }  
136.         }  
137.           
138.         public boolean lock(long timeout, TimeUnit unit) throws Exception {  
139.             boolean succeed = false;  
140.             try {  
141.                 do {  
142.                     final long mark = System.nanoTime();  
143.                     timeout = TimeUnit.NANOSECONDS.convert(timeout, unit);  
144.                     try {  
145.                         succeed = doLock(timeout, TimeUnit.NANOSECONDS);  
146.                         break;  
147.                     } catch (KeeperException.ConnectionLossException e) {  
148.                         timeout -= (System.nanoTime() - mark);  
149.                         if(isVerbose() && LOGGER.isInfoEnabled()) {  
150.                             LOGGER.info("connection was lost, directory: {}, name: {}, message: {}", new Object[]{directory, this.name, e.getMessage()});  
151.                         }  
152.                     }  
153.                 }  
154.                 while(timeout > 0);  
155.             } finally {  
156.                 if(!succeed) { // Unlock quietly  
157.                     try {  
158.                         unlock();  
159.                     } catch(Exception e) {  
160.                         LOGGER.warn("failed to unlock, directory: " + directory + ", name: " + this.name, e);  
161.                     }  
162.                 }  
163.             }  
164.             return succeed;  
165.         }  
166.           
167.         public void unlock() throws Exception {  
168.             try {  
169.                 zookeeper.delete(directory + "/" + this.name, -1);  
170.             } catch (KeeperException.NoNodeException e) {  
171.                 LOGGER.warn("node does NOT exist, directory: {}, name: {}, message: {}", new Object[]{directory, this.name, e.getMessage()});  
172.             } finally {  
173.                 this.name = null;  
174.             }  
175.         }  
176.           
177.         /** 
178.          *  
179.          */  
180.         private Boolean doLock(long timeout, TimeUnit unit) throws Exception {  
181.             boolean succeed = false;  
182.             do {  
183.                 //  
184.                 final long mark = System.nanoTime();  
185.                 timeout = TimeUnit.NANOSECONDS.convert(timeout, unit);  
186.   
187.                 //  
188.                 if (this.name == null) {  
189.                     this.name = findOrCreateChild();  
190.                 }  
191.                   
192.                 //  
193.                 final List<String> children = zookeeper.getChildren(directory, false);  
194.                 if (children.isEmpty()) {  
195.                     this.name = null;  
196.                     LOGGER.warn("could not find any child, directory: {}, name: {}", new Object[]{directory, this.name});  
197.                 } else {  
198.                     final SequenceComparator comparator = new SequenceComparator();  
199.                     Collections.sort(children, comparator);  
200.                     final int index = Collections.binarySearch(children, this.name, comparator);  
201.                     if (index > 0) { // Not the first one  
202.                         this.latch = new CountDownLatch(1);  
203.                         final String previous = children.get(index - 1);  
204.                         final Stat stat = zookeeper.exists(directory + "/" + previous, this);  
205.                         if (stat != null) {  
206.                             this.latch.await(timeout, TimeUnit.NANOSECONDS);  
207.                             this.latch = null;  
208.                         } else {  
209.                             LOGGER.warn("could not find the previous child, directory: {}, name: {}", new Object[]{directory, this.name});  
210.                         }  
211.                     } else {  
212.                         final String owner = children.get(0);  
213.                         if (this.name != null && owner != null && this.name.equals(owner)) {  
214.                             succeed = true;  
215.                         } else {  
216.                             LOGGER.warn("the lock should be held by current thread, directory: {}, name: {}, owner: {}", new Object[]{directory, this.name, owner});  
217.                         }  
218.                     }  
219.                 }  
220.                   
221.                 //  
222.                 timeout -= (System.nanoTime() - mark);  
223.             } while (!succeed && timeout >= 0);  
224.             return succeed;  
225.         }  
226.           
227.         private String findOrCreateChild() throws Exception {  
228.             //  
229.             final String prefix = zookeeper.getSessionId() + "-";  
230.             final List<String> children = zookeeper.getChildren(directory, false);  
231.             for (String child : children) {  
232.                 if (child.startsWith(prefix)) {  
233.                     if(isVerbose() && LOGGER.isInfoEnabled()) {  
234.                         LOGGER.info("found a child, directory: {}, child: {}", new Object[]{directory, child});  
235.                     }  
236.                     return child;  
237.                 }  
238.             }  
239.               
240.             //  
241.             final String data = Thread.currentThread().getId() + "@" + processName;  
242.             final String path = zookeeper.create(directory + "/" + prefix, data.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);  
243.             final String child = path.substring(path.lastIndexOf("/") + 1);  
244.             if(isVerbose() && LOGGER.isInfoEnabled()) {  
245.                 LOGGER.info("created a child, directory: {}, path: {}", new Object[]{directory, child});  
246.             }  
247.             return child;  
248.         }  
249.     }  
250.       
251.     /** 
252.      *  
253.      */  
254.     private static class SequenceComparator implements Comparator<String> {  
255.   
256.         @Override  
257.         public int compare(String lhs, String rhs) {  
258.             final int index1 = lhs.lastIndexOf('-');  
259.             final int index2 = rhs.lastIndexOf('-');  
260.             final int sequence1 = Integer.parseInt(lhs.substring(index1 + 1));  
261.             final int sequence2 = Integer.parseInt(rhs.substring(index2 + 1));  
262.             return sequence1 - sequence2;  
263.         }  
264.     }  
265. }  

    ZooKeeperLock是fair的，并且在Node中保存的数据是线程ID，进程ID以及主机名。需要注意的是，应该为ZooKeeper部署集群，此外还需要保证传入ZooKeeperLock构造函数中的ZooKepper实例已经跟Server建立的连接，否则zookeeper.getSessionId()会返回0，从而导致错误。

 

3 disclaimer

   笔者只对以上代码进行了简单的测试，因此可能存在错误，请慎重使用。如果发现问题，感谢反馈。