Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 129 |
|
0.00% |
0 / 12 |
CRAP | |
0.00% |
0 / 1 |
MemcLockManager | |
0.00% |
0 / 129 |
|
0.00% |
0 / 12 |
2970 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
20 | |||
getLocksOnServer | |
0.00% |
0 / 40 |
|
0.00% |
0 / 1 |
272 | |||
freeLocksOnServer | |
0.00% |
0 / 34 |
|
0.00% |
0 / 1 |
132 | |||
releaseAllLocks | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
isServerUp | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
getCache | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
30 | |||
recordKeyForPath | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
newLockArray | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
sanitizeLockArray | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
20 | |||
acquireMutexes | |
0.00% |
0 / 18 |
|
0.00% |
0 / 1 |
30 | |||
releaseMutexes | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
6 | |||
__destruct | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
12 |
1 | <?php |
2 | /** |
3 | * This program is free software; you can redistribute it and/or modify |
4 | * it under the terms of the GNU General Public License as published by |
5 | * the Free Software Foundation; either version 2 of the License, or |
6 | * (at your option) any later version. |
7 | * |
8 | * This program is distributed in the hope that it will be useful, |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
11 | * GNU General Public License for more details. |
12 | * |
13 | * You should have received a copy of the GNU General Public License along |
14 | * with this program; if not, write to the Free Software Foundation, Inc., |
15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
16 | * http://www.gnu.org/copyleft/gpl.html |
17 | * |
18 | * @file |
19 | */ |
20 | use Wikimedia\WaitConditionLoop; |
21 | |
22 | /** |
23 | * Manage locks using memcached servers. |
24 | * |
25 | * Version of LockManager based on using memcached servers. |
26 | * This is meant for multi-wiki systems that may share files. |
27 | * All locks are non-blocking, which avoids deadlocks. |
28 | * |
29 | * All lock requests for a resource, identified by a hash string, will map to one |
30 | * bucket. Each bucket maps to one or several peer servers, each running memcached. |
31 | * A majority of peers must agree for a lock to be acquired. |
32 | * |
33 | * @ingroup LockManager |
34 | * @since 1.20 |
35 | */ |
36 | class MemcLockManager extends QuorumLockManager { |
37 | /** @var array Mapping of lock types to the type actually used */ |
38 | protected $lockTypeMap = [ |
39 | self::LOCK_SH => self::LOCK_SH, |
40 | self::LOCK_UW => self::LOCK_SH, |
41 | self::LOCK_EX => self::LOCK_EX |
42 | ]; |
43 | |
44 | /** @var MemcachedBagOStuff[] Map of (server name => MemcachedBagOStuff) */ |
45 | protected $cacheServers = []; |
46 | /** @var MapCacheLRU Server status cache */ |
47 | protected $statusCache; |
48 | |
49 | /** |
50 | * Construct a new instance from configuration. |
51 | * |
52 | * @param array $config Parameters include: |
53 | * - lockServers : Associative array of server names to "<IP>:<port>" strings. |
54 | * - srvsByBucket : An array of up to 16 arrays, each containing the server names |
55 | * in a bucket. Each bucket should have an odd number of servers. |
56 | * If omitted, all servers will be in one bucket. [optional]. |
57 | * - memcConfig : Configuration array for MemcachedBagOStuff::construct() with an |
58 | * additional 'class' parameter specifying which MemcachedBagOStuff |
59 | * subclass to use. The server names will be injected. [optional] |
60 | * @throws Exception |
61 | */ |
62 | public function __construct( array $config ) { |
63 | parent::__construct( $config ); |
64 | |
65 | if ( isset( $config['srvsByBucket'] ) ) { |
66 | // Sanitize srvsByBucket config to prevent PHP errors |
67 | $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); |
68 | $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive |
69 | } else { |
70 | $this->srvsByBucket = [ array_keys( $config['lockServers'] ) ]; |
71 | } |
72 | |
73 | $memcConfig = $config['memcConfig'] ?? []; |
74 | $memcConfig += [ 'class' => MemcachedPhpBagOStuff::class ]; // default |
75 | |
76 | $class = $memcConfig['class']; |
77 | if ( !is_subclass_of( $class, MemcachedBagOStuff::class ) ) { |
78 | throw new InvalidArgumentException( "$class is not of type MemcachedBagOStuff." ); |
79 | } |
80 | |
81 | foreach ( $config['lockServers'] as $name => $address ) { |
82 | $params = [ 'servers' => [ $address ] ] + $memcConfig; |
83 | $this->cacheServers[$name] = new $class( $params ); |
84 | } |
85 | |
86 | $this->statusCache = new MapCacheLRU( 100 ); |
87 | } |
88 | |
89 | protected function getLocksOnServer( $lockSrv, array $pathsByType ) { |
90 | $status = StatusValue::newGood(); |
91 | |
92 | $memc = $this->getCache( $lockSrv ); |
93 | // List of affected paths |
94 | $paths = array_merge( ...array_values( $pathsByType ) ); |
95 | $paths = array_unique( $paths ); |
96 | // List of affected lock record keys |
97 | $keys = array_map( [ $this, 'recordKeyForPath' ], $paths ); |
98 | |
99 | // Lock all of the active lock record keys... |
100 | if ( !$this->acquireMutexes( $memc, $keys ) ) { |
101 | $status->fatal( 'lockmanager-fail-conflict' ); |
102 | return $status; |
103 | } |
104 | |
105 | // Fetch all the existing lock records... |
106 | $lockRecords = $memc->getMulti( $keys ); |
107 | |
108 | $now = time(); |
109 | // Check if the requested locks conflict with existing ones... |
110 | foreach ( $pathsByType as $type => $paths2 ) { |
111 | foreach ( $paths2 as $path ) { |
112 | $locksKey = $this->recordKeyForPath( $path ); |
113 | $locksHeld = isset( $lockRecords[$locksKey] ) |
114 | ? self::sanitizeLockArray( $lockRecords[$locksKey] ) |
115 | : self::newLockArray(); // init |
116 | foreach ( $locksHeld[self::LOCK_EX] as $session => $expiry ) { |
117 | if ( $expiry < $now ) { // stale? |
118 | unset( $locksHeld[self::LOCK_EX][$session] ); |
119 | } elseif ( $session !== $this->session ) { |
120 | $status->fatal( 'lockmanager-fail-conflict' ); |
121 | } |
122 | } |
123 | if ( $type === self::LOCK_EX ) { |
124 | foreach ( $locksHeld[self::LOCK_SH] as $session => $expiry ) { |
125 | if ( $expiry < $now ) { // stale? |
126 | unset( $locksHeld[self::LOCK_SH][$session] ); |
127 | } elseif ( $session !== $this->session ) { |
128 | $status->fatal( 'lockmanager-fail-conflict' ); |
129 | } |
130 | } |
131 | } |
132 | if ( $status->isOK() ) { |
133 | // Register the session in the lock record array |
134 | $locksHeld[$type][$this->session] = $now + $this->lockTTL; |
135 | // We will update this record if none of the other locks conflict |
136 | $lockRecords[$locksKey] = $locksHeld; |
137 | } |
138 | } |
139 | } |
140 | |
141 | // If there were no lock conflicts, update all the lock records... |
142 | if ( $status->isOK() ) { |
143 | foreach ( $paths as $path ) { |
144 | $locksKey = $this->recordKeyForPath( $path ); |
145 | $locksHeld = $lockRecords[$locksKey]; |
146 | $ok = $memc->set( $locksKey, $locksHeld, self::MAX_LOCK_TTL ); |
147 | if ( !$ok ) { |
148 | $status->fatal( 'lockmanager-fail-acquirelock', $path ); |
149 | } else { |
150 | $this->logger->debug( __METHOD__ . ": acquired lock on key $locksKey." ); |
151 | } |
152 | } |
153 | } |
154 | |
155 | // Unlock all of the active lock record keys... |
156 | $this->releaseMutexes( $memc, $keys ); |
157 | |
158 | return $status; |
159 | } |
160 | |
161 | protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { |
162 | $status = StatusValue::newGood(); |
163 | |
164 | $memc = $this->getCache( $lockSrv ); |
165 | // List of affected paths |
166 | $paths = array_merge( ...array_values( $pathsByType ) ); |
167 | $paths = array_unique( $paths ); |
168 | // List of affected lock record keys |
169 | $keys = array_map( [ $this, 'recordKeyForPath' ], $paths ); |
170 | |
171 | // Lock all of the active lock record keys... |
172 | if ( !$this->acquireMutexes( $memc, $keys ) ) { |
173 | foreach ( $paths as $path ) { |
174 | $status->fatal( 'lockmanager-fail-releaselock', $path ); |
175 | } |
176 | |
177 | return $status; |
178 | } |
179 | |
180 | // Fetch all the existing lock records... |
181 | $lockRecords = $memc->getMulti( $keys ); |
182 | |
183 | // Remove the requested locks from all records... |
184 | foreach ( $pathsByType as $type => $paths2 ) { |
185 | foreach ( $paths2 as $path ) { |
186 | $locksKey = $this->recordKeyForPath( $path ); // lock record |
187 | if ( !isset( $lockRecords[$locksKey] ) ) { |
188 | $status->warning( 'lockmanager-fail-releaselock', $path ); |
189 | continue; // nothing to do |
190 | } |
191 | $locksHeld = $this->sanitizeLockArray( $lockRecords[$locksKey] ); |
192 | if ( isset( $locksHeld[$type][$this->session] ) ) { |
193 | unset( $locksHeld[$type][$this->session] ); // unregister this session |
194 | $lockRecords[$locksKey] = $locksHeld; |
195 | } else { |
196 | $status->warning( 'lockmanager-fail-releaselock', $path ); |
197 | } |
198 | } |
199 | } |
200 | |
201 | // Persist the new lock record values... |
202 | foreach ( $paths as $path ) { |
203 | $locksKey = $this->recordKeyForPath( $path ); |
204 | if ( !isset( $lockRecords[$locksKey] ) ) { |
205 | continue; // nothing to do |
206 | } |
207 | $locksHeld = $lockRecords[$locksKey]; |
208 | if ( $locksHeld === $this->newLockArray() ) { |
209 | $ok = $memc->delete( $locksKey ); |
210 | } else { |
211 | $ok = $memc->set( $locksKey, $locksHeld, self::MAX_LOCK_TTL ); |
212 | } |
213 | if ( $ok ) { |
214 | $this->logger->debug( __METHOD__ . ": released lock on key $locksKey." ); |
215 | } else { |
216 | $status->fatal( 'lockmanager-fail-releaselock', $path ); |
217 | } |
218 | } |
219 | |
220 | // Unlock all of the active lock record keys... |
221 | $this->releaseMutexes( $memc, $keys ); |
222 | |
223 | return $status; |
224 | } |
225 | |
226 | /** |
227 | * @see QuorumLockManager::releaseAllLocks() |
228 | * @return StatusValue |
229 | */ |
230 | protected function releaseAllLocks() { |
231 | return StatusValue::newGood(); // not supported |
232 | } |
233 | |
234 | /** |
235 | * @see QuorumLockManager::isServerUp() |
236 | * @param string $lockSrv |
237 | * @return bool |
238 | */ |
239 | protected function isServerUp( $lockSrv ) { |
240 | return (bool)$this->getCache( $lockSrv ); |
241 | } |
242 | |
243 | /** |
244 | * Get the MemcachedBagOStuff object for a $lockSrv |
245 | * |
246 | * @param string $lockSrv Server name |
247 | * @return MemcachedBagOStuff|null |
248 | */ |
249 | protected function getCache( $lockSrv ) { |
250 | if ( !isset( $this->cacheServers[$lockSrv] ) ) { |
251 | throw new InvalidArgumentException( "Invalid cache server '$lockSrv'." ); |
252 | } |
253 | |
254 | $online = $this->statusCache->get( "online:$lockSrv", 30 ); |
255 | if ( $online === null ) { |
256 | $online = $this->cacheServers[$lockSrv]->set( __CLASS__ . ':ping', 1, 1 ); |
257 | if ( !$online ) { // server down? |
258 | $this->logger->warning( __METHOD__ . ": Could not contact $lockSrv." ); |
259 | } |
260 | $this->statusCache->set( "online:$lockSrv", (int)$online ); |
261 | } |
262 | |
263 | return $online ? $this->cacheServers[$lockSrv] : null; |
264 | } |
265 | |
266 | /** |
267 | * @param string $path |
268 | * @return string |
269 | */ |
270 | protected function recordKeyForPath( $path ) { |
271 | return implode( ':', [ __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ] ); |
272 | } |
273 | |
274 | /** |
275 | * @return array An empty lock structure for a key |
276 | */ |
277 | protected function newLockArray() { |
278 | return [ self::LOCK_SH => [], self::LOCK_EX => [] ]; |
279 | } |
280 | |
281 | /** |
282 | * @param array $a |
283 | * @return array An empty lock structure for a key |
284 | */ |
285 | protected function sanitizeLockArray( $a ) { |
286 | if ( is_array( $a ) && isset( $a[self::LOCK_EX] ) && isset( $a[self::LOCK_SH] ) ) { |
287 | return $a; |
288 | } |
289 | |
290 | $this->logger->error( __METHOD__ . ": reset invalid lock array." ); |
291 | |
292 | return $this->newLockArray(); |
293 | } |
294 | |
295 | /** |
296 | * @param MemcachedBagOStuff $memc |
297 | * @param array $keys List of keys to acquire |
298 | * @return bool |
299 | */ |
300 | protected function acquireMutexes( MemcachedBagOStuff $memc, array $keys ) { |
301 | $lockedKeys = []; |
302 | |
303 | // Acquire the keys in lexicographical order, to avoid deadlock problems. |
304 | // If P1 is waiting to acquire a key P2 has, P2 can't also be waiting for a key P1 has. |
305 | sort( $keys ); |
306 | |
307 | // Try to quickly loop to acquire the keys, but back off after a few rounds. |
308 | // This reduces memcached spam, especially in the rare case where a server acquires |
309 | // some lock keys and dies without releasing them. Lock keys expire after a few minutes. |
310 | $loop = new WaitConditionLoop( |
311 | static function () use ( $memc, $keys, &$lockedKeys ) { |
312 | foreach ( array_diff( $keys, $lockedKeys ) as $key ) { |
313 | if ( $memc->add( "$key:mutex", 1, 180 ) ) { // lock record |
314 | $lockedKeys[] = $key; |
315 | } |
316 | } |
317 | |
318 | return array_diff( $keys, $lockedKeys ) |
319 | ? WaitConditionLoop::CONDITION_CONTINUE |
320 | : true; |
321 | }, |
322 | 3.0 // timeout |
323 | ); |
324 | $loop->invoke(); |
325 | |
326 | if ( count( $lockedKeys ) != count( $keys ) ) { |
327 | $this->releaseMutexes( $memc, $lockedKeys ); // failed; release what was locked |
328 | return false; |
329 | } |
330 | |
331 | return true; |
332 | } |
333 | |
334 | /** |
335 | * @param MemcachedBagOStuff $memc |
336 | * @param array $keys List of acquired keys |
337 | */ |
338 | protected function releaseMutexes( MemcachedBagOStuff $memc, array $keys ) { |
339 | foreach ( $keys as $key ) { |
340 | $memc->delete( "$key:mutex" ); |
341 | } |
342 | } |
343 | |
344 | /** |
345 | * Make sure remaining locks get cleared |
346 | */ |
347 | public function __destruct() { |
348 | while ( count( $this->locksHeld ) ) { |
349 | foreach ( $this->locksHeld as $path => $locks ) { |
350 | $this->doUnlock( [ $path ], self::LOCK_EX ); |
351 | $this->doUnlock( [ $path ], self::LOCK_SH ); |
352 | } |
353 | } |
354 | } |
355 | } |