MediaWiki REL1_35
PoolCounterRedis.php
Go to the documentation of this file.
1<?php
20use Psr\Log\LoggerInterface;
21
55 protected $ring;
57 protected $pool;
59 protected $logger;
61 protected $serversByLabel;
63 protected $keySha1;
65 protected $lockTTL;
66
68 protected $conn;
70 protected $slot;
72 protected $onRelease;
74 protected $session;
76 protected $slotTime;
77
78 private const AWAKE_ONE = 1; // wake-up if when a slot can be taken from an existing process
79 private const AWAKE_ALL = 2; // wake-up if an existing process finishes and wake up such others
80
82 protected static $active = null;
83
84 public function __construct( $conf, $type, $key ) {
85 parent::__construct( $conf, $type, $key );
86
87 $this->serversByLabel = $conf['servers'];
88
89 $serverLabels = array_keys( $conf['servers'] );
90 $this->ring = new HashRing( array_fill_keys( $serverLabels, 10 ) );
91
92 $conf['redisConfig']['serializer'] = 'none'; // for use with Lua
93 $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] );
94 $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' );
95
96 $this->keySha1 = sha1( $this->key );
97 $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode
98 $this->lockTTL = $met ? 2 * $met : 3600;
99
100 if ( self::$active === null ) {
101 self::$active = [];
102 register_shutdown_function( [ __CLASS__, 'releaseAll' ] );
103 }
104 }
105
109 protected function getConnection() {
110 if ( !isset( $this->conn ) ) {
111 $conn = false;
112 $servers = $this->ring->getLocations( $this->key, 3 );
113 ArrayUtils::consistentHashSort( $servers, $this->key );
114 foreach ( $servers as $server ) {
115 $conn = $this->pool->getConnection( $this->serversByLabel[$server], $this->logger );
116 if ( $conn ) {
117 break;
118 }
119 }
120 if ( !$conn ) {
121 return Status::newFatal( 'pool-servererror', implode( ', ', $servers ) );
122 }
123 $this->conn = $conn;
124 }
125 return Status::newGood( $this->conn );
126 }
127
128 public function acquireForMe( $timeout = null ) {
129 $status = $this->precheckAcquire();
130 if ( !$status->isGood() ) {
131 return $status;
132 }
133
134 return $this->waitForSlotOrNotif( self::AWAKE_ONE, $timeout );
135 }
136
137 public function acquireForAnyone( $timeout = null ) {
138 $status = $this->precheckAcquire();
139 if ( !$status->isGood() ) {
140 return $status;
141 }
142
143 return $this->waitForSlotOrNotif( self::AWAKE_ALL, $timeout );
144 }
145
146 public function release() {
147 if ( $this->slot === null ) {
148 return Status::newGood( PoolCounter::NOT_LOCKED ); // not locked
149 }
150
151 $status = $this->getConnection();
152 if ( !$status->isOK() ) {
153 return $status;
154 }
156 $conn = $status->value;
157 '@phan-var RedisConnRef $conn';
158
159 // phpcs:disable Generic.Files.LineLength
160 static $script =
162<<<LUA
163 local kSlots,kSlotsNextRelease,kWakeup,kWaiting = unpack(KEYS)
164 local rMaxWorkers,rExpiry,rSlot,rSlotTime,rAwakeAll,rTime = unpack(ARGV)
165 -- Add the slots back to the list (if rSlot is "w" then it is not a slot).
166 -- Treat the list as expired if the "next release" time sorted-set is missing.
167 if rSlot ~= 'w' and redis.call('exists',kSlotsNextRelease) == 1 then
168 if 1*redis.call('zScore',kSlotsNextRelease,rSlot) ~= (rSlotTime + rExpiry) then
169 -- Slot lock expired and was released already
170 elseif redis.call('lLen',kSlots) >= 1*rMaxWorkers then
171 -- Slots somehow got out of sync; reset the list for sanity
172 redis.call('del',kSlots,kSlotsNextRelease)
173 elseif redis.call('lLen',kSlots) == (1*rMaxWorkers - 1) and redis.call('zCard',kWaiting) == 0 then
174 -- Slot list will be made full; clear it to save space (it re-inits as needed)
175 -- since nothing is waiting on being unblocked by a push to the list
176 redis.call('del',kSlots,kSlotsNextRelease)
177 else
178 -- Add slot back to pool and update the "next release" time
179 redis.call('rPush',kSlots,rSlot)
180 redis.call('zAdd',kSlotsNextRelease,rTime + 30,rSlot)
181 -- Always keep renewing the expiry on use
182 redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
183 redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
184 end
185 end
186 -- Update an ephemeral list to wake up other clients that can
187 -- reuse any cached work from this process. Only do this if no
188 -- slots are currently free (e.g. clients could be waiting).
189 if 1*rAwakeAll == 1 then
190 local count = redis.call('zCard',kWaiting)
191 for i = 1,count do
192 redis.call('rPush',kWakeup,'w')
193 end
194 redis.call('pexpire',kWakeup,1)
195 end
196 return 1
197LUA;
198 // phpcs:enable
199
200 try {
201 $conn->luaEval( $script,
202 [
203 $this->getSlotListKey(),
204 $this->getSlotRTimeSetKey(),
205 $this->getWakeupListKey(),
206 $this->getWaitSetKey(),
207 $this->workers,
208 $this->lockTTL,
209 $this->slot,
210 $this->slotTime, // used for CAS-style sanity check
211 ( $this->onRelease === self::AWAKE_ALL ) ? 1 : 0,
212 microtime( true )
213 ],
214 4 # number of first argument(s) that are keys
215 );
216 } catch ( RedisException $e ) {
217 return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
218 }
219
220 $this->slot = null;
221 $this->slotTime = null;
222 $this->onRelease = null;
223 unset( self::$active[$this->session] );
224
225 $this->onRelease();
226
227 return Status::newGood( PoolCounter::RELEASED );
228 }
229
235 protected function waitForSlotOrNotif( $doWakeup, $timeout = null ) {
236 if ( $this->slot !== null ) {
237 return Status::newGood( PoolCounter::LOCK_HELD ); // already acquired
238 }
239
240 $status = $this->getConnection();
241 if ( !$status->isOK() ) {
242 return $status;
243 }
245 $conn = $status->value;
246 '@phan-var RedisConnRef $conn';
247
248 $now = microtime( true );
250 try {
251 $slot = $this->initAndPopPoolSlotList( $conn, $now );
252 if ( ctype_digit( $slot ) ) {
253 // Pool slot acquired by this process
254 $slotTime = $now;
255 } elseif ( $slot === 'QUEUE_FULL' ) {
256 // Too many processes are waiting for pooled processes to finish
257 return Status::newGood( PoolCounter::QUEUE_FULL );
258 } elseif ( $slot === 'QUEUE_WAIT' ) {
259 // This process is now registered as waiting
260 $keys = ( $doWakeup == self::AWAKE_ALL )
261 // Wait for an open slot or wake-up signal (preferring the latter)
262 ? [ $this->getWakeupListKey(), $this->getSlotListKey() ]
263 // Just wait for an actual pool slot
264 : [ $this->getSlotListKey() ];
265
266 $res = $conn->blPop( $keys, $timeout );
267 if ( $res === [] ) {
268 $conn->zRem( $this->getWaitSetKey(), $this->session ); // no longer waiting
269 return Status::newGood( PoolCounter::TIMEOUT );
270 }
271
272 $slot = $res[1]; // pool slot or "w" for wake-up notifications
273 $slotTime = microtime( true ); // last microtime() was a few RTTs ago
274 // Unregister this process as waiting and bump slot "next release" time
276 } else {
277 return Status::newFatal( 'pool-error-unknown', "Server gave slot '$slot'." );
278 }
279 } catch ( RedisException $e ) {
280 return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
281 }
282
283 if ( $slot !== 'w' ) {
284 $this->slot = $slot;
285 $this->slotTime = $slotTime;
286 $this->onRelease = $doWakeup;
287 self::$active[$this->session] = $this;
288 }
289
290 $this->onAcquire();
291
292 return Status::newGood( $slot === 'w' ? PoolCounter::DONE : PoolCounter::LOCKED );
293 }
294
300 protected function initAndPopPoolSlotList( RedisConnRef $conn, $now ) {
301 static $script =
303<<<LUA
304 local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
305 local rMaxWorkers,rMaxQueue,rTimeout,rExpiry,rSess,rTime = unpack(ARGV)
306 -- Initialize if the "next release" time sorted-set is empty. The slot key
307 -- itself is empty if all slots are busy or when nothing is initialized.
308 -- If the list is empty but the set is not, then it is the latter case.
309 -- For sanity, if the list exists but not the set, then reset everything.
310 if redis.call('exists',kSlotsNextRelease) == 0 then
311 redis.call('del',kSlots)
312 for i = 1,1*rMaxWorkers do
313 redis.call('rPush',kSlots,i)
314 redis.call('zAdd',kSlotsNextRelease,-1,i)
315 end
316 -- Otherwise do maintenance to clean up after network partitions
317 else
318 -- Find stale slot locks and add free them (avoid duplicates for sanity)
319 local staleLocks = redis.call('zRangeByScore',kSlotsNextRelease,0,rTime)
320 for k,slot in ipairs(staleLocks) do
321 redis.call('lRem',kSlots,0,slot)
322 redis.call('rPush',kSlots,slot)
323 redis.call('zAdd',kSlotsNextRelease,rTime + 30,slot)
324 end
325 -- Find stale wait slot entries and remove them
326 redis.call('zRemRangeByScore',kSlotWaits,0,rTime - 2*rTimeout)
327 end
328 local slot
329 -- Try to acquire a slot if possible now
330 if redis.call('lLen',kSlots) > 0 then
331 slot = redis.call('lPop',kSlots)
332 -- Update the slot "next release" time
333 redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,slot)
334 elseif redis.call('zCard',kSlotWaits) >= 1*rMaxQueue then
335 slot = 'QUEUE_FULL'
336 else
337 slot = 'QUEUE_WAIT'
338 -- Register this process as waiting
339 redis.call('zAdd',kSlotWaits,rTime,rSess)
340 redis.call('expireAt',kSlotWaits,math.ceil(rTime + 2*rTimeout))
341 end
342 -- Always keep renewing the expiry on use
343 redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
344 redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
345 return slot
346LUA;
347 return $conn->luaEval( $script,
348 [
349 $this->getSlotListKey(),
350 $this->getSlotRTimeSetKey(),
351 $this->getWaitSetKey(),
352 $this->workers,
353 $this->maxqueue,
354 $this->timeout,
355 $this->lockTTL,
356 $this->session,
357 $now
358 ],
359 3 # number of first argument(s) that are keys
360 );
361 }
362
369 protected function registerAcquisitionTime( RedisConnRef $conn, $slot, $now ) {
370 static $script =
372<<<LUA
373 local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
374 local rSlot,rExpiry,rSess,rTime = unpack(ARGV)
375 -- If rSlot is 'w' then the client was told to wake up but got no slot
376 if rSlot ~= 'w' then
377 -- Update the slot "next release" time
378 redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,rSlot)
379 -- Always keep renewing the expiry on use
380 redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
381 redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
382 end
383 -- Unregister this process as waiting
384 redis.call('zRem',kSlotWaits,rSess)
385 return 1
386LUA;
387 return $conn->luaEval( $script,
388 [
389 $this->getSlotListKey(),
390 $this->getSlotRTimeSetKey(),
391 $this->getWaitSetKey(),
392 $slot,
393 $this->lockTTL,
394 $this->session,
395 $now
396 ],
397 3 # number of first argument(s) that are keys
398 );
399 }
400
404 protected function getSlotListKey() {
405 return "poolcounter:l-slots-{$this->keySha1}-{$this->workers}";
406 }
407
411 protected function getSlotRTimeSetKey() {
412 return "poolcounter:z-renewtime-{$this->keySha1}-{$this->workers}";
413 }
414
418 protected function getWaitSetKey() {
419 return "poolcounter:z-wait-{$this->keySha1}-{$this->workers}";
420 }
421
425 protected function getWakeupListKey() {
426 return "poolcounter:l-wakeup-{$this->keySha1}-{$this->workers}";
427 }
428
432 public static function releaseAll() {
433 foreach ( self::$active as $poolCounter ) {
434 try {
435 if ( $poolCounter->slot !== null ) {
436 $poolCounter->release();
437 }
438 } catch ( Exception $e ) {
439 }
440 }
441 }
442}
Convenience class for weighted consistent hash rings.
Definition HashRing.php:44
Version of PoolCounter that uses Redis.
int $onRelease
AWAKE_* constant.
acquireForAnyone( $timeout=null)
I want to do this task, but if anyone else does it instead, it's also fine for me.
__construct( $conf, $type, $key)
LoggerInterface $logger
string $slot
Pool slot value.
int $slotTime
UNIX timestamp.
initAndPopPoolSlotList(RedisConnRef $conn, $now)
array $serversByLabel
(server label => host) map
string $session
Unique string to identify this process.
static releaseAll()
Try to make sure that locks get released (even with exceptions and fatals)
release()
I have successfully finished my task.
static PoolCounterRedis[] $active
List of active PoolCounterRedis objects in this script.
waitForSlotOrNotif( $doWakeup, $timeout=null)
registerAcquisitionTime(RedisConnRef $conn, $slot, $now)
acquireForMe( $timeout=null)
I want to do this task and I need to do it myself.
int $lockTTL
TTL for locks to expire (work should finish in this time)
string $keySha1
SHA-1 of the key.
RedisConnectionPool $pool
When you have many workers (threads/servers) giving service, and a cached item expensive to produce e...
const QUEUE_FULL
onAcquire()
Update any lock tracking information when the lock is acquired.
string $key
All workers with the same key share the lock.
precheckAcquire()
Checks that the lock request is sane.
const NOT_LOCKED
int $timeout
Maximum time in seconds to wait for the lock.
const LOCK_HELD
onRelease()
Update any lock tracking information when the lock is released.
Helper class to handle automatically marking connectons as reusable (via RAII pattern)
luaEval( $script, array $params, $numKeys)
Helper class to manage Redis connections.
static singleton(array $options)