MediaWiki  master
PoolCounterRedis.php
Go to the documentation of this file.
1 <?php
20 use Psr\Log\LoggerInterface;
21 
55  protected $ring;
57  protected $pool;
59  protected $logger;
61  protected $serversByLabel;
63  protected $keySha1;
65  protected $lockTTL;
66 
68  protected $conn;
70  protected $slot;
72  protected $onRelease;
74  protected $session;
76  protected $slotTime;
77 
78  private const AWAKE_ONE = 1; // wake-up if when a slot can be taken from an existing process
79  private const AWAKE_ALL = 2; // wake-up if an existing process finishes and wake up such others
80 
82  protected static $active = null;
83 
84  public function __construct( $conf, $type, $key ) {
85  parent::__construct( $conf, $type, $key );
86 
87  $this->serversByLabel = $conf['servers'];
88 
89  $serverLabels = array_keys( $conf['servers'] );
90  $this->ring = new HashRing( array_fill_keys( $serverLabels, 10 ) );
91 
92  $conf['redisConfig']['serializer'] = 'none'; // for use with Lua
93  $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] );
94  $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' );
95 
96  $this->keySha1 = sha1( $this->key );
97  $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode
98  $this->lockTTL = $met ? 2 * $met : 3600;
99 
100  if ( self::$active === null ) {
101  self::$active = [];
102  register_shutdown_function( [ __CLASS__, 'releaseAll' ] );
103  }
104  }
105 
109  protected function getConnection() {
110  if ( !isset( $this->conn ) ) {
111  $conn = false;
112  $servers = $this->ring->getLocations( $this->key, 3 );
113  ArrayUtils::consistentHashSort( $servers, $this->key );
114  foreach ( $servers as $server ) {
115  $conn = $this->pool->getConnection( $this->serversByLabel[$server], $this->logger );
116  if ( $conn ) {
117  break;
118  }
119  }
120  if ( !$conn ) {
121  return Status::newFatal( 'pool-servererror', implode( ', ', $servers ) );
122  }
123  $this->conn = $conn;
124  }
125  return Status::newGood( $this->conn );
126  }
127 
128  public function acquireForMe( $timeout = null ) {
129  $status = $this->precheckAcquire();
130  if ( !$status->isGood() ) {
131  return $status;
132  }
133 
134  return $this->waitForSlotOrNotif( self::AWAKE_ONE, $timeout );
135  }
136 
137  public function acquireForAnyone( $timeout = null ) {
138  $status = $this->precheckAcquire();
139  if ( !$status->isGood() ) {
140  return $status;
141  }
142 
143  return $this->waitForSlotOrNotif( self::AWAKE_ALL, $timeout );
144  }
145 
146  public function release() {
147  if ( $this->slot === null ) {
148  return Status::newGood( PoolCounter::NOT_LOCKED ); // not locked
149  }
150 
151  $status = $this->getConnection();
152  if ( !$status->isOK() ) {
153  return $status;
154  }
156  $conn = $status->value;
157  '@phan-var RedisConnRef $conn';
158 
159  // phpcs:disable Generic.Files.LineLength
160  static $script =
162 <<<LUA
163  local kSlots,kSlotsNextRelease,kWakeup,kWaiting = unpack(KEYS)
164  local rMaxWorkers,rExpiry,rSlot,rSlotTime,rAwakeAll,rTime = unpack(ARGV)
165  -- Add the slots back to the list (if rSlot is "w" then it is not a slot).
166  -- Treat the list as expired if the "next release" time sorted-set is missing.
167  if rSlot ~= 'w' and redis.call('exists',kSlotsNextRelease) == 1 then
168  if 1*redis.call('zScore',kSlotsNextRelease,rSlot) ~= (rSlotTime + rExpiry) then
169  -- Slot lock expired and was released already
170  elseif redis.call('lLen',kSlots) >= 1*rMaxWorkers then
171  -- Slots somehow got out of sync; reset the list for sanity
172  redis.call('del',kSlots,kSlotsNextRelease)
173  elseif redis.call('lLen',kSlots) == (1*rMaxWorkers - 1) and redis.call('zCard',kWaiting) == 0 then
174  -- Slot list will be made full; clear it to save space (it re-inits as needed)
175  -- since nothing is waiting on being unblocked by a push to the list
176  redis.call('del',kSlots,kSlotsNextRelease)
177  else
178  -- Add slot back to pool and update the "next release" time
179  redis.call('rPush',kSlots,rSlot)
180  redis.call('zAdd',kSlotsNextRelease,rTime + 30,rSlot)
181  -- Always keep renewing the expiry on use
182  redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
183  redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
184  end
185  end
186  -- Update an ephemeral list to wake up other clients that can
187  -- reuse any cached work from this process. Only do this if no
188  -- slots are currently free (e.g. clients could be waiting).
189  if 1*rAwakeAll == 1 then
190  local count = redis.call('zCard',kWaiting)
191  for i = 1,count do
192  redis.call('rPush',kWakeup,'w')
193  end
194  redis.call('pexpire',kWakeup,1)
195  end
196  return 1
197 LUA;
198  // phpcs:enable
199 
200  try {
201  $conn->luaEval( $script,
202  [
203  $this->getSlotListKey(),
204  $this->getSlotRTimeSetKey(),
205  $this->getWakeupListKey(),
206  $this->getWaitSetKey(),
207  $this->workers,
208  $this->lockTTL,
209  $this->slot,
210  $this->slotTime, // used for CAS-style sanity check
211  ( $this->onRelease === self::AWAKE_ALL ) ? 1 : 0,
212  microtime( true )
213  ],
214  4 # number of first argument(s) that are keys
215  );
216  } catch ( RedisException $e ) {
217  return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
218  }
219 
220  $this->slot = null;
221  $this->slotTime = null;
222  $this->onRelease = null;
223  unset( self::$active[$this->session] );
224 
225  $this->onRelease();
226 
228  }
229 
235  protected function waitForSlotOrNotif( $doWakeup, $timeout = null ) {
236  if ( $this->slot !== null ) {
237  return Status::newGood( PoolCounter::LOCK_HELD ); // already acquired
238  }
239 
240  $status = $this->getConnection();
241  if ( !$status->isOK() ) {
242  return $status;
243  }
245  $conn = $status->value;
246  '@phan-var RedisConnRef $conn';
247 
248  $now = microtime( true );
250  try {
251  $slot = $this->initAndPopPoolSlotList( $conn, $now );
252  if ( ctype_digit( $slot ) ) {
253  // Pool slot acquired by this process
254  $slotTime = $now;
255  } elseif ( $slot === 'QUEUE_FULL' ) {
256  // Too many processes are waiting for pooled processes to finish
258  } elseif ( $slot === 'QUEUE_WAIT' ) {
259  // This process is now registered as waiting
260  $keys = ( $doWakeup == self::AWAKE_ALL )
261  // Wait for an open slot or wake-up signal (preferring the latter)
262  ? [ $this->getWakeupListKey(), $this->getSlotListKey() ]
263  // Just wait for an actual pool slot
264  : [ $this->getSlotListKey() ];
265 
266  $res = $conn->blPop( $keys, $timeout );
267  if ( $res === [] ) {
268  $conn->zRem( $this->getWaitSetKey(), $this->session ); // no longer waiting
270  }
271 
272  $slot = $res[1]; // pool slot or "w" for wake-up notifications
273  $slotTime = microtime( true ); // last microtime() was a few RTTs ago
274  // Unregister this process as waiting and bump slot "next release" time
276  } else {
277  return Status::newFatal( 'pool-error-unknown', "Server gave slot '$slot'." );
278  }
279  } catch ( RedisException $e ) {
280  return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
281  }
282 
283  if ( $slot !== 'w' ) {
284  $this->slot = $slot;
285  $this->slotTime = $slotTime;
286  $this->onRelease = $doWakeup;
287  self::$active[$this->session] = $this;
288  }
289 
290  $this->onAcquire();
291 
293  }
294 
300  protected function initAndPopPoolSlotList( RedisConnRef $conn, $now ) {
301  static $script =
303 <<<LUA
304  local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
305  local rMaxWorkers,rMaxQueue,rTimeout,rExpiry,rSess,rTime = unpack(ARGV)
306  -- Initialize if the "next release" time sorted-set is empty. The slot key
307  -- itself is empty if all slots are busy or when nothing is initialized.
308  -- If the list is empty but the set is not, then it is the latter case.
309  -- For sanity, if the list exists but not the set, then reset everything.
310  if redis.call('exists',kSlotsNextRelease) == 0 then
311  redis.call('del',kSlots)
312  for i = 1,1*rMaxWorkers do
313  redis.call('rPush',kSlots,i)
314  redis.call('zAdd',kSlotsNextRelease,-1,i)
315  end
316  -- Otherwise do maintenance to clean up after network partitions
317  else
318  -- Find stale slot locks and add free them (avoid duplicates for sanity)
319  local staleLocks = redis.call('zRangeByScore',kSlotsNextRelease,0,rTime)
320  for k,slot in ipairs(staleLocks) do
321  redis.call('lRem',kSlots,0,slot)
322  redis.call('rPush',kSlots,slot)
323  redis.call('zAdd',kSlotsNextRelease,rTime + 30,slot)
324  end
325  -- Find stale wait slot entries and remove them
326  redis.call('zRemRangeByScore',kSlotWaits,0,rTime - 2*rTimeout)
327  end
328  local slot
329  -- Try to acquire a slot if possible now
330  if redis.call('lLen',kSlots) > 0 then
331  slot = redis.call('lPop',kSlots)
332  -- Update the slot "next release" time
333  redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,slot)
334  elseif redis.call('zCard',kSlotWaits) >= 1*rMaxQueue then
335  slot = 'QUEUE_FULL'
336  else
337  slot = 'QUEUE_WAIT'
338  -- Register this process as waiting
339  redis.call('zAdd',kSlotWaits,rTime,rSess)
340  redis.call('expireAt',kSlotWaits,math.ceil(rTime + 2*rTimeout))
341  end
342  -- Always keep renewing the expiry on use
343  redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
344  redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
345  return slot
346 LUA;
347  return $conn->luaEval( $script,
348  [
349  $this->getSlotListKey(),
350  $this->getSlotRTimeSetKey(),
351  $this->getWaitSetKey(),
352  $this->workers,
353  $this->maxqueue,
354  $this->timeout,
355  $this->lockTTL,
356  $this->session,
357  $now
358  ],
359  3 # number of first argument(s) that are keys
360  );
361  }
362 
369  protected function registerAcquisitionTime( RedisConnRef $conn, $slot, $now ) {
370  static $script =
372 <<<LUA
373  local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
374  local rSlot,rExpiry,rSess,rTime = unpack(ARGV)
375  -- If rSlot is 'w' then the client was told to wake up but got no slot
376  if rSlot ~= 'w' then
377  -- Update the slot "next release" time
378  redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,rSlot)
379  -- Always keep renewing the expiry on use
380  redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
381  redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
382  end
383  -- Unregister this process as waiting
384  redis.call('zRem',kSlotWaits,rSess)
385  return 1
386 LUA;
387  return $conn->luaEval( $script,
388  [
389  $this->getSlotListKey(),
390  $this->getSlotRTimeSetKey(),
391  $this->getWaitSetKey(),
392  $slot,
393  $this->lockTTL,
394  $this->session,
395  $now
396  ],
397  3 # number of first argument(s) that are keys
398  );
399  }
400 
404  protected function getSlotListKey() {
405  return "poolcounter:l-slots-{$this->keySha1}-{$this->workers}";
406  }
407 
411  protected function getSlotRTimeSetKey() {
412  return "poolcounter:z-renewtime-{$this->keySha1}-{$this->workers}";
413  }
414 
418  protected function getWaitSetKey() {
419  return "poolcounter:z-wait-{$this->keySha1}-{$this->workers}";
420  }
421 
425  protected function getWakeupListKey() {
426  return "poolcounter:l-wakeup-{$this->keySha1}-{$this->workers}";
427  }
428 
432  public static function releaseAll() {
433  foreach ( self::$active as $poolCounter ) {
434  try {
435  if ( $poolCounter->slot !== null ) {
436  $poolCounter->release();
437  }
438  } catch ( Exception $e ) {
439  }
440  }
441  }
442 }
PoolCounterRedis\AWAKE_ONE
const AWAKE_ONE
Definition: PoolCounterRedis.php:78
PoolCounterRedis\$onRelease
int $onRelease
AWAKE_* constant.
Definition: PoolCounterRedis.php:72
ArrayUtils\consistentHashSort
static consistentHashSort(&$array, $key, $separator="\000")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
RedisConnectionPool\singleton
static singleton(array $options)
Definition: RedisConnectionPool.php:149
PoolCounterRedis\getWakeupListKey
getWakeupListKey()
Definition: PoolCounterRedis.php:425
StatusValue\newFatal
static newFatal( $message,... $parameters)
Factory function for fatal errors.
Definition: StatusValue.php:70
PoolCounterRedis\$conn
RedisConnRef $conn
Definition: PoolCounterRedis.php:68
PoolCounterRedis\getConnection
getConnection()
Definition: PoolCounterRedis.php:109
PoolCounterRedis\$slot
string $slot
Pool slot value.
Definition: PoolCounterRedis.php:70
PoolCounter\onAcquire
onAcquire()
Update any lock tracking information when the lock is acquired.
Definition: PoolCounter.php:193
PoolCounterRedis\$lockTTL
int $lockTTL
TTL for locks to expire (work should finish in this time)
Definition: PoolCounterRedis.php:65
PoolCounter\LOCK_HELD
const LOCK_HELD
Definition: PoolCounter.php:55
MediaWiki\Logger\LoggerFactory\getInstance
static getInstance( $channel)
Get a named logger instance from the currently configured logger factory.
Definition: LoggerFactory.php:92
PoolCounterRedis\$pool
RedisConnectionPool $pool
Definition: PoolCounterRedis.php:57
PoolCounter\NOT_LOCKED
const NOT_LOCKED
Definition: PoolCounter.php:52
PoolCounterRedis\acquireForMe
acquireForMe( $timeout=null)
I want to do this task and I need to do it myself.
Definition: PoolCounterRedis.php:128
PoolCounter\LOCKED
const LOCKED
Definition: PoolCounter.php:47
PoolCounterRedis\AWAKE_ALL
const AWAKE_ALL
Definition: PoolCounterRedis.php:79
PoolCounterRedis\$session
string $session
Unique string to identify this process.
Definition: PoolCounterRedis.php:74
$res
$res
Definition: testCompression.php:57
PoolCounterRedis\initAndPopPoolSlotList
initAndPopPoolSlotList(RedisConnRef $conn, $now)
Definition: PoolCounterRedis.php:300
PoolCounter\onRelease
onRelease()
Update any lock tracking information when the lock is released.
Definition: PoolCounter.php:201
PoolCounter
When you have many workers (threads/servers) giving service, and a cached item expensive to produce e...
Definition: PoolCounter.php:45
PoolCounterRedis\acquireForAnyone
acquireForAnyone( $timeout=null)
I want to do this task, but if anyone else does it instead, it's also fine for me.
Definition: PoolCounterRedis.php:137
PoolCounterRedis\$serversByLabel
array $serversByLabel
(server label => host) map
Definition: PoolCounterRedis.php:61
PoolCounterRedis\$logger
LoggerInterface $logger
Definition: PoolCounterRedis.php:59
PoolCounterRedis\waitForSlotOrNotif
waitForSlotOrNotif( $doWakeup, $timeout=null)
Definition: PoolCounterRedis.php:235
PoolCounter\$key
string $key
All workers with the same key share the lock.
Definition: PoolCounter.php:58
PoolCounterRedis\$ring
HashRing $ring
Definition: PoolCounterRedis.php:55
RedisConnectionPool
Helper class to manage Redis connections.
Definition: RedisConnectionPool.php:41
PoolCounter\QUEUE_FULL
const QUEUE_FULL
Definition: PoolCounter.php:53
StatusValue\newGood
static newGood( $value=null)
Factory function for good results.
Definition: StatusValue.php:82
PoolCounterRedis\registerAcquisitionTime
registerAcquisitionTime(RedisConnRef $conn, $slot, $now)
Definition: PoolCounterRedis.php:369
PoolCounterRedis\releaseAll
static releaseAll()
Try to make sure that locks get released (even with exceptions and fatals)
Definition: PoolCounterRedis.php:432
RedisConnRef\luaEval
luaEval( $script, array $params, $numKeys)
Definition: RedisConnRef.php:244
PoolCounterRedis\getWaitSetKey
getWaitSetKey()
Definition: PoolCounterRedis.php:418
PoolCounterRedis\__construct
__construct( $conf, $type, $key)
Definition: PoolCounterRedis.php:84
PoolCounterRedis\$slotTime
int $slotTime
UNIX timestamp.
Definition: PoolCounterRedis.php:76
PoolCounterRedis\getSlotRTimeSetKey
getSlotRTimeSetKey()
Definition: PoolCounterRedis.php:411
RedisConnRef
Helper class to handle automatically marking connectons as reusable (via RAII pattern)
Definition: RedisConnRef.php:31
$keys
$keys
Definition: testCompression.php:72
PoolCounterRedis\release
release()
I have successfully finished my task.
Definition: PoolCounterRedis.php:146
PoolCounterRedis\getSlotListKey
getSlotListKey()
Definition: PoolCounterRedis.php:404
PoolCounterRedis\$active
static PoolCounterRedis[] $active
List of active PoolCounterRedis objects in this script.
Definition: PoolCounterRedis.php:82
PoolCounter\RELEASED
const RELEASED
Definition: PoolCounter.php:48
PoolCounter\TIMEOUT
const TIMEOUT
Definition: PoolCounter.php:54
HashRing
Convenience class for weighted consistent hash rings.
Definition: HashRing.php:44
PoolCounter\precheckAcquire
precheckAcquire()
Checks that the lock request is sane.
Definition: PoolCounter.php:168
PoolCounter\DONE
const DONE
Definition: PoolCounter.php:49
PoolCounter\$timeout
int $timeout
Maximum time in seconds to wait for the lock.
Definition: PoolCounter.php:71
PoolCounterRedis
Version of PoolCounter that uses Redis.
Definition: PoolCounterRedis.php:53
PoolCounterRedis\$keySha1
string $keySha1
SHA-1 of the key.
Definition: PoolCounterRedis.php:63
$type
$type
Definition: testCompression.php:52