MediaWiki  master
PoolCounterRedis.php
Go to the documentation of this file.
1 <?php
22 use Psr\Log\LoggerInterface;
23 
57  protected $ring;
59  protected $pool;
61  protected $logger;
63  protected $serversByLabel;
65  protected $keySha1;
67  protected $lockTTL;
69  protected $conn;
71  protected $slot;
73  protected $onRelease;
75  protected $session;
77  protected $slotTime;
78 
79  private const AWAKE_ONE = 1; // wake-up if when a slot can be taken from an existing process
80  private const AWAKE_ALL = 2; // wake-up if an existing process finishes and wake up such others
81 
83  protected static $active = null;
84 
85  public function __construct( $conf, $type, $key ) {
86  parent::__construct( $conf, $type, $key );
87 
88  $this->serversByLabel = $conf['servers'];
89 
90  $serverLabels = array_keys( $conf['servers'] );
91  $this->ring = new HashRing( array_fill_keys( $serverLabels, 10 ) );
92 
93  $conf['redisConfig']['serializer'] = 'none'; // for use with Lua
94  $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] );
95  $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' );
96 
97  $this->keySha1 = sha1( $this->key );
98  $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode
99  $this->lockTTL = $met ? 2 * (int)$met : 3600;
100 
101  if ( self::$active === null ) {
102  self::$active = [];
103  register_shutdown_function( [ __CLASS__, 'releaseAll' ] );
104  }
105  }
106 
110  protected function getConnection() {
111  if ( !isset( $this->conn ) ) {
112  $conn = false;
113  $servers = $this->ring->getLocations( $this->key, 3 );
114  ArrayUtils::consistentHashSort( $servers, $this->key );
115  foreach ( $servers as $server ) {
116  $conn = $this->pool->getConnection( $this->serversByLabel[$server], $this->logger );
117  if ( $conn ) {
118  break;
119  }
120  }
121  if ( !$conn ) {
122  return Status::newFatal( 'pool-servererror', implode( ', ', $servers ) );
123  }
124  $this->conn = $conn;
125  }
126  return Status::newGood( $this->conn );
127  }
128 
129  public function acquireForMe( $timeout = null ) {
130  $status = $this->precheckAcquire();
131  if ( !$status->isGood() ) {
132  return $status;
133  }
134 
135  return $this->waitForSlotOrNotif( self::AWAKE_ONE, $timeout );
136  }
137 
138  public function acquireForAnyone( $timeout = null ) {
139  $status = $this->precheckAcquire();
140  if ( !$status->isGood() ) {
141  return $status;
142  }
143 
144  return $this->waitForSlotOrNotif( self::AWAKE_ALL, $timeout );
145  }
146 
147  public function release() {
148  if ( $this->slot === null ) {
149  return Status::newGood( PoolCounter::NOT_LOCKED ); // not locked
150  }
151 
152  $status = $this->getConnection();
153  if ( !$status->isOK() ) {
154  return $status;
155  }
157  $conn = $status->value;
158  '@phan-var RedisConnRef $conn';
159 
160  // phpcs:disable Generic.Files.LineLength
161  static $script =
163 <<<LUA
164  local kSlots,kSlotsNextRelease,kWakeup,kWaiting = unpack(KEYS)
165  local rMaxWorkers,rExpiry,rSlot,rSlotTime,rAwakeAll,rTime = unpack(ARGV)
166  -- Add the slots back to the list (if rSlot is "w" then it is not a slot).
167  -- Treat the list as expired if the "next release" time sorted-set is missing.
168  if rSlot ~= 'w' and redis.call('exists',kSlotsNextRelease) == 1 then
169  if 1*redis.call('zScore',kSlotsNextRelease,rSlot) ~= (rSlotTime + rExpiry) then
170  -- Slot lock expired and was released already
171  elseif redis.call('lLen',kSlots) >= 1*rMaxWorkers then
172  -- Slots somehow got out of sync; reset the list
173  redis.call('del',kSlots,kSlotsNextRelease)
174  elseif redis.call('lLen',kSlots) == (1*rMaxWorkers - 1) and redis.call('zCard',kWaiting) == 0 then
175  -- Slot list will be made full; clear it to save space (it re-inits as needed)
176  -- since nothing is waiting on being unblocked by a push to the list
177  redis.call('del',kSlots,kSlotsNextRelease)
178  else
179  -- Add slot back to pool and update the "next release" time
180  redis.call('rPush',kSlots,rSlot)
181  redis.call('zAdd',kSlotsNextRelease,rTime + 30,rSlot)
182  -- Always keep renewing the expiry on use
183  redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
184  redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
185  end
186  end
187  -- Update an ephemeral list to wake up other clients that can
188  -- reuse any cached work from this process. Only do this if no
189  -- slots are currently free (e.g. clients could be waiting).
190  if 1*rAwakeAll == 1 then
191  local count = redis.call('zCard',kWaiting)
192  for i = 1,count do
193  redis.call('rPush',kWakeup,'w')
194  end
195  redis.call('pexpire',kWakeup,1)
196  end
197  return 1
198 LUA;
199  // phpcs:enable
200 
201  try {
202  $conn->luaEval( $script,
203  [
204  $this->getSlotListKey(),
205  $this->getSlotRTimeSetKey(),
206  $this->getWakeupListKey(),
207  $this->getWaitSetKey(),
208  $this->workers,
209  $this->lockTTL,
210  $this->slot,
211  $this->slotTime, // used for CAS-style check
212  ( $this->onRelease === self::AWAKE_ALL ) ? 1 : 0,
213  microtime( true )
214  ],
215  4 # number of first argument(s) that are keys
216  );
217  } catch ( RedisException $e ) {
218  return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
219  }
220 
221  $this->slot = null;
222  $this->slotTime = null;
223  $this->onRelease = null;
224  unset( self::$active[$this->session] );
225 
226  $this->onRelease();
227 
228  return Status::newGood( PoolCounter::RELEASED );
229  }
230 
236  protected function waitForSlotOrNotif( $doWakeup, $timeout = null ) {
237  if ( $this->slot !== null ) {
238  return Status::newGood( PoolCounter::LOCK_HELD ); // already acquired
239  }
240 
241  $status = $this->getConnection();
242  if ( !$status->isOK() ) {
243  return $status;
244  }
246  $conn = $status->value;
247  '@phan-var RedisConnRef $conn';
248 
249  $now = microtime( true );
251  try {
252  $slot = $this->initAndPopPoolSlotList( $conn, $now );
253  if ( ctype_digit( $slot ) ) {
254  // Pool slot acquired by this process
255  $slotTime = $now;
256  } elseif ( $slot === 'QUEUE_FULL' ) {
257  // Too many processes are waiting for pooled processes to finish
258  return Status::newGood( PoolCounter::QUEUE_FULL );
259  } elseif ( $slot === 'QUEUE_WAIT' ) {
260  // This process is now registered as waiting
261  $keys = ( $doWakeup == self::AWAKE_ALL )
262  // Wait for an open slot or wake-up signal (preferring the latter)
263  ? [ $this->getWakeupListKey(), $this->getSlotListKey() ]
264  // Just wait for an actual pool slot
265  : [ $this->getSlotListKey() ];
266 
267  $res = $conn->blPop( $keys, $timeout );
268  if ( $res === [] ) {
269  $conn->zRem( $this->getWaitSetKey(), $this->session ); // no longer waiting
270  return Status::newGood( PoolCounter::TIMEOUT );
271  }
272 
273  $slot = $res[1]; // pool slot or "w" for wake-up notifications
274  $slotTime = microtime( true ); // last microtime() was a few RTTs ago
275  // Unregister this process as waiting and bump slot "next release" time
277  } else {
278  return Status::newFatal( 'pool-error-unknown', "Server gave slot '$slot'." );
279  }
280  } catch ( RedisException $e ) {
281  return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
282  }
283 
284  if ( $slot !== 'w' ) {
285  $this->slot = $slot;
286  $this->slotTime = $slotTime;
287  $this->onRelease = $doWakeup;
288  self::$active[$this->session] = $this;
289  }
290 
291  $this->onAcquire();
292 
293  return Status::newGood( $slot === 'w' ? PoolCounter::DONE : PoolCounter::LOCKED );
294  }
295 
301  protected function initAndPopPoolSlotList( RedisConnRef $conn, $now ) {
302  static $script =
304 <<<LUA
305  local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
306  local rMaxWorkers,rMaxQueue,rTimeout,rExpiry,rSess,rTime = unpack(ARGV)
307  -- Initialize if the "next release" time sorted-set is empty. The slot key
308  -- itself is empty if all slots are busy or when nothing is initialized.
309  -- If the list is empty but the set is not, then it is the latter case.
310  -- If the list exists but not the set, then reset everything.
311  if redis.call('exists',kSlotsNextRelease) == 0 then
312  redis.call('del',kSlots)
313  for i = 1,1*rMaxWorkers do
314  redis.call('rPush',kSlots,i)
315  redis.call('zAdd',kSlotsNextRelease,-1,i)
316  end
317  -- Otherwise do maintenance to clean up after network partitions
318  else
319  -- Find stale slot locks and add free them (avoid duplicates)
320  local staleLocks = redis.call('zRangeByScore',kSlotsNextRelease,0,rTime)
321  for k,slot in ipairs(staleLocks) do
322  redis.call('lRem',kSlots,0,slot)
323  redis.call('rPush',kSlots,slot)
324  redis.call('zAdd',kSlotsNextRelease,rTime + 30,slot)
325  end
326  -- Find stale wait slot entries and remove them
327  redis.call('zRemRangeByScore',kSlotWaits,0,rTime - 2*rTimeout)
328  end
329  local slot
330  -- Try to acquire a slot if possible now
331  if redis.call('lLen',kSlots) > 0 then
332  slot = redis.call('lPop',kSlots)
333  -- Update the slot "next release" time
334  redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,slot)
335  elseif redis.call('zCard',kSlotWaits) >= 1*rMaxQueue then
336  slot = 'QUEUE_FULL'
337  else
338  slot = 'QUEUE_WAIT'
339  -- Register this process as waiting
340  redis.call('zAdd',kSlotWaits,rTime,rSess)
341  redis.call('expireAt',kSlotWaits,math.ceil(rTime + 2*rTimeout))
342  end
343  -- Always keep renewing the expiry on use
344  redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
345  redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
346  return slot
347 LUA;
348  return $conn->luaEval( $script,
349  [
350  $this->getSlotListKey(),
351  $this->getSlotRTimeSetKey(),
352  $this->getWaitSetKey(),
353  $this->workers,
354  $this->maxqueue,
355  $this->timeout,
356  $this->lockTTL,
357  $this->session,
358  $now
359  ],
360  3 # number of first argument(s) that are keys
361  );
362  }
363 
370  protected function registerAcquisitionTime( RedisConnRef $conn, $slot, $now ) {
371  static $script =
373 <<<LUA
374  local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
375  local rSlot,rExpiry,rSess,rTime = unpack(ARGV)
376  -- If rSlot is 'w' then the client was told to wake up but got no slot
377  if rSlot ~= 'w' then
378  -- Update the slot "next release" time
379  redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,rSlot)
380  -- Always keep renewing the expiry on use
381  redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
382  redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
383  end
384  -- Unregister this process as waiting
385  redis.call('zRem',kSlotWaits,rSess)
386  return 1
387 LUA;
388  return $conn->luaEval( $script,
389  [
390  $this->getSlotListKey(),
391  $this->getSlotRTimeSetKey(),
392  $this->getWaitSetKey(),
393  $slot,
394  $this->lockTTL,
395  $this->session,
396  $now
397  ],
398  3 # number of first argument(s) that are keys
399  );
400  }
401 
405  protected function getSlotListKey() {
406  return "poolcounter:l-slots-{$this->keySha1}-{$this->workers}";
407  }
408 
412  protected function getSlotRTimeSetKey() {
413  return "poolcounter:z-renewtime-{$this->keySha1}-{$this->workers}";
414  }
415 
419  protected function getWaitSetKey() {
420  return "poolcounter:z-wait-{$this->keySha1}-{$this->workers}";
421  }
422 
426  protected function getWakeupListKey() {
427  return "poolcounter:l-wakeup-{$this->keySha1}-{$this->workers}";
428  }
429 
433  public static function releaseAll() {
434  $e = null;
435  foreach ( self::$active as $poolCounter ) {
436  try {
437  if ( $poolCounter->slot !== null ) {
438  $poolCounter->release();
439  }
440  } catch ( Exception $e ) {
441  }
442  }
443  if ( $e ) {
444  throw $e;
445  }
446  }
447 }
static consistentHashSort(&$array, $key, $separator="\000")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
Convenience class for weighted consistent hash rings.
Definition: HashRing.php:44
static getInstance( $channel)
Get a named logger instance from the currently configured logger factory.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: Status.php:58
Version of PoolCounter that uses Redis.
acquireForAnyone( $timeout=null)
I want to do this task, but if anyone else does it instead, it's also fine for me.
__construct( $conf, $type, $key)
int null $onRelease
AWAKE_* constant.
float null $slotTime
UNIX timestamp.
LoggerInterface $logger
initAndPopPoolSlotList(RedisConnRef $conn, $now)
array $serversByLabel
(server label => host) map
string $session
Unique string to identify this process.
RedisConnRef $conn
static releaseAll()
Try to make sure that locks get released (even with exceptions and fatals)
release()
I have successfully finished my task.
static PoolCounterRedis[] $active
List of active PoolCounterRedis objects in this script.
waitForSlotOrNotif( $doWakeup, $timeout=null)
registerAcquisitionTime(RedisConnRef $conn, $slot, $now)
acquireForMe( $timeout=null)
I want to do this task and I need to do it myself.
int $lockTTL
TTL for locks to expire (work should finish in this time)
string null $slot
Pool slot value.
string $keySha1
SHA-1 of the key.
RedisConnectionPool $pool
Semaphore semantics to restrict how many workers may concurrently perform a task.
Definition: PoolCounter.php:51
const QUEUE_FULL
Definition: PoolCounter.php:59
const LOCKED
Definition: PoolCounter.php:53
onAcquire()
Update any lock tracking information when the lock is acquired.
const TIMEOUT
Definition: PoolCounter.php:60
string $key
All workers with the same key share the lock.
Definition: PoolCounter.php:64
const RELEASED
Definition: PoolCounter.php:54
precheckAcquire()
Checks that the lock request is sensible.
const NOT_LOCKED
Definition: PoolCounter.php:58
int $timeout
Maximum time in seconds to wait for the lock.
Definition: PoolCounter.php:77
const LOCK_HELD
Definition: PoolCounter.php:61
onRelease()
Update any lock tracking information when the lock is released.
Helper class to handle automatically marking connections as reusable (via RAII pattern)
luaEval( $script, array $params, $numKeys)
static singleton(array $options)