MediaWiki  master
NameTableStore.php
Go to the documentation of this file.
1 <?php
21 namespace MediaWiki\Storage;
22 
23 use Exception;
24 use IExpiringStore;
25 use Psr\Log\LoggerInterface;
26 use WANObjectCache;
27 use Wikimedia\Assert\Assert;
31 
37 
39  private $loadBalancer;
40 
42  private $cache;
43 
45  private $logger;
46 
48  private $tableCache = null;
49 
51  private $domain;
52 
54  private $cacheTTL;
55 
57  private $table;
59  private $idField;
61  private $nameField;
65  private $insertCallback;
66 
85  public function __construct(
86  ILoadBalancer $dbLoadBalancer,
88  LoggerInterface $logger,
89  $table,
90  $idField,
91  $nameField,
92  callable $normalizationCallback = null,
93  $dbDomain = false,
94  callable $insertCallback = null
95  ) {
96  $this->loadBalancer = $dbLoadBalancer;
97  $this->cache = $cache;
98  $this->logger = $logger;
99  $this->table = $table;
100  $this->idField = $idField;
101  $this->nameField = $nameField;
102  $this->normalizationCallback = $normalizationCallback;
103  $this->domain = $dbDomain;
104  $this->cacheTTL = IExpiringStore::TTL_MONTH;
105  $this->insertCallback = $insertCallback;
106  }
107 
114  private function getDBConnection( $index, $flags = 0 ) {
115  return $this->loadBalancer->getConnectionRef( $index, [], $this->domain, $flags );
116  }
117 
126  private function getCacheKey() {
127  return $this->cache->makeGlobalKey(
128  'NameTableSqlStore',
129  $this->table,
130  $this->loadBalancer->resolveDomainID( $this->domain )
131  );
132  }
133 
138  private function normalizeName( $name ) {
139  if ( $this->normalizationCallback === null ) {
140  return $name;
141  }
142  return call_user_func( $this->normalizationCallback, $name );
143  }
144 
162  public function acquireId( string $name ) {
163  $name = $this->normalizeName( $name );
164 
166  $searchResult = array_search( $name, $table, true );
167  if ( $searchResult === false ) {
168  $id = $this->store( $name );
169  if ( $id === null ) {
170  // RACE: $name was already in the db, probably just inserted, so load from primary DB.
171  // Use DBO_TRX to avoid missing inserts due to other threads or REPEATABLE-READs.
173 
174  $searchResult = array_search( $name, $table, true );
175  if ( $searchResult === false ) {
176  // Insert failed due to IGNORE flag, but DB_PRIMARY didn't give us the data
177  $m = "No insert possible but primary DB didn't give us a record for " .
178  "'{$name}' in '{$this->table}'";
179  $this->logger->error( $m );
180  throw new NameTableAccessException( $m );
181  }
182  } else {
183  if ( isset( $table[$id] ) ) {
184  // This can happen when a transaction is rolled back and acquireId is called in
185  // an onTransactionResolution() callback, which gets executed before retryStore()
186  // has a chance to run. The right thing to do in this case is to discard the old
187  // value. According to the contract of acquireId, the caller should not have
188  // used it outside the transaction, so it should not be persisted anywhere after
189  // the rollback.
190  $m = "Got ID $id for '$name' from insert"
191  . " into '{$this->table}', but ID $id was previously associated with"
192  . " the name '{$table[$id]}'. Overriding the old value, which presumably"
193  . " has been removed from the database due to a transaction rollback.";
194 
195  $this->logger->warning( $m );
196  }
197 
198  $table[$id] = $name;
199  $searchResult = $id;
200 
201  // As store returned an ID we know we inserted so delete from WAN cache
202  $dbw = $this->getDBConnection( DB_PRIMARY );
203  $dbw->onTransactionPreCommitOrIdle( function () {
204  $this->cache->delete( $this->getCacheKey() );
205  }, __METHOD__ );
206  }
207  $this->tableCache = $table;
208  }
209 
210  return $searchResult;
211  }
212 
225  public function reloadMap( $connFlags = 0 ) {
226  if ( $connFlags !== 0 && defined( 'MW_PHPUNIT_TEST' ) ) {
227  // HACK: We can't use $connFlags while doing PHPUnit tests, because the
228  // fake database tables are bound to a single connection.
229  $connFlags = 0;
230  }
231 
232  $dbw = $this->getDBConnection( DB_PRIMARY, $connFlags );
233  $this->tableCache = $this->loadTable( $dbw );
234  $dbw->onTransactionPreCommitOrIdle( function () {
235  $this->cache->reap( $this->getCacheKey(), INF );
236  }, __METHOD__ );
237 
238  return $this->tableCache;
239  }
240 
251  public function getId( string $name ) {
252  $name = $this->normalizeName( $name );
253 
255  $searchResult = array_search( $name, $table, true );
256 
257  if ( $searchResult !== false ) {
258  return $searchResult;
259  }
260 
261  throw NameTableAccessException::newFromDetails( $this->table, 'name', $name );
262  }
263 
275  public function getName( int $id ) {
277  if ( array_key_exists( $id, $table ) ) {
278  return $table[$id];
279  }
280  $fname = __METHOD__;
281 
282  $table = $this->cache->getWithSetCallback(
283  $this->getCacheKey(),
284  $this->cacheTTL,
285  function ( $oldValue, &$ttl, &$setOpts ) use ( $id, $fname ) {
286  // Check if cached value is up-to-date enough to have $id
287  if ( is_array( $oldValue ) && array_key_exists( $id, $oldValue ) ) {
288  // Completely leave the cache key alone
289  $ttl = WANObjectCache::TTL_UNCACHEABLE;
290  // Use the old value
291  return $oldValue;
292  }
293  // Regenerate from replica DB, and primary DB if needed
294  foreach ( [ DB_REPLICA, DB_PRIMARY ] as $source ) {
295  // Log a fallback to primary
296  if ( $source === DB_PRIMARY ) {
297  $this->logger->info(
298  $fname . ' falling back to primary select from ' .
299  $this->table . ' with id ' . $id
300  );
301  }
302  $db = $this->getDBConnection( $source );
303  $cacheSetOpts = Database::getCacheSetOptions( $db );
304  $table = $this->loadTable( $db );
305  if ( array_key_exists( $id, $table ) ) {
306  break; // found it
307  }
308  }
309  // Use the value from last source checked
310  $setOpts += $cacheSetOpts;
311 
312  return $table;
313  },
314  [ 'minAsOf' => INF ] // force callback run
315  );
316 
317  $this->tableCache = $table;
318 
319  if ( array_key_exists( $id, $table ) ) {
320  return $table[$id];
321  }
322 
323  throw NameTableAccessException::newFromDetails( $this->table, 'id', $id );
324  }
325 
333  public function getMap() {
334  return $this->getTableFromCachesOrReplica();
335  }
336 
340  private function getTableFromCachesOrReplica() {
341  if ( $this->tableCache !== null ) {
342  return $this->tableCache;
343  }
344 
345  $table = $this->cache->getWithSetCallback(
346  $this->getCacheKey(),
347  $this->cacheTTL,
348  function ( $oldValue, &$ttl, &$setOpts ) {
349  $dbr = $this->getDBConnection( DB_REPLICA );
350  $setOpts += Database::getCacheSetOptions( $dbr );
351  return $this->loadTable( $dbr );
352  }
353  );
354 
355  $this->tableCache = $table;
356 
357  return $table;
358  }
359 
367  private function loadTable( IDatabase $db ) {
368  $result = $db->select(
369  $this->table,
370  [
371  'id' => $this->idField,
372  'name' => $this->nameField
373  ],
374  [],
375  __METHOD__,
376  [ 'ORDER BY' => 'id' ]
377  );
378 
379  $assocArray = [];
380  foreach ( $result as $row ) {
381  $assocArray[$row->id] = $row->name;
382  }
383 
384  return $assocArray;
385  }
386 
393  private function store( string $name ) {
394  Assert::parameter( $name !== '', '$name', 'should not be an empty string' );
395  // Note: this is only called internally so normalization of $name has already occurred.
396 
397  $dbw = $this->getDBConnection( DB_PRIMARY );
398 
399  $id = null;
400  $dbw->doAtomicSection(
401  __METHOD__,
402  function ( IDatabase $unused, $fname )
403  use ( $name, &$id, $dbw ) {
404  // NOTE: use IDatabase from the parent scope here, not the function parameter.
405  // If $dbw is a wrapper around the actual DB, we need to call the wrapper here,
406  // not the inner instance.
407  $dbw->insert(
408  $this->table,
409  $this->getFieldsToStore( $name ),
410  $fname,
411  [ 'IGNORE' ]
412  );
413 
414  if ( $dbw->affectedRows() === 0 ) {
415  $this->logger->info(
416  'Tried to insert name into table ' . $this->table . ', but value already existed.'
417  );
418 
419  return;
420  }
421 
422  $id = $dbw->insertId();
423 
424  // Any open transaction may still be rolled back. If that happens, we have to re-try the
425  // insertion and restore a consistent state of the cached table.
426  $dbw->onAtomicSectionCancel(
427  function ( $trigger, IDatabase $unused ) use ( $name, $id, $dbw ) {
428  $this->retryStore( $dbw, $name, $id );
429  },
430  $fname );
431  },
432  IDatabase::ATOMIC_CANCELABLE
433  );
434 
435  return $id;
436  }
437 
446  private function retryStore( IDatabase $dbw, $name, $id ) {
447  // NOTE: in the closure below, use the IDatabase from the original method call,
448  // not the one passed to the closure as a parameter.
449  // If $dbw is a wrapper around the actual DB, we need to call the wrapper,
450  // not the inner instance.
451 
452  try {
453  $dbw->doAtomicSection(
454  __METHOD__,
455  function ( IDatabase $unused, $fname ) use ( $name, $id, $dbw ) {
456  // Try to insert a row with the ID we originally got.
457  // If that fails (because of a key conflict), we will just try to get another ID again later.
458  $dbw->insert(
459  $this->table,
460  $this->getFieldsToStore( $name, $id ),
461  $fname
462  );
463 
464  // Make sure we re-load the map in case this gets rolled back again.
465  // We could re-try once more, but that bears the risk of an infinite loop.
466  // So let's just give up on the ID.
467  $dbw->onAtomicSectionCancel(
468  function ( $trigger, IDatabase $unused ) {
469  $this->logger->warning(
470  'Re-insertion of name into table ' . $this->table
471  . ' was rolled back. Giving up and reloading the cache.'
472  );
474  },
475  $fname
476  );
477 
478  $this->logger->info(
479  'Re-insert name into table ' . $this->table . ' after failed transaction.'
480  );
481  },
482  IDatabase::ATOMIC_CANCELABLE
483  );
484  } catch ( Exception $ex ) {
485  $this->logger->error(
486  'Re-insertion of name into table ' . $this->table . ' failed: ' . $ex->getMessage()
487  );
488  } finally {
489  // NOTE: we reload regardless of whether the above insert succeeded. There is
490  // only three possibilities: the insert succeeded, so the new map will have
491  // the desired $id/$name mapping. Or the insert failed because another
492  // process already inserted that same $id/$name mapping, in which case the
493  // new map will also have it. Or another process grabbed the desired ID for
494  // another name, or the database refuses to insert the given ID into the
495  // auto increment field - in that case, the new map will not have a mapping
496  // for $name (or has a different mapping for $name). In that last case, we can
497  // only hope that the ID produced within the failed transaction has not been
498  // used outside that transaction.
499 
501  }
502  }
503 
509  private function getFieldsToStore( $name, $id = null ) {
510  $fields = [];
511 
512  $fields[$this->nameField] = $name;
513 
514  if ( $id !== null ) {
515  $fields[$this->idField] = $id;
516  }
517 
518  if ( $this->insertCallback !== null ) {
519  $fields = call_user_func( $this->insertCallback, $fields );
520  }
521  return $fields;
522  }
523 
524 }
MediaWiki\Storage\NameTableStore\$logger
LoggerInterface $logger
Definition: NameTableStore.php:45
Wikimedia\Rdbms\Database
Relational database abstraction object.
Definition: Database.php:52
MediaWiki\Storage\NameTableStore\acquireId
acquireId(string $name)
Acquire the id of the given name.
Definition: NameTableStore.php:162
Wikimedia\Rdbms\IDatabase\onAtomicSectionCancel
onAtomicSectionCancel(callable $callback, $fname=__METHOD__)
Run a callback when the atomic section is cancelled.
MediaWiki\Storage\NameTableStore\$normalizationCallback
null callable $normalizationCallback
Definition: NameTableStore.php:63
MediaWiki\Storage\NameTableStore\$insertCallback
null callable $insertCallback
Definition: NameTableStore.php:65
MediaWiki\Storage\NameTableStore\$tableCache
string[] $tableCache
Definition: NameTableStore.php:48
MediaWiki\Storage\NameTableStore\$idField
string $idField
Definition: NameTableStore.php:59
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
$dbr
$dbr
Definition: testCompression.php:54
MediaWiki\Storage\NameTableStore\$cacheTTL
int $cacheTTL
Definition: NameTableStore.php:54
IExpiringStore
Generic interface providing TTL constants for lightweight expiring object stores.
Definition: IExpiringStore.php:13
Wikimedia\Rdbms\Database\getCacheSetOptions
static getCacheSetOptions(?IDatabase ... $dbs)
Merge the result of getSessionLagStatus() for several DBs using the most pessimistic values to estima...
Definition: Database.php:5240
MediaWiki\Storage\NameTableStore\getCacheKey
getCacheKey()
Gets the cache key for names.
Definition: NameTableStore.php:126
MediaWiki\Storage\NameTableStore\getTableFromCachesOrReplica
getTableFromCachesOrReplica()
Definition: NameTableStore.php:340
MediaWiki\Storage\NameTableStore\__construct
__construct(ILoadBalancer $dbLoadBalancer, WANObjectCache $cache, LoggerInterface $logger, $table, $idField, $nameField, callable $normalizationCallback=null, $dbDomain=false, callable $insertCallback=null)
Definition: NameTableStore.php:85
MediaWiki\Storage\NameTableStore\normalizeName
normalizeName( $name)
Definition: NameTableStore.php:138
MediaWiki\Storage\NameTableStore\$domain
bool string $domain
Definition: NameTableStore.php:51
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
MediaWiki\Storage\NameTableStore\retryStore
retryStore(IDatabase $dbw, $name, $id)
After the initial insertion got rolled back, this can be used to try the insertion again,...
Definition: NameTableStore.php:446
MediaWiki\Storage\NameTableStore\getDBConnection
getDBConnection( $index, $flags=0)
Definition: NameTableStore.php:114
MediaWiki\Storage\NameTableStore\$cache
WANObjectCache $cache
Definition: NameTableStore.php:42
MediaWiki\Storage\NameTableStore\$nameField
string $nameField
Definition: NameTableStore.php:61
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
MediaWiki\Storage\NameTableStore\reloadMap
reloadMap( $connFlags=0)
Reloads the name table from the primary database, and purges the WAN cache entry.
Definition: NameTableStore.php:225
MediaWiki\Storage\NameTableStore\loadTable
loadTable(IDatabase $db)
Gets the table from the db.
Definition: NameTableStore.php:367
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:137
MediaWiki\Storage\NameTableAccessException\newFromDetails
static newFromDetails( $tableName, $accessType, $accessValue)
Definition: NameTableAccessException.php:41
MediaWiki\Storage\NameTableStore
Definition: NameTableStore.php:36
MediaWiki\Storage\NameTableStore\getFieldsToStore
getFieldsToStore( $name, $id=null)
Definition: NameTableStore.php:509
MediaWiki\Storage
Definition: BlobAccessException.php:23
Wikimedia\Rdbms\IDatabase\insert
insert( $table, $rows, $fname=__METHOD__, $options=[])
Insert the given row(s) into a table.
Wikimedia\Rdbms\ILoadBalancer\CONN_TRX_AUTOCOMMIT
const CONN_TRX_AUTOCOMMIT
DB handle should have DBO_TRX disabled and the caller will leave it as such.
Definition: ILoadBalancer.php:104
Wikimedia\Rdbms\IDatabase\doAtomicSection
doAtomicSection( $fname, callable $callback, $cancelable=self::ATOMIC_NOT_CANCELABLE)
Perform an atomic section of reversable SQL statements from a callback.
MediaWiki\Storage\NameTableStore\$table
string $table
Definition: NameTableStore.php:57
MediaWiki\Storage\NameTableAccessException
Exception representing a failure to look up a row from a name table.
Definition: NameTableAccessException.php:33
Wikimedia\Rdbms\IDatabase\select
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
$source
$source
Definition: mwdoc-filter.php:34
MediaWiki\Storage\NameTableStore\store
store(string $name)
Stores the given name in the DB, returning the ID when an insert occurs.
Definition: NameTableStore.php:393
MediaWiki\Storage\NameTableStore\$loadBalancer
ILoadBalancer $loadBalancer
Definition: NameTableStore.php:39
MediaWiki\Storage\NameTableStore\getName
getName(int $id)
Get the name of the given id.
Definition: NameTableStore.php:275
MediaWiki\Storage\NameTableStore\getId
getId(string $name)
Get the id of the given name.
Definition: NameTableStore.php:251
MediaWiki\Storage\NameTableStore\getMap
getMap()
Get the whole table, in no particular order as a map of ids to names.
Definition: NameTableStore.php:333
Wikimedia\Rdbms\ILoadBalancer
Database cluster connection, tracking, load balancing, and transaction manager interface.
Definition: ILoadBalancer.php:81