MediaWiki  master
LocalRepo.php
Go to the documentation of this file.
1 <?php
29 
37 class LocalRepo extends FileRepo {
39  protected $fileFactory = [ LocalFile::class, 'newFromTitle' ];
41  protected $fileFactoryKey = [ LocalFile::class, 'newFromKey' ];
43  protected $fileFromRowFactory = [ LocalFile::class, 'newFromRow' ];
45  protected $oldFileFromRowFactory = [ OldLocalFile::class, 'newFromRow' ];
47  protected $oldFileFactory = [ OldLocalFile::class, 'newFromTitle' ];
49  protected $oldFileFactoryKey = [ OldLocalFile::class, 'newFromKey' ];
50 
51  function __construct( array $info = null ) {
52  parent::__construct( $info );
53 
54  $this->hasSha1Storage = isset( $info['storageLayout'] )
55  && $info['storageLayout'] === 'sha1';
56 
57  if ( $this->hasSha1Storage() ) {
58  $this->backend = new FileBackendDBRepoWrapper( [
59  'backend' => $this->backend,
60  'repoName' => $this->name,
61  'dbHandleFactory' => $this->getDBFactory()
62  ] );
63  }
64  }
65 
71  function newFileFromRow( $row ) {
72  if ( isset( $row->img_name ) ) {
73  return call_user_func( $this->fileFromRowFactory, $row, $this );
74  } elseif ( isset( $row->oi_name ) ) {
75  return call_user_func( $this->oldFileFromRowFactory, $row, $this );
76  } else {
77  throw new MWException( __METHOD__ . ': invalid row' );
78  }
79  }
80 
86  function newFromArchiveName( $title, $archiveName ) {
87  return OldLocalFile::newFromArchiveName( $title, $this, $archiveName );
88  }
89 
100  function cleanupDeletedBatch( array $storageKeys ) {
101  if ( $this->hasSha1Storage() ) {
102  wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
103  return Status::newGood();
104  }
105 
106  $backend = $this->backend; // convenience
107  $root = $this->getZonePath( 'deleted' );
108  $dbw = $this->getMasterDB();
109  $status = $this->newGood();
110  $storageKeys = array_unique( $storageKeys );
111  foreach ( $storageKeys as $key ) {
112  $hashPath = $this->getDeletedHashPath( $key );
113  $path = "$root/$hashPath$key";
114  $dbw->startAtomic( __METHOD__ );
115  // Check for usage in deleted/hidden files and preemptively
116  // lock the key to avoid any future use until we are finished.
117  $deleted = $this->deletedFileHasKey( $key, 'lock' );
118  $hidden = $this->hiddenFileHasKey( $key, 'lock' );
119  if ( !$deleted && !$hidden ) { // not in use now
120  wfDebug( __METHOD__ . ": deleting $key\n" );
121  $op = [ 'op' => 'delete', 'src' => $path ];
122  if ( !$backend->doOperation( $op )->isOK() ) {
123  $status->error( 'undelete-cleanup-error', $path );
124  $status->failCount++;
125  }
126  } else {
127  wfDebug( __METHOD__ . ": $key still in use\n" );
128  $status->successCount++;
129  }
130  $dbw->endAtomic( __METHOD__ );
131  }
132 
133  return $status;
134  }
135 
143  protected function deletedFileHasKey( $key, $lock = null ) {
144  $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
145 
146  $dbw = $this->getMasterDB();
147 
148  return (bool)$dbw->selectField( 'filearchive', '1',
149  [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ],
150  __METHOD__, $options
151  );
152  }
153 
161  protected function hiddenFileHasKey( $key, $lock = null ) {
162  $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
163 
164  $sha1 = self::getHashFromKey( $key );
165  $ext = File::normalizeExtension( substr( $key, strcspn( $key, '.' ) + 1 ) );
166 
167  $dbw = $this->getMasterDB();
168 
169  return (bool)$dbw->selectField( 'oldimage', '1',
170  [ 'oi_sha1' => $sha1,
171  'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ),
172  $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE ],
173  __METHOD__, $options
174  );
175  }
176 
183  public static function getHashFromKey( $key ) {
184  $sha1 = strtok( $key, '.' );
185  if ( is_string( $sha1 ) && strlen( $sha1 ) === 32 && $sha1[0] === '0' ) {
186  $sha1 = substr( $sha1, 1 );
187  }
188  return $sha1;
189  }
190 
197  function checkRedirect( Title $title ) {
198  $title = File::normalizeTitle( $title, 'exception' );
199 
200  $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
201  if ( $memcKey === false ) {
202  $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
203  $expiry = 300; // no invalidation, 5 minutes
204  } else {
205  $expiry = 86400; // has invalidation, 1 day
206  }
207 
208  $method = __METHOD__;
209  $redirDbKey = $this->wanCache->getWithSetCallback(
210  $memcKey,
211  $expiry,
212  function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
213  $dbr = $this->getReplicaDB(); // possibly remote DB
214 
215  $setOpts += Database::getCacheSetOptions( $dbr );
216 
217  $row = $dbr->selectRow(
218  [ 'page', 'redirect' ],
219  [ 'rd_namespace', 'rd_title' ],
220  [
221  'page_namespace' => $title->getNamespace(),
222  'page_title' => $title->getDBkey(),
223  'rd_from = page_id'
224  ],
225  $method
226  );
227 
228  return ( $row && $row->rd_namespace == NS_FILE )
229  ? Title::makeTitle( $row->rd_namespace, $row->rd_title )->getDBkey()
230  : ''; // negative cache
231  },
232  [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
233  );
234 
235  // @note: also checks " " for b/c
236  if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
237  // Page is a redirect to another file
238  return Title::newFromText( $redirDbKey, NS_FILE );
239  }
240 
241  return false; // no redirect
242  }
243 
244  public function findFiles( array $items, $flags = 0 ) {
245  $finalFiles = []; // map of (DB key => corresponding File) for matches
246 
247  $searchSet = []; // map of (normalized DB key => search params)
248  foreach ( $items as $item ) {
249  if ( is_array( $item ) ) {
250  $title = File::normalizeTitle( $item['title'] );
251  if ( $title ) {
252  $searchSet[$title->getDBkey()] = $item;
253  }
254  } else {
255  $title = File::normalizeTitle( $item );
256  if ( $title ) {
257  $searchSet[$title->getDBkey()] = [];
258  }
259  }
260  }
261 
262  $fileMatchesSearch = function ( File $file, array $search ) {
263  // Note: file name comparison done elsewhere (to handle redirects)
264  $user = ( !empty( $search['private'] ) && $search['private'] instanceof User )
265  ? $search['private']
266  : null;
267 
268  return (
269  $file->exists() &&
270  (
271  ( empty( $search['time'] ) && !$file->isOld() ) ||
272  ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
273  ) &&
274  ( !empty( $search['private'] ) || !$file->isDeleted( File::DELETED_FILE ) ) &&
275  $file->userCan( File::DELETED_FILE, $user )
276  );
277  };
278 
279  $applyMatchingFiles = function ( IResultWrapper $res, &$searchSet, &$finalFiles )
280  use ( $fileMatchesSearch, $flags )
281  {
282  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
283  $info = $this->getInfo();
284  foreach ( $res as $row ) {
285  $file = $this->newFileFromRow( $row );
286  // There must have been a search for this DB key, but this has to handle the
287  // cases were title capitalization is different on the client and repo wikis.
288  $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
289  if ( !empty( $info['initialCapital'] ) ) {
290  // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
291  $dbKeysLook[] = $contLang->lcfirst( $file->getName() );
292  }
293  foreach ( $dbKeysLook as $dbKey ) {
294  if ( isset( $searchSet[$dbKey] )
295  && $fileMatchesSearch( $file, $searchSet[$dbKey] )
296  ) {
297  $finalFiles[$dbKey] = ( $flags & FileRepo::NAME_AND_TIME_ONLY )
298  ? [ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
299  : $file;
300  unset( $searchSet[$dbKey] );
301  }
302  }
303  }
304  };
305 
306  $dbr = $this->getReplicaDB();
307 
308  // Query image table
309  $imgNames = [];
310  foreach ( array_keys( $searchSet ) as $dbKey ) {
311  $imgNames[] = $this->getNameFromTitle( File::normalizeTitle( $dbKey ) );
312  }
313 
314  if ( count( $imgNames ) ) {
315  $fileQuery = LocalFile::getQueryInfo();
316  $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], [ 'img_name' => $imgNames ],
317  __METHOD__, [], $fileQuery['joins'] );
318  $applyMatchingFiles( $res, $searchSet, $finalFiles );
319  }
320 
321  // Query old image table
322  $oiConds = []; // WHERE clause array for each file
323  foreach ( $searchSet as $dbKey => $search ) {
324  if ( isset( $search['time'] ) ) {
325  $oiConds[] = $dbr->makeList(
326  [
327  'oi_name' => $this->getNameFromTitle( File::normalizeTitle( $dbKey ) ),
328  'oi_timestamp' => $dbr->timestamp( $search['time'] )
329  ],
330  LIST_AND
331  );
332  }
333  }
334 
335  if ( count( $oiConds ) ) {
336  $fileQuery = OldLocalFile::getQueryInfo();
337  $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'],
338  $dbr->makeList( $oiConds, LIST_OR ),
339  __METHOD__, [], $fileQuery['joins'] );
340  $applyMatchingFiles( $res, $searchSet, $finalFiles );
341  }
342 
343  // Check for redirects...
344  foreach ( $searchSet as $dbKey => $search ) {
345  if ( !empty( $search['ignoreRedirect'] ) ) {
346  continue;
347  }
348 
349  $title = File::normalizeTitle( $dbKey );
350  $redir = $this->checkRedirect( $title ); // hopefully hits memcached
351 
352  if ( $redir && $redir->getNamespace() == NS_FILE ) {
353  $file = $this->newFile( $redir );
354  if ( $file && $fileMatchesSearch( $file, $search ) ) {
355  $file->redirectedFrom( $title->getDBkey() );
356  if ( $flags & FileRepo::NAME_AND_TIME_ONLY ) {
357  $finalFiles[$dbKey] = [
358  'title' => $file->getTitle()->getDBkey(),
359  'timestamp' => $file->getTimestamp()
360  ];
361  } else {
362  $finalFiles[$dbKey] = $file;
363  }
364  }
365  }
366  }
367 
368  return $finalFiles;
369  }
370 
378  function findBySha1( $hash ) {
379  $dbr = $this->getReplicaDB();
380  $fileQuery = LocalFile::getQueryInfo();
381  $res = $dbr->select(
382  $fileQuery['tables'],
383  $fileQuery['fields'],
384  [ 'img_sha1' => $hash ],
385  __METHOD__,
386  [ 'ORDER BY' => 'img_name' ],
387  $fileQuery['joins']
388  );
389 
390  $result = [];
391  foreach ( $res as $row ) {
392  $result[] = $this->newFileFromRow( $row );
393  }
394  $res->free();
395 
396  return $result;
397  }
398 
408  function findBySha1s( array $hashes ) {
409  if ( $hashes === [] ) {
410  return []; // empty parameter
411  }
412 
413  $dbr = $this->getReplicaDB();
414  $fileQuery = LocalFile::getQueryInfo();
415  $res = $dbr->select(
416  $fileQuery['tables'],
417  $fileQuery['fields'],
418  [ 'img_sha1' => $hashes ],
419  __METHOD__,
420  [ 'ORDER BY' => 'img_name' ],
421  $fileQuery['joins']
422  );
423 
424  $result = [];
425  foreach ( $res as $row ) {
426  $file = $this->newFileFromRow( $row );
427  $result[$file->getSha1()][] = $file;
428  }
429  $res->free();
430 
431  return $result;
432  }
433 
441  public function findFilesByPrefix( $prefix, $limit ) {
442  $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ];
443 
444  // Query database
445  $dbr = $this->getReplicaDB();
446  $fileQuery = LocalFile::getQueryInfo();
447  $res = $dbr->select(
448  $fileQuery['tables'],
449  $fileQuery['fields'],
450  'img_name ' . $dbr->buildLike( $prefix, $dbr->anyString() ),
451  __METHOD__,
452  $selectOptions,
453  $fileQuery['joins']
454  );
455 
456  // Build file objects
457  $files = [];
458  foreach ( $res as $row ) {
459  $files[] = $this->newFileFromRow( $row );
460  }
461 
462  return $files;
463  }
464 
469  function getReplicaDB() {
470  return wfGetDB( DB_REPLICA );
471  }
472 
477  function getMasterDB() {
478  return wfGetDB( DB_MASTER );
479  }
480 
485  protected function getDBFactory() {
486  return function ( $index ) {
487  return wfGetDB( $index );
488  };
489  }
490 
499  function getSharedCacheKey( ...$args ) {
500  return $this->wanCache->makeKey( ...$args );
501  }
502 
510  $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
511  if ( $key ) {
512  $this->getMasterDB()->onTransactionPreCommitOrIdle(
513  function () use ( $key ) {
514  $this->wanCache->delete( $key );
515  },
516  __METHOD__
517  );
518  }
519  }
520 
527  function getInfo() {
528  global $wgFavicon;
529 
530  return array_merge( parent::getInfo(), [
531  'favicon' => wfExpandUrl( $wgFavicon ),
532  ] );
533  }
534 
535  public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
536  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
537  }
538 
539  public function storeBatch( array $triplets, $flags = 0 ) {
540  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
541  }
542 
543  public function cleanupBatch( array $files, $flags = 0 ) {
544  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
545  }
546 
547  public function publish(
548  $src,
549  $dstRel,
550  $archiveRel,
551  $flags = 0,
552  array $options = []
553  ) {
554  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
555  }
556 
557  public function publishBatch( array $ntuples, $flags = 0 ) {
558  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
559  }
560 
561  public function delete( $srcRel, $archiveRel ) {
562  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
563  }
564 
565  public function deleteBatch( array $sourceDestPairs ) {
566  return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
567  }
568 
576  protected function skipWriteOperationIfSha1( $function, array $args ) {
577  $this->assertWritableRepo(); // fail out if read-only
578 
579  if ( $this->hasSha1Storage() ) {
580  wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
581  return Status::newGood();
582  } else {
583  return parent::$function( ...$args );
584  }
585  }
586 }
assertWritableRepo()
Throw an exception if this repo is read-only by design.
Definition: FileRepo.php:1929
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
getLocalCacheKey(... $args)
Get a key for this repo in the local cache domain.
Definition: FileRepo.php:1869
newFromArchiveName( $title, $archiveName)
Definition: LocalRepo.php:86
static newFromArchiveName( $title, $repo, $archiveName)
wfExpandUrl( $url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
store( $srcPath, $dstZone, $dstRel, $flags=0)
Definition: LocalRepo.php:535
findBySha1s(array $hashes)
Get an array of arrays or iterators of file objects for files that have the given SHA-1 content hashe...
Definition: LocalRepo.php:408
getReplicaDB()
Get a connection to the replica DB.
Definition: LocalRepo.php:469
A repository that stores files in the local filesystem and registers them in the wiki&#39;s own database...
Definition: LocalRepo.php:37
getName()
Return the name of this file.
Definition: File.php:307
doOperation(array $op, array $opts=[])
Same as doOperations() except it takes a single operation.
publishBatch(array $ntuples, $flags=0)
Definition: LocalRepo.php:557
cleanupBatch(array $files, $flags=0)
Definition: LocalRepo.php:543
isOld()
Returns true if the image is an old version STUB.
Definition: File.php:1889
getTitle()
Return the associated title object.
Definition: File.php:336
const DB_MASTER
Definition: defines.php:26
callable $oldFileFromRowFactory
Definition: LocalRepo.php:45
deleteBatch(array $sourceDestPairs)
Definition: LocalRepo.php:565
const DELETED_FILE
Definition: File.php:63
The User object encapsulates all of the user-specific settings (user_id, name, rights, email address, options, last login time).
Definition: User.php:51
if( $line===false) $args
Definition: mcc.php:124
invalidateImageRedirect(Title $title)
Invalidates image redirect cache related to that image.
Definition: LocalRepo.php:509
callable $fileFactoryKey
Definition: LocalRepo.php:41
const LIST_AND
Definition: Defines.php:39
getMasterDB()
Get a connection to the master DB.
Definition: LocalRepo.php:477
getDBkey()
Get the main part with underscores.
Definition: Title.php:1014
cleanupDeletedBatch(array $storageKeys)
Delete files in the deleted directory if they are not referenced in the filearchive table...
Definition: LocalRepo.php:100
userCan( $field, User $user=null)
Determine if the current user is allowed to view a particular field of this file, if it&#39;s marked as d...
Definition: File.php:2172
static newGood( $value=null)
Factory function for good results.
Definition: StatusValue.php:81
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Result wrapper for grabbing data queried from an IDatabase object.
getDBFactory()
Get a callback to get a DB handle given an index (DB_REPLICA/DB_MASTER)
Definition: LocalRepo.php:485
newFile( $title, $time=false)
Create a new File object from the local repository.
Definition: FileRepo.php:414
getInfo()
Return information about the repository.
Definition: LocalRepo.php:527
skipWriteOperationIfSha1( $function, array $args)
Skips the write operation if storage is sha1-based, executes it normally otherwise.
Definition: LocalRepo.php:576
getNamespace()
Get the namespace index, i.e.
Definition: Title.php:1035
const NS_FILE
Definition: Defines.php:66
getZonePath( $zone)
Get the storage path corresponding to one of the zones.
Definition: FileRepo.php:390
__construct(array $info=null)
Definition: LocalRepo.php:51
callable $fileFactory
Definition: LocalRepo.php:39
hiddenFileHasKey( $key, $lock=null)
Check if a hidden (revision delete) file has this sha1 key.
Definition: LocalRepo.php:161
static getHashFromKey( $key)
Gets the SHA1 hash from a storage key.
Definition: LocalRepo.php:183
getNameFromTitle(Title $title)
Get the name of a file from its title object.
Definition: FileRepo.php:674
const LIST_OR
Definition: Defines.php:42
findFilesByPrefix( $prefix, $limit)
Return an array of files where the name starts with $prefix.
Definition: LocalRepo.php:441
const NAME_AND_TIME_ONLY
Definition: FileRepo.php:47
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new oldlocalfile object...
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:584
redirectedFrom( $from)
Definition: File.php:2261
deletedFileHasKey( $key, $lock=null)
Check if a deleted (filearchive) file has this sha1 key.
Definition: LocalRepo.php:143
getTimestamp()
Get the 14-character timestamp of the file upload.
Definition: File.php:2120
callable $fileFromRowFactory
Definition: LocalRepo.php:43
callable $oldFileFactory
Definition: LocalRepo.php:47
storeBatch(array $triplets, $flags=0)
Definition: LocalRepo.php:539
exists()
Returns true if file exists in the repository.
Definition: File.php:901
if(!is_readable( $file)) $ext
Definition: router.php:48
Base class for file repositories.
Definition: FileRepo.php:41
getSharedCacheKey(... $args)
Get a key on the primary cache for this repository.
Definition: LocalRepo.php:499
findFiles(array $items, $flags=0)
Definition: LocalRepo.php:244
FileBackend $backend
Definition: FileRepo.php:64
const DB_REPLICA
Definition: defines.php:25
getDeletedHashPath( $key)
Get a relative path for a deletion archive key, e.g.
Definition: FileRepo.php:1531
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object...
Definition: LocalFile.php:216
publish( $src, $dstRel, $archiveRel, $flags=0, array $options=[])
Definition: LocalRepo.php:547
static normalizeExtension( $extension)
Normalize a file extension to the common form, making it lowercase and checking some synonyms...
Definition: File.php:234
Implements some public methods and some protected utility functions which are required by multiple ch...
Definition: File.php:61
findBySha1( $hash)
Get an array or iterator of file objects for files that have a given SHA-1 content hash...
Definition: LocalRepo.php:378
callable $oldFileFactoryKey
Definition: LocalRepo.php:49
hasSha1Storage()
Returns whether or not storage is SHA-1 based.
Definition: FileRepo.php:1963
isDeleted( $field)
Is this file a "deleted" file in a private archive? STUB.
Definition: File.php:1900
static normalizeTitle( $title, $exception=false)
Given a string or Title object return either a valid Title object with namespace NS_FILE or null...
Definition: File.php:194
checkRedirect(Title $title)
Checks if there is a redirect named as $title.
Definition: LocalRepo.php:197
Proxy backend that manages file layout rewriting for FileRepo.
newGood( $value=null)
Create a new good result.
Definition: FileRepo.php:1778
newFileFromRow( $row)
Definition: LocalRepo.php:71
$wgFavicon
The URL path of the shortcut icon.
static newFromText( $text, $defaultNamespace=NS_MAIN)
Create a new Title from text, such as what one would find in a link.
Definition: Title.php:317