MediaWiki  master
ForeignAPIRepo.php
Go to the documentation of this file.
1 <?php
26 
42 class ForeignAPIRepo extends FileRepo {
43  /* This version string is used in the user agent for requests and will help
44  * server maintainers in identify ForeignAPI usage.
45  * Update the version every time you make breaking or significant changes. */
46  private const VERSION = "2.1";
47 
51  private const IMAGE_INFO_PROPS = [
52  'url',
53  'timestamp',
54  ];
55 
56  protected $fileFactory = [ ForeignAPIFile::class, 'newFromTitle' ];
58  protected $apiThumbCacheExpiry = 86400; // 1 day (24*3600)
59 
61  protected $fileCacheExpiry = 2592000; // 1 month (30*24*3600)
62 
64  protected $mFileExists = [];
65 
67  private $mApiBase;
68 
72  public function __construct( $info ) {
73  global $wgLocalFileRepo;
74  parent::__construct( $info );
75 
76  // https://commons.wikimedia.org/w/api.php
77  $this->mApiBase = $info['apibase'] ?? null;
78 
79  if ( isset( $info['apiThumbCacheExpiry'] ) ) {
80  $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry'];
81  }
82  if ( isset( $info['fileCacheExpiry'] ) ) {
83  $this->fileCacheExpiry = $info['fileCacheExpiry'];
84  }
85  if ( !$this->scriptDirUrl ) {
86  // hack for description fetches
87  $this->scriptDirUrl = dirname( $this->mApiBase );
88  }
89  // If we can cache thumbs we can guess sensible defaults for these
90  if ( $this->canCacheThumbs() && !$this->url ) {
91  $this->url = $wgLocalFileRepo['url'];
92  }
93  if ( $this->canCacheThumbs() && !$this->thumbUrl ) {
94  $this->thumbUrl = $this->url . '/thumb';
95  }
96  }
97 
101  private function getApiUrl() {
102  return $this->mApiBase;
103  }
104 
113  public function newFile( $title, $time = false ) {
114  if ( $time ) {
115  return false;
116  }
117 
118  return parent::newFile( $title, $time );
119  }
120 
125  public function fileExistsBatch( array $files ) {
126  $results = [];
127  foreach ( $files as $k => $f ) {
128  if ( isset( $this->mFileExists[$f] ) ) {
129  $results[$k] = $this->mFileExists[$f];
130  unset( $files[$k] );
131  } elseif ( self::isVirtualUrl( $f ) ) {
132  # @todo FIXME: We need to be able to handle virtual
133  # URLs better, at least when we know they refer to the
134  # same repo.
135  $results[$k] = false;
136  unset( $files[$k] );
137  } elseif ( FileBackend::isStoragePath( $f ) ) {
138  $results[$k] = false;
139  unset( $files[$k] );
140  wfWarn( "Got mwstore:// path '$f'." );
141  }
142  }
143 
144  $data = $this->fetchImageQuery( [
145  'titles' => implode( '|', $files ),
146  'prop' => 'imageinfo' ]
147  );
148 
149  if ( isset( $data['query']['pages'] ) ) {
150  # First, get results from the query. Note we only care whether the image exists,
151  # not whether it has a description page.
152  foreach ( $data['query']['pages'] as $p ) {
153  $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' );
154  }
155  # Second, copy the results to any redirects that were queried
156  if ( isset( $data['query']['redirects'] ) ) {
157  foreach ( $data['query']['redirects'] as $r ) {
158  $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']];
159  }
160  }
161  # Third, copy the results to any non-normalized titles that were queried
162  if ( isset( $data['query']['normalized'] ) ) {
163  foreach ( $data['query']['normalized'] as $n ) {
164  $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']];
165  }
166  }
167  # Finally, copy the results to the output
168  foreach ( $files as $key => $file ) {
169  $results[$key] = $this->mFileExists[$file];
170  }
171  }
172 
173  return $results;
174  }
175 
180  public function getFileProps( $virtualUrl ) {
181  return [];
182  }
183 
188  public function fetchImageQuery( $query ) {
189  global $wgLanguageCode;
190 
191  $query = array_merge( $query,
192  [
193  'format' => 'json',
194  'action' => 'query',
195  'redirects' => 'true'
196  ] );
197 
198  if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
199  $query['uselang'] = $wgLanguageCode;
200  }
201 
202  $data = $this->httpGetCached( 'Metadata', $query );
203 
204  if ( $data ) {
205  return FormatJson::decode( $data, true );
206  } else {
207  return null;
208  }
209  }
210 
215  public function getImageInfo( $data ) {
216  if ( $data && isset( $data['query']['pages'] ) ) {
217  foreach ( $data['query']['pages'] as $info ) {
218  if ( isset( $info['imageinfo'][0] ) ) {
219  $return = $info['imageinfo'][0];
220  if ( isset( $info['pageid'] ) ) {
221  $return['pageid'] = $info['pageid'];
222  }
223  return $return;
224  }
225  }
226  }
227 
228  return false;
229  }
230 
235  public function findBySha1( $hash ) {
236  $results = $this->fetchImageQuery( [
237  'aisha1base36' => $hash,
238  'aiprop' => ForeignAPIFile::getProps(),
239  'list' => 'allimages',
240  ] );
241  $ret = [];
242  if ( isset( $results['query']['allimages'] ) ) {
243  foreach ( $results['query']['allimages'] as $img ) {
244  // 1.14 was broken, doesn't return name attribute
245  if ( !isset( $img['name'] ) ) {
246  continue;
247  }
248  $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img );
249  }
250  }
251 
252  return $ret;
253  }
254 
264  private function getThumbUrl(
265  $name, $width = -1, $height = -1, &$result = null, $otherParams = ''
266  ) {
267  $data = $this->fetchImageQuery( [
268  'titles' => 'File:' . $name,
269  'iiprop' => self::getIIProps(),
270  'iiurlwidth' => $width,
271  'iiurlheight' => $height,
272  'iiurlparam' => $otherParams,
273  'prop' => 'imageinfo' ] );
274  $info = $this->getImageInfo( $data );
275 
276  if ( $data && $info && isset( $info['thumburl'] ) ) {
277  wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] );
278  $result = $info;
279 
280  return $info['thumburl'];
281  } else {
282  return false;
283  }
284  }
285 
295  public function getThumbError(
296  $name, $width = -1, $height = -1, $otherParams = '', $lang = null
297  ) {
298  $data = $this->fetchImageQuery( [
299  'titles' => 'File:' . $name,
300  'iiprop' => self::getIIProps(),
301  'iiurlwidth' => $width,
302  'iiurlheight' => $height,
303  'iiurlparam' => $otherParams,
304  'prop' => 'imageinfo',
305  'uselang' => $lang,
306  ] );
307  $info = $this->getImageInfo( $data );
308 
309  if ( $data && $info && isset( $info['thumberror'] ) ) {
310  wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] );
311 
312  return new MediaTransformError(
313  'thumbnail_error_remote',
314  $width,
315  $height,
316  $this->getDisplayName(),
317  $info['thumberror'] // already parsed message from foreign repo
318  );
319  } else {
320  return false;
321  }
322  }
323 
337  public function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
338  // We can't check the local cache using FileRepo functions because
339  // we override fileExistsBatch(). We have to use the FileBackend directly.
340  $backend = $this->getBackend(); // convenience
341 
342  if ( !$this->canCacheThumbs() ) {
343  $result = null; // can't pass "null" by reference, but it's ok as default value
344 
345  return $this->getThumbUrl( $name, $width, $height, $result, $params );
346  }
347 
348  $key = $this->getLocalCacheKey( 'file-thumb-url', sha1( $name ) );
349  $sizekey = "$width:$height:$params";
350 
351  /* Get the array of urls that we already know */
352  $knownThumbUrls = $this->wanCache->get( $key );
353  if ( !$knownThumbUrls ) {
354  /* No knownThumbUrls for this file */
355  $knownThumbUrls = [];
356  } elseif ( isset( $knownThumbUrls[$sizekey] ) ) {
357  wfDebug( __METHOD__ . ': Got thumburl from local cache: ' .
358  "{$knownThumbUrls[$sizekey]}" );
359 
360  return $knownThumbUrls[$sizekey];
361  }
362 
363  $metadata = null;
364  $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
365 
366  if ( !$foreignUrl ) {
367  wfDebug( __METHOD__ . " Could not find thumburl" );
368 
369  return false;
370  }
371 
372  // We need the same filename as the remote one :)
373  $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) );
374  if ( !$this->validateFilename( $fileName ) ) {
375  wfDebug( __METHOD__ . " The deduced filename $fileName is not safe" );
376 
377  return false;
378  }
379  $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
380  $localFilename = $localPath . "/" . $fileName;
381  $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) .
382  rawurlencode( $name ) . "/" . rawurlencode( $fileName );
383 
384  if ( $backend->fileExists( [ 'src' => $localFilename ] )
385  && isset( $metadata['timestamp'] )
386  ) {
387  wfDebug( __METHOD__ . " Thumbnail was already downloaded before" );
388  $modified = $backend->getFileTimestamp( [ 'src' => $localFilename ] );
389  $remoteModified = strtotime( $metadata['timestamp'] );
390  $current = time();
391  $diff = abs( $modified - $current );
392  if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) {
393  /* Use our current and already downloaded thumbnail */
394  $knownThumbUrls[$sizekey] = $localUrl;
395  $this->wanCache->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
396 
397  return $localUrl;
398  }
399  /* There is a new Commons file, or existing thumbnail older than a month */
400  }
401 
402  $thumb = self::httpGet( $foreignUrl, 'default', [], $mtime );
403  if ( !$thumb ) {
404  wfDebug( __METHOD__ . " Could not download thumb" );
405 
406  return false;
407  }
408 
409  # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
410  $backend->prepare( [ 'dir' => dirname( $localFilename ) ] );
411  $params = [ 'dst' => $localFilename, 'content' => $thumb ];
412  if ( !$backend->quickCreate( $params )->isOK() ) {
413  wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'" );
414 
415  return $foreignUrl;
416  }
417  $knownThumbUrls[$sizekey] = $localUrl;
418 
419  $ttl = $mtime
420  ? $this->wanCache->adaptiveTTL( $mtime, $this->apiThumbCacheExpiry )
422  $this->wanCache->set( $key, $knownThumbUrls, $ttl );
423  wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache" );
424 
425  return $localUrl;
426  }
427 
434  public function getZoneUrl( $zone, $ext = null ) {
435  switch ( $zone ) {
436  case 'public':
437  return $this->url;
438  case 'thumb':
439  return $this->thumbUrl;
440  default:
441  return parent::getZoneUrl( $zone, $ext );
442  }
443  }
444 
450  public function getZonePath( $zone ) {
451  $supported = [ 'public', 'thumb' ];
452  if ( in_array( $zone, $supported ) ) {
453  return parent::getZonePath( $zone );
454  }
455 
456  return false;
457  }
458 
463  public function canCacheThumbs() {
464  return ( $this->apiThumbCacheExpiry > 0 );
465  }
466 
471  public static function getUserAgent() {
472  return MediaWikiServices::getInstance()->getHttpRequestFactory()->getUserAgent() .
473  " ForeignAPIRepo/" . self::VERSION;
474  }
475 
482  public function getInfo() {
483  $info = parent::getInfo();
484  $info['apiurl'] = $this->getApiUrl();
485 
486  $query = [
487  'format' => 'json',
488  'action' => 'query',
489  'meta' => 'siteinfo',
490  'siprop' => 'general',
491  ];
492 
493  $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
494 
495  if ( $data ) {
496  $siteInfo = FormatJson::decode( $data, true );
497  $general = $siteInfo['query']['general'];
498 
499  $info['articlepath'] = $general['articlepath'];
500  $info['server'] = $general['server'];
501 
502  if ( isset( $general['favicon'] ) ) {
503  $info['favicon'] = $general['favicon'];
504  }
505  }
506 
507  return $info;
508  }
509 
517  public static function httpGet(
518  $url, $timeout = 'default', $options = [], &$mtime = false
519  ) {
520  $options['timeout'] = $timeout;
521  /* Http::get */
523  wfDebug( "ForeignAPIRepo: HTTP GET: $url" );
524  $options['method'] = "GET";
525 
526  if ( !isset( $options['timeout'] ) ) {
527  $options['timeout'] = 'default';
528  }
529 
530  $options['userAgent'] = self::getUserAgent();
531 
532  $req = MediaWikiServices::getInstance()->getHttpRequestFactory()
533  ->create( $url, $options, __METHOD__ );
534  $status = $req->execute();
535 
536  if ( $status->isOK() ) {
537  $lmod = $req->getResponseHeader( 'Last-Modified' );
538  $mtime = $lmod ? wfTimestamp( TS_UNIX, $lmod ) : false;
539 
540  return $req->getContent();
541  } else {
542  $logger = LoggerFactory::getInstance( 'http' );
543  $logger->warning(
544  $status->getWikiText( false, false, 'en' ),
545  [ 'caller' => 'ForeignAPIRepo::httpGet' ]
546  );
547 
548  return false;
549  }
550  }
551 
556  protected static function getIIProps() {
557  return implode( '|', self::IMAGE_INFO_PROPS );
558  }
559 
567  public function httpGetCached( $attribute, $query, $cacheTTL = 3600 ) {
568  if ( $this->mApiBase ) {
569  $url = wfAppendQuery( $this->mApiBase, $query );
570  } else {
571  $url = $this->makeUrl( $query, 'api' );
572  }
573 
574  return $this->wanCache->getWithSetCallback(
575  $this->getLocalCacheKey( $attribute, sha1( $url ) ),
576  $cacheTTL,
577  function ( $curValue, &$ttl ) use ( $url ) {
578  $html = self::httpGet( $url, 'default', [], $mtime );
579  if ( $html !== false ) {
580  $ttl = $mtime ? $this->wanCache->adaptiveTTL( $mtime, $ttl ) : $ttl;
581  } else {
582  $ttl = $this->wanCache->adaptiveTTL( $mtime, $ttl );
583  $html = null; // caches negatives
584  }
585 
586  return $html;
587  },
588  [ 'pcGroup' => 'http-get:3', 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
589  );
590  }
591 
596  public function enumFiles( $callback ) {
597  // @phan-suppress-previous-line PhanPluginNeverReturnMethod
598  throw new MWException( 'enumFiles is not supported by ' . static::class );
599  }
600 
604  protected function assertWritableRepo() {
605  throw new MWException( static::class . ': write operations are not supported.' );
606  }
607 }
ForeignAPIRepo\getApiUrl
getApiUrl()
Definition: ForeignAPIRepo.php:101
Page\PageIdentity
Interface for objects (potentially) representing an editable wiki page.
Definition: PageIdentity.php:64
ForeignAPIRepo\findBySha1
findBySha1( $hash)
Definition: ForeignAPIRepo.php:235
ForeignAPIRepo\httpGet
static httpGet( $url, $timeout='default', $options=[], &$mtime=false)
Definition: ForeignAPIRepo.php:517
MediaTransformError
Basic media transform error class.
Definition: MediaTransformError.php:31
ForeignAPIRepo\$apiThumbCacheExpiry
int $apiThumbCacheExpiry
Check back with Commons after this expiry.
Definition: ForeignAPIRepo.php:58
ForeignAPIRepo\getUserAgent
static getUserAgent()
The user agent the ForeignAPIRepo will use.
Definition: ForeignAPIRepo.php:471
FileRepo\validateFilename
validateFilename( $filename)
Determine if a relative path is valid, i.e.
Definition: FileRepo.php:1742
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:202
$lang
if(!isset( $args[0])) $lang
Definition: testCompression.php:37
FileRepo\makeUrl
makeUrl( $query='', $entry='index')
Make an url to this repo.
Definition: FileRepo.php:807
ForeignAPIRepo\$fileCacheExpiry
int $fileCacheExpiry
Redownload thumbnail files after this expiry.
Definition: ForeignAPIRepo.php:61
ForeignAPIRepo\$fileFactory
$fileFactory
Definition: ForeignAPIRepo.php:56
$wgLocalFileRepo
$wgLocalFileRepo
File repository structures.
Definition: DefaultSettings.php:705
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1665
FileRepo\$thumbUrl
string $thumbUrl
The base thumbnail URL.
Definition: FileRepo.php:112
ForeignAPIRepo\IMAGE_INFO_PROPS
const IMAGE_INFO_PROPS
List of iiprop values for the thumbnail fetch queries.
Definition: ForeignAPIRepo.php:51
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
ForeignAPIRepo\__construct
__construct( $info)
Definition: ForeignAPIRepo.php:72
ForeignAPIRepo\getThumbUrlFromCache
getThumbUrlFromCache( $name, $width, $height, $params="")
Return the imageurl from cache if possible.
Definition: ForeignAPIRepo.php:337
FileRepo\$backend
FileBackend $backend
Definition: FileRepo.php:68
ForeignAPIRepo\assertWritableRepo
assertWritableRepo()
Definition: ForeignAPIRepo.php:604
ForeignAPIRepo\getImageInfo
getImageInfo( $data)
Definition: ForeignAPIRepo.php:215
FileRepo
Base class for file repositories.
Definition: FileRepo.php:45
wfAppendQuery
wfAppendQuery( $url, $query)
Append a query string to an existing URL, which may or may not already have query string parameters a...
Definition: GlobalFunctions.php:422
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:146
MWException
MediaWiki exception.
Definition: MWException.php:29
MediaWiki\Logger\LoggerFactory
PSR-3 logger instance factory.
Definition: LoggerFactory.php:45
FileBackend\getFileTimestamp
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
FileBackend\isStoragePath
static isStoragePath( $path)
Check if a given path is a "mwstore://" path.
Definition: FileBackend.php:1509
ForeignAPIRepo\$mFileExists
array $mFileExists
Definition: ForeignAPIRepo.php:64
ForeignAPIRepo\getFileProps
getFileProps( $virtualUrl)
Definition: ForeignAPIRepo.php:180
$title
$title
Definition: testCompression.php:38
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:666
ForeignAPIRepo\httpGetCached
httpGetCached( $attribute, $query, $cacheTTL=3600)
HTTP GET request to a mediawiki API (with caching)
Definition: ForeignAPIRepo.php:567
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:894
FileRepo\getLocalCacheKey
getLocalCacheKey( $kClassSuffix,... $components)
Get a site-local, repository-qualified, WAN cache key.
Definition: FileRepo.php:1903
FileRepo\getDisplayName
getDisplayName()
Get the human-readable name of the repo.
Definition: FileRepo.php:1841
ForeignAPIRepo\getThumbUrl
getThumbUrl( $name, $width=-1, $height=-1, &$result=null, $otherParams='')
Definition: ForeignAPIRepo.php:264
$wgLanguageCode
$wgLanguageCode
Site language code.
Definition: DefaultSettings.php:3438
FileBackend\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
ForeignAPIRepo\fileExistsBatch
fileExistsBatch(array $files)
Definition: ForeignAPIRepo.php:125
FileBackend\quickCreate
quickCreate(array $params, array $opts=[])
Performs a single quick create operation.
Definition: FileBackend.php:755
FileBackend\prepare
prepare(array $params)
Prepare a storage directory for usage.
Definition: FileBackend.php:866
FileRepo\getBackend
getBackend()
Get the file backend instance.
Definition: FileRepo.php:251
ForeignAPIRepo\getIIProps
static getIIProps()
Definition: ForeignAPIRepo.php:556
ForeignAPIRepo\getZoneUrl
getZoneUrl( $zone, $ext=null)
Definition: ForeignAPIRepo.php:434
FileRepo\getHashPath
getHashPath( $name)
Get a relative path including trailing slash, e.g.
Definition: FileRepo.php:746
FileRepo\$name
string $name
Definition: FileRepo.php:159
ForeignAPIRepo\enumFiles
enumFiles( $callback)
Definition: ForeignAPIRepo.php:596
ForeignAPIRepo\VERSION
const VERSION
Definition: ForeignAPIRepo.php:46
ForeignAPIRepo
A foreign repository for a remote MediaWiki accessible through api.php requests.
Definition: ForeignAPIRepo.php:42
ForeignAPIRepo\newFile
newFile( $title, $time=false)
Per docs in FileRepo, this needs to return false if we don't support versioned files.
Definition: ForeignAPIRepo.php:113
ForeignAPIRepo\getZonePath
getZonePath( $zone)
Get the local directory corresponding to one of the basic zones.
Definition: ForeignAPIRepo.php:450
ForeignAPIFile\getProps
static getProps()
Get the property string for iiprop and aiprop.
Definition: ForeignAPIFile.php:94
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
wfWarn
wfWarn( $msg, $callerOffset=1, $level=E_USER_NOTICE)
Send a warning either to the debug log or in a PHP error depending on $wgDevelopmentWarnings.
Definition: GlobalFunctions.php:1043
ForeignAPIRepo\$mApiBase
string $mApiBase
Definition: ForeignAPIRepo.php:67
NS_FILE
const NS_FILE
Definition: Defines.php:70
MediaWiki\Linker\LinkTarget
Definition: LinkTarget.php:26
ForeignAPIFile
Foreign file accessible through api.php requests.
Definition: ForeignAPIFile.php:32
ForeignAPIRepo\getThumbError
getThumbError( $name, $width=-1, $height=-1, $otherParams='', $lang=null)
Definition: ForeignAPIRepo.php:295
ForeignAPIRepo\getInfo
getInfo()
Get information about the repo - overrides/extends the parent class's information.
Definition: ForeignAPIRepo.php:482
ForeignAPIRepo\canCacheThumbs
canCacheThumbs()
Are we locally caching the thumbnails?
Definition: ForeignAPIRepo.php:463
FileRepo\$url
string false $url
Public zone URL.
Definition: FileRepo.php:109
PROTO_HTTP
const PROTO_HTTP
Definition: Defines.php:192
ForeignAPIRepo\fetchImageQuery
fetchImageQuery( $query)
Definition: ForeignAPIRepo.php:188
wfExpandUrl
wfExpandUrl( $url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
Definition: GlobalFunctions.php:474