MediaWiki  master
ForeignAPIRepo.php
Go to the documentation of this file.
1 <?php
25 
41 class ForeignAPIRepo extends FileRepo {
42  /* This version string is used in the user agent for requests and will help
43  * server maintainers in identify ForeignAPI usage.
44  * Update the version every time you make breaking or significant changes. */
45  private const VERSION = "2.1";
46 
50  private const IMAGE_INFO_PROPS = [
51  'url',
52  'timestamp',
53  ];
54 
55  protected $fileFactory = [ ForeignAPIFile::class, 'newFromTitle' ];
57  protected $apiThumbCacheExpiry = 86400; // 1 day (24*3600)
58 
60  protected $fileCacheExpiry = 2592000; // 1 month (30*24*3600)
61 
63  protected $mFileExists = [];
64 
66  private $mApiBase;
67 
71  public function __construct( $info ) {
72  global $wgLocalFileRepo;
73  parent::__construct( $info );
74 
75  // https://commons.wikimedia.org/w/api.php
76  $this->mApiBase = $info['apibase'] ?? null;
77 
78  if ( isset( $info['apiThumbCacheExpiry'] ) ) {
79  $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry'];
80  }
81  if ( isset( $info['fileCacheExpiry'] ) ) {
82  $this->fileCacheExpiry = $info['fileCacheExpiry'];
83  }
84  if ( !$this->scriptDirUrl ) {
85  // hack for description fetches
86  $this->scriptDirUrl = dirname( $this->mApiBase );
87  }
88  // If we can cache thumbs we can guess sane defaults for these
89  if ( $this->canCacheThumbs() && !$this->url ) {
90  $this->url = $wgLocalFileRepo['url'];
91  }
92  if ( $this->canCacheThumbs() && !$this->thumbUrl ) {
93  $this->thumbUrl = $this->url . '/thumb';
94  }
95  }
96 
100  private function getApiUrl() {
101  return $this->mApiBase;
102  }
103 
112  public function newFile( $title, $time = false ) {
113  if ( $time ) {
114  return false;
115  }
116 
117  return parent::newFile( $title, $time );
118  }
119 
124  public function fileExistsBatch( array $files ) {
125  $results = [];
126  foreach ( $files as $k => $f ) {
127  if ( isset( $this->mFileExists[$f] ) ) {
128  $results[$k] = $this->mFileExists[$f];
129  unset( $files[$k] );
130  } elseif ( self::isVirtualUrl( $f ) ) {
131  # @todo FIXME: We need to be able to handle virtual
132  # URLs better, at least when we know they refer to the
133  # same repo.
134  $results[$k] = false;
135  unset( $files[$k] );
136  } elseif ( FileBackend::isStoragePath( $f ) ) {
137  $results[$k] = false;
138  unset( $files[$k] );
139  wfWarn( "Got mwstore:// path '$f'." );
140  }
141  }
142 
143  $data = $this->fetchImageQuery( [
144  'titles' => implode( '|', $files ),
145  'prop' => 'imageinfo' ]
146  );
147 
148  if ( isset( $data['query']['pages'] ) ) {
149  # First, get results from the query. Note we only care whether the image exists,
150  # not whether it has a description page.
151  foreach ( $data['query']['pages'] as $p ) {
152  $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' );
153  }
154  # Second, copy the results to any redirects that were queried
155  if ( isset( $data['query']['redirects'] ) ) {
156  foreach ( $data['query']['redirects'] as $r ) {
157  $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']];
158  }
159  }
160  # Third, copy the results to any non-normalized titles that were queried
161  if ( isset( $data['query']['normalized'] ) ) {
162  foreach ( $data['query']['normalized'] as $n ) {
163  $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']];
164  }
165  }
166  # Finally, copy the results to the output
167  foreach ( $files as $key => $file ) {
168  $results[$key] = $this->mFileExists[$file];
169  }
170  }
171 
172  return $results;
173  }
174 
179  public function getFileProps( $virtualUrl ) {
180  return [];
181  }
182 
187  public function fetchImageQuery( $query ) {
188  global $wgLanguageCode;
189 
190  $query = array_merge( $query,
191  [
192  'format' => 'json',
193  'action' => 'query',
194  'redirects' => 'true'
195  ] );
196 
197  if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
198  $query['uselang'] = $wgLanguageCode;
199  }
200 
201  $data = $this->httpGetCached( 'Metadata', $query );
202 
203  if ( $data ) {
204  return FormatJson::decode( $data, true );
205  } else {
206  return null;
207  }
208  }
209 
214  public function getImageInfo( $data ) {
215  if ( $data && isset( $data['query']['pages'] ) ) {
216  foreach ( $data['query']['pages'] as $info ) {
217  if ( isset( $info['imageinfo'][0] ) ) {
218  $return = $info['imageinfo'][0];
219  if ( isset( $info['pageid'] ) ) {
220  $return['pageid'] = $info['pageid'];
221  }
222  return $return;
223  }
224  }
225  }
226 
227  return false;
228  }
229 
234  public function findBySha1( $hash ) {
235  $results = $this->fetchImageQuery( [
236  'aisha1base36' => $hash,
237  'aiprop' => ForeignAPIFile::getProps(),
238  'list' => 'allimages',
239  ] );
240  $ret = [];
241  if ( isset( $results['query']['allimages'] ) ) {
242  foreach ( $results['query']['allimages'] as $img ) {
243  // 1.14 was broken, doesn't return name attribute
244  if ( !isset( $img['name'] ) ) {
245  continue;
246  }
247  $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img );
248  }
249  }
250 
251  return $ret;
252  }
253 
263  private function getThumbUrl(
264  $name, $width = -1, $height = -1, &$result = null, $otherParams = ''
265  ) {
266  $data = $this->fetchImageQuery( [
267  'titles' => 'File:' . $name,
268  'iiprop' => self::getIIProps(),
269  'iiurlwidth' => $width,
270  'iiurlheight' => $height,
271  'iiurlparam' => $otherParams,
272  'prop' => 'imageinfo' ] );
273  $info = $this->getImageInfo( $data );
274 
275  if ( $data && $info && isset( $info['thumburl'] ) ) {
276  wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] );
277  $result = $info;
278 
279  return $info['thumburl'];
280  } else {
281  return false;
282  }
283  }
284 
294  public function getThumbError(
295  $name, $width = -1, $height = -1, $otherParams = '', $lang = null
296  ) {
297  $data = $this->fetchImageQuery( [
298  'titles' => 'File:' . $name,
299  'iiprop' => self::getIIProps(),
300  'iiurlwidth' => $width,
301  'iiurlheight' => $height,
302  'iiurlparam' => $otherParams,
303  'prop' => 'imageinfo',
304  'uselang' => $lang,
305  ] );
306  $info = $this->getImageInfo( $data );
307 
308  if ( $data && $info && isset( $info['thumberror'] ) ) {
309  wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] );
310 
311  return new MediaTransformError(
312  'thumbnail_error_remote',
313  $width,
314  $height,
315  $this->getDisplayName(),
316  $info['thumberror'] // already parsed message from foreign repo
317  );
318  } else {
319  return false;
320  }
321  }
322 
336  public function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
337  // We can't check the local cache using FileRepo functions because
338  // we override fileExistsBatch(). We have to use the FileBackend directly.
339  $backend = $this->getBackend(); // convenience
340 
341  if ( !$this->canCacheThumbs() ) {
342  $result = null; // can't pass "null" by reference, but it's ok as default value
343 
344  return $this->getThumbUrl( $name, $width, $height, $result, $params );
345  }
346 
347  $key = $this->getLocalCacheKey( 'file-thumb-url', sha1( $name ) );
348  $sizekey = "$width:$height:$params";
349 
350  /* Get the array of urls that we already know */
351  $knownThumbUrls = $this->wanCache->get( $key );
352  if ( !$knownThumbUrls ) {
353  /* No knownThumbUrls for this file */
354  $knownThumbUrls = [];
355  } elseif ( isset( $knownThumbUrls[$sizekey] ) ) {
356  wfDebug( __METHOD__ . ': Got thumburl from local cache: ' .
357  "{$knownThumbUrls[$sizekey]}" );
358 
359  return $knownThumbUrls[$sizekey];
360  }
361 
362  $metadata = null;
363  $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
364 
365  if ( !$foreignUrl ) {
366  wfDebug( __METHOD__ . " Could not find thumburl" );
367 
368  return false;
369  }
370 
371  // We need the same filename as the remote one :)
372  $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) );
373  if ( !$this->validateFilename( $fileName ) ) {
374  wfDebug( __METHOD__ . " The deduced filename $fileName is not safe" );
375 
376  return false;
377  }
378  $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
379  $localFilename = $localPath . "/" . $fileName;
380  $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) .
381  rawurlencode( $name ) . "/" . rawurlencode( $fileName );
382 
383  if ( $backend->fileExists( [ 'src' => $localFilename ] )
384  && isset( $metadata['timestamp'] )
385  ) {
386  wfDebug( __METHOD__ . " Thumbnail was already downloaded before" );
387  $modified = $backend->getFileTimestamp( [ 'src' => $localFilename ] );
388  $remoteModified = strtotime( $metadata['timestamp'] );
389  $current = time();
390  $diff = abs( $modified - $current );
391  if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) {
392  /* Use our current and already downloaded thumbnail */
393  $knownThumbUrls[$sizekey] = $localUrl;
394  $this->wanCache->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
395 
396  return $localUrl;
397  }
398  /* There is a new Commons file, or existing thumbnail older than a month */
399  }
400 
401  $thumb = self::httpGet( $foreignUrl, 'default', [], $mtime );
402  if ( !$thumb ) {
403  wfDebug( __METHOD__ . " Could not download thumb" );
404 
405  return false;
406  }
407 
408  # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
409  $backend->prepare( [ 'dir' => dirname( $localFilename ) ] );
410  $params = [ 'dst' => $localFilename, 'content' => $thumb ];
411  if ( !$backend->quickCreate( $params )->isOK() ) {
412  wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'" );
413 
414  return $foreignUrl;
415  }
416  $knownThumbUrls[$sizekey] = $localUrl;
417 
418  $ttl = $mtime
419  ? $this->wanCache->adaptiveTTL( $mtime, $this->apiThumbCacheExpiry )
421  $this->wanCache->set( $key, $knownThumbUrls, $ttl );
422  wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache" );
423 
424  return $localUrl;
425  }
426 
433  public function getZoneUrl( $zone, $ext = null ) {
434  switch ( $zone ) {
435  case 'public':
436  return $this->url;
437  case 'thumb':
438  return $this->thumbUrl;
439  default:
440  return parent::getZoneUrl( $zone, $ext );
441  }
442  }
443 
449  public function getZonePath( $zone ) {
450  $supported = [ 'public', 'thumb' ];
451  if ( in_array( $zone, $supported ) ) {
452  return parent::getZonePath( $zone );
453  }
454 
455  return false;
456  }
457 
462  public function canCacheThumbs() {
463  return ( $this->apiThumbCacheExpiry > 0 );
464  }
465 
470  public static function getUserAgent() {
471  return Http::userAgent() . " ForeignAPIRepo/" . self::VERSION;
472  }
473 
480  public function getInfo() {
481  $info = parent::getInfo();
482  $info['apiurl'] = $this->getApiUrl();
483 
484  $query = [
485  'format' => 'json',
486  'action' => 'query',
487  'meta' => 'siteinfo',
488  'siprop' => 'general',
489  ];
490 
491  $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
492 
493  if ( $data ) {
494  $siteInfo = FormatJson::decode( $data, true );
495  $general = $siteInfo['query']['general'];
496 
497  $info['articlepath'] = $general['articlepath'];
498  $info['server'] = $general['server'];
499 
500  if ( isset( $general['favicon'] ) ) {
501  $info['favicon'] = $general['favicon'];
502  }
503  }
504 
505  return $info;
506  }
507 
518  public static function httpGet(
519  $url, $timeout = 'default', $options = [], &$mtime = false
520  ) {
521  $options['timeout'] = $timeout;
522  /* Http::get */
524  wfDebug( "ForeignAPIRepo: HTTP GET: $url" );
525  $options['method'] = "GET";
526 
527  if ( !isset( $options['timeout'] ) ) {
528  $options['timeout'] = 'default';
529  }
530 
531  $req = MWHttpRequest::factory( $url, $options, __METHOD__ );
532  $req->setUserAgent( self::getUserAgent() );
533  $status = $req->execute();
534 
535  if ( $status->isOK() ) {
536  $lmod = $req->getResponseHeader( 'Last-Modified' );
537  $mtime = $lmod ? wfTimestamp( TS_UNIX, $lmod ) : false;
538 
539  return $req->getContent();
540  } else {
541  $logger = LoggerFactory::getInstance( 'http' );
542  $logger->warning(
543  $status->getWikiText( false, false, 'en' ),
544  [ 'caller' => 'ForeignAPIRepo::httpGet' ]
545  );
546 
547  return false;
548  }
549  }
550 
555  protected static function getIIProps() {
556  return implode( '|', self::IMAGE_INFO_PROPS );
557  }
558 
566  public function httpGetCached( $attribute, $query, $cacheTTL = 3600 ) {
567  if ( $this->mApiBase ) {
568  $url = wfAppendQuery( $this->mApiBase, $query );
569  } else {
570  $url = $this->makeUrl( $query, 'api' );
571  }
572 
573  return $this->wanCache->getWithSetCallback(
574  $this->getLocalCacheKey( $attribute, sha1( $url ) ),
575  $cacheTTL,
576  function ( $curValue, &$ttl ) use ( $url ) {
577  $html = self::httpGet( $url, 'default', [], $mtime );
578  if ( $html !== false ) {
579  $ttl = $mtime ? $this->wanCache->adaptiveTTL( $mtime, $ttl ) : $ttl;
580  } else {
581  $ttl = $this->wanCache->adaptiveTTL( $mtime, $ttl );
582  $html = null; // caches negatives
583  }
584 
585  return $html;
586  },
587  [ 'pcGroup' => 'http-get:3', 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
588  );
589  }
590 
595  public function enumFiles( $callback ) {
596  throw new MWException( 'enumFiles is not supported by ' . static::class );
597  }
598 
602  protected function assertWritableRepo() {
603  throw new MWException( static::class . ': write operations are not supported.' );
604  }
605 }
ForeignAPIRepo\getApiUrl
getApiUrl()
Definition: ForeignAPIRepo.php:100
ForeignAPIRepo\findBySha1
findBySha1( $hash)
Definition: ForeignAPIRepo.php:234
ForeignAPIRepo\httpGet
static httpGet( $url, $timeout='default', $options=[], &$mtime=false)
Like a HttpRequestFactory::get request, but with custom User-Agent.
Definition: ForeignAPIRepo.php:518
MediaTransformError
Basic media transform error class.
Definition: MediaTransformError.php:31
ForeignAPIRepo\$apiThumbCacheExpiry
int $apiThumbCacheExpiry
Check back with Commons after this expiry.
Definition: ForeignAPIRepo.php:57
ForeignAPIRepo\getUserAgent
static getUserAgent()
The user agent the ForeignAPIRepo will use.
Definition: ForeignAPIRepo.php:470
FileRepo\validateFilename
validateFilename( $filename)
Determine if a relative path is valid, i.e.
Definition: FileRepo.php:1741
$lang
if(!isset( $args[0])) $lang
Definition: testCompression.php:37
FileRepo\makeUrl
makeUrl( $query='', $entry='index')
Make an url to this repo.
Definition: FileRepo.php:792
ForeignAPIRepo\$fileCacheExpiry
int $fileCacheExpiry
Redownload thumbnail files after this expiry.
Definition: ForeignAPIRepo.php:60
ForeignAPIRepo\$fileFactory
$fileFactory
Definition: ForeignAPIRepo.php:55
$wgLocalFileRepo
$wgLocalFileRepo
File repository structures.
Definition: DefaultSettings.php:618
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1815
Http\userAgent
static userAgent()
A standard user-agent we can use for external requests.
Definition: Http.php:97
FileRepo\$thumbUrl
string $thumbUrl
The base thumbnail URL.
Definition: FileRepo.php:108
ForeignAPIRepo\IMAGE_INFO_PROPS
const IMAGE_INFO_PROPS
List of iiprop values for the thumbnail fetch queries.
Definition: ForeignAPIRepo.php:50
NS_FILE
const NS_FILE
Definition: Defines.php:75
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
ForeignAPIRepo\__construct
__construct( $info)
Definition: ForeignAPIRepo.php:71
ForeignAPIRepo\getThumbUrlFromCache
getThumbUrlFromCache( $name, $width, $height, $params="")
Return the imageurl from cache if possible.
Definition: ForeignAPIRepo.php:336
FileRepo\$backend
FileBackend $backend
Definition: FileRepo.php:64
ForeignAPIRepo\assertWritableRepo
assertWritableRepo()
Definition: ForeignAPIRepo.php:602
ForeignAPIRepo\getImageInfo
getImageInfo( $data)
Definition: ForeignAPIRepo.php:214
FileRepo
Base class for file repositories.
Definition: FileRepo.php:41
wfAppendQuery
wfAppendQuery( $url, $query)
Append a query string to an existing URL, which may or may not already have query string parameters a...
Definition: GlobalFunctions.php:438
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:174
MWException
MediaWiki exception.
Definition: MWException.php:29
MediaWiki\Logger\LoggerFactory
PSR-3 logger instance factory.
Definition: LoggerFactory.php:45
FileBackend\getFileTimestamp
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
FileBackend\isStoragePath
static isStoragePath( $path)
Check if a given path is a "mwstore://" path.
Definition: FileBackend.php:1528
ForeignAPIRepo\$mFileExists
array $mFileExists
Definition: ForeignAPIRepo.php:63
ForeignAPIRepo\getFileProps
getFileProps( $virtualUrl)
Definition: ForeignAPIRepo.php:179
$title
$title
Definition: testCompression.php:38
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:591
ForeignAPIRepo\httpGetCached
httpGetCached( $attribute, $query, $cacheTTL=3600)
HTTP GET request to a mediawiki API (with caching)
Definition: ForeignAPIRepo.php:566
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:910
FileRepo\getLocalCacheKey
getLocalCacheKey( $kClassSuffix,... $components)
Get a site-local, repository-qualified, WAN cache key.
Definition: FileRepo.php:1902
FileRepo\getDisplayName
getDisplayName()
Get the human-readable name of the repo.
Definition: FileRepo.php:1840
ForeignAPIRepo\getThumbUrl
getThumbUrl( $name, $width=-1, $height=-1, &$result=null, $otherParams='')
Definition: ForeignAPIRepo.php:263
$wgLanguageCode
$wgLanguageCode
Site language code.
Definition: DefaultSettings.php:3116
FileBackend\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
PROTO_HTTP
const PROTO_HTTP
Definition: Defines.php:208
ForeignAPIRepo\fileExistsBatch
fileExistsBatch(array $files)
Definition: ForeignAPIRepo.php:124
FileBackend\quickCreate
quickCreate(array $params, array $opts=[])
Performs a single quick create operation.
Definition: FileBackend.php:765
FileBackend\prepare
prepare(array $params)
Prepare a storage directory for usage.
Definition: FileBackend.php:876
FileRepo\getBackend
getBackend()
Get the file backend instance.
Definition: FileRepo.php:244
ForeignAPIRepo\getIIProps
static getIIProps()
Definition: ForeignAPIRepo.php:555
ForeignAPIRepo\getZoneUrl
getZoneUrl( $zone, $ext=null)
Definition: ForeignAPIRepo.php:433
FileRepo\getHashPath
getHashPath( $name)
Get a relative path including trailing slash, e.g.
Definition: FileRepo.php:731
FileRepo\$name
string $name
Definition: FileRepo.php:152
ForeignAPIRepo\enumFiles
enumFiles( $callback)
Definition: ForeignAPIRepo.php:595
ForeignAPIRepo\VERSION
const VERSION
Definition: ForeignAPIRepo.php:45
ForeignAPIRepo
A foreign repository with a remote MediaWiki with an API thingy.
Definition: ForeignAPIRepo.php:41
ForeignAPIRepo\newFile
newFile( $title, $time=false)
Per docs in FileRepo, this needs to return false if we don't support versioned files.
Definition: ForeignAPIRepo.php:112
ForeignAPIRepo\getZonePath
getZonePath( $zone)
Get the local directory corresponding to one of the basic zones.
Definition: ForeignAPIRepo.php:449
ForeignAPIFile\getProps
static getProps()
Get the property string for iiprop and aiprop.
Definition: ForeignAPIFile.php:96
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
wfWarn
wfWarn( $msg, $callerOffset=1, $level=E_USER_NOTICE)
Send a warning either to the debug log or in a PHP error depending on $wgDevelopmentWarnings.
Definition: GlobalFunctions.php:1074
ForeignAPIRepo\$mApiBase
string $mApiBase
Definition: ForeignAPIRepo.php:66
ForeignAPIFile
Foreign file accessible through api.php requests.
Definition: ForeignAPIFile.php:32
ForeignAPIRepo\getThumbError
getThumbError( $name, $width=-1, $height=-1, $otherParams='', $lang=null)
Definition: ForeignAPIRepo.php:294
ForeignAPIRepo\getInfo
getInfo()
Get information about the repo - overrides/extends the parent class's information.
Definition: ForeignAPIRepo.php:480
ForeignAPIRepo\canCacheThumbs
canCacheThumbs()
Are we locally caching the thumbnails?
Definition: ForeignAPIRepo.php:462
FileRepo\$url
string false $url
Public zone URL.
Definition: FileRepo.php:105
ForeignAPIRepo\fetchImageQuery
fetchImageQuery( $query)
Definition: ForeignAPIRepo.php:187
MWHttpRequest\factory
static factory( $url, array $options=null, $caller=__METHOD__)
Generate a new request object.
Definition: MWHttpRequest.php:195
wfExpandUrl
wfExpandUrl( $url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
Definition: GlobalFunctions.php:490