MediaWiki  master
ForeignAPIRepo.php
Go to the documentation of this file.
1 <?php
23 
39 class ForeignAPIRepo extends FileRepo {
40  /* This version string is used in the user agent for requests and will help
41  * server maintainers in identify ForeignAPI usage.
42  * Update the version every time you make breaking or significant changes. */
43  private const VERSION = "2.1";
44 
48  private const IMAGE_INFO_PROPS = [
49  'url',
50  'timestamp',
51  ];
52 
53  protected $fileFactory = [ ForeignAPIFile::class, 'newFromTitle' ];
55  protected $apiThumbCacheExpiry = 86400; // 1 day (24*3600)
56 
58  protected $fileCacheExpiry = 2592000; // 1 month (30*24*3600)
59 
61  protected $mFileExists = [];
62 
64  private $mApiBase;
65 
69  public function __construct( $info ) {
70  global $wgLocalFileRepo;
71  parent::__construct( $info );
72 
73  // https://commons.wikimedia.org/w/api.php
74  $this->mApiBase = $info['apibase'] ?? null;
75 
76  if ( isset( $info['apiThumbCacheExpiry'] ) ) {
77  $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry'];
78  }
79  if ( isset( $info['fileCacheExpiry'] ) ) {
80  $this->fileCacheExpiry = $info['fileCacheExpiry'];
81  }
82  if ( !$this->scriptDirUrl ) {
83  // hack for description fetches
84  $this->scriptDirUrl = dirname( $this->mApiBase );
85  }
86  // If we can cache thumbs we can guess sane defaults for these
87  if ( $this->canCacheThumbs() && !$this->url ) {
88  $this->url = $wgLocalFileRepo['url'];
89  }
90  if ( $this->canCacheThumbs() && !$this->thumbUrl ) {
91  $this->thumbUrl = $this->url . '/thumb';
92  }
93  }
94 
98  private function getApiUrl() {
99  return $this->mApiBase;
100  }
101 
110  public function newFile( $title, $time = false ) {
111  if ( $time ) {
112  return false;
113  }
114 
115  return parent::newFile( $title, $time );
116  }
117 
122  public function fileExistsBatch( array $files ) {
123  $results = [];
124  foreach ( $files as $k => $f ) {
125  if ( isset( $this->mFileExists[$f] ) ) {
126  $results[$k] = $this->mFileExists[$f];
127  unset( $files[$k] );
128  } elseif ( self::isVirtualUrl( $f ) ) {
129  # @todo FIXME: We need to be able to handle virtual
130  # URLs better, at least when we know they refer to the
131  # same repo.
132  $results[$k] = false;
133  unset( $files[$k] );
134  } elseif ( FileBackend::isStoragePath( $f ) ) {
135  $results[$k] = false;
136  unset( $files[$k] );
137  wfWarn( "Got mwstore:// path '$f'." );
138  }
139  }
140 
141  $data = $this->fetchImageQuery( [
142  'titles' => implode( '|', $files ),
143  'prop' => 'imageinfo' ]
144  );
145 
146  if ( isset( $data['query']['pages'] ) ) {
147  # First, get results from the query. Note we only care whether the image exists,
148  # not whether it has a description page.
149  foreach ( $data['query']['pages'] as $p ) {
150  $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' );
151  }
152  # Second, copy the results to any redirects that were queried
153  if ( isset( $data['query']['redirects'] ) ) {
154  foreach ( $data['query']['redirects'] as $r ) {
155  $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']];
156  }
157  }
158  # Third, copy the results to any non-normalized titles that were queried
159  if ( isset( $data['query']['normalized'] ) ) {
160  foreach ( $data['query']['normalized'] as $n ) {
161  $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']];
162  }
163  }
164  # Finally, copy the results to the output
165  foreach ( $files as $key => $file ) {
166  $results[$key] = $this->mFileExists[$file];
167  }
168  }
169 
170  return $results;
171  }
172 
177  public function getFileProps( $virtualUrl ) {
178  return [];
179  }
180 
185  public function fetchImageQuery( $query ) {
186  global $wgLanguageCode;
187 
188  $query = array_merge( $query,
189  [
190  'format' => 'json',
191  'action' => 'query',
192  'redirects' => 'true'
193  ] );
194 
195  if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
196  $query['uselang'] = $wgLanguageCode;
197  }
198 
199  $data = $this->httpGetCached( 'Metadata', $query );
200 
201  if ( $data ) {
202  return FormatJson::decode( $data, true );
203  } else {
204  return null;
205  }
206  }
207 
212  public function getImageInfo( $data ) {
213  if ( $data && isset( $data['query']['pages'] ) ) {
214  foreach ( $data['query']['pages'] as $info ) {
215  if ( isset( $info['imageinfo'][0] ) ) {
216  $return = $info['imageinfo'][0];
217  if ( isset( $info['pageid'] ) ) {
218  $return['pageid'] = $info['pageid'];
219  }
220  return $return;
221  }
222  }
223  }
224 
225  return false;
226  }
227 
232  public function findBySha1( $hash ) {
233  $results = $this->fetchImageQuery( [
234  'aisha1base36' => $hash,
235  'aiprop' => ForeignAPIFile::getProps(),
236  'list' => 'allimages',
237  ] );
238  $ret = [];
239  if ( isset( $results['query']['allimages'] ) ) {
240  foreach ( $results['query']['allimages'] as $img ) {
241  // 1.14 was broken, doesn't return name attribute
242  if ( !isset( $img['name'] ) ) {
243  continue;
244  }
245  $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img );
246  }
247  }
248 
249  return $ret;
250  }
251 
261  private function getThumbUrl(
262  $name, $width = -1, $height = -1, &$result = null, $otherParams = ''
263  ) {
264  $data = $this->fetchImageQuery( [
265  'titles' => 'File:' . $name,
266  'iiprop' => self::getIIProps(),
267  'iiurlwidth' => $width,
268  'iiurlheight' => $height,
269  'iiurlparam' => $otherParams,
270  'prop' => 'imageinfo' ] );
271  $info = $this->getImageInfo( $data );
272 
273  if ( $data && $info && isset( $info['thumburl'] ) ) {
274  wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] );
275  $result = $info;
276 
277  return $info['thumburl'];
278  } else {
279  return false;
280  }
281  }
282 
292  public function getThumbError(
293  $name, $width = -1, $height = -1, $otherParams = '', $lang = null
294  ) {
295  $data = $this->fetchImageQuery( [
296  'titles' => 'File:' . $name,
297  'iiprop' => self::getIIProps(),
298  'iiurlwidth' => $width,
299  'iiurlheight' => $height,
300  'iiurlparam' => $otherParams,
301  'prop' => 'imageinfo',
302  'uselang' => $lang,
303  ] );
304  $info = $this->getImageInfo( $data );
305 
306  if ( $data && $info && isset( $info['thumberror'] ) ) {
307  wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] );
308 
309  return new MediaTransformError(
310  'thumbnail_error_remote',
311  $width,
312  $height,
313  $this->getDisplayName(),
314  $info['thumberror'] // already parsed message from foreign repo
315  );
316  } else {
317  return false;
318  }
319  }
320 
334  public function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
335  // We can't check the local cache using FileRepo functions because
336  // we override fileExistsBatch(). We have to use the FileBackend directly.
337  $backend = $this->getBackend(); // convenience
338 
339  if ( !$this->canCacheThumbs() ) {
340  $result = null; // can't pass "null" by reference, but it's ok as default value
341 
342  return $this->getThumbUrl( $name, $width, $height, $result, $params );
343  }
344 
345  $key = $this->getLocalCacheKey( 'file-thumb-url', sha1( $name ) );
346  $sizekey = "$width:$height:$params";
347 
348  /* Get the array of urls that we already know */
349  $knownThumbUrls = $this->wanCache->get( $key );
350  if ( !$knownThumbUrls ) {
351  /* No knownThumbUrls for this file */
352  $knownThumbUrls = [];
353  } elseif ( isset( $knownThumbUrls[$sizekey] ) ) {
354  wfDebug( __METHOD__ . ': Got thumburl from local cache: ' .
355  "{$knownThumbUrls[$sizekey]}" );
356 
357  return $knownThumbUrls[$sizekey];
358  }
359 
360  $metadata = null;
361  $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
362 
363  if ( !$foreignUrl ) {
364  wfDebug( __METHOD__ . " Could not find thumburl" );
365 
366  return false;
367  }
368 
369  // We need the same filename as the remote one :)
370  $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) );
371  if ( !$this->validateFilename( $fileName ) ) {
372  wfDebug( __METHOD__ . " The deduced filename $fileName is not safe" );
373 
374  return false;
375  }
376  $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
377  $localFilename = $localPath . "/" . $fileName;
378  $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) .
379  rawurlencode( $name ) . "/" . rawurlencode( $fileName );
380 
381  if ( $backend->fileExists( [ 'src' => $localFilename ] )
382  && isset( $metadata['timestamp'] )
383  ) {
384  wfDebug( __METHOD__ . " Thumbnail was already downloaded before" );
385  $modified = $backend->getFileTimestamp( [ 'src' => $localFilename ] );
386  $remoteModified = strtotime( $metadata['timestamp'] );
387  $current = time();
388  $diff = abs( $modified - $current );
389  if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) {
390  /* Use our current and already downloaded thumbnail */
391  $knownThumbUrls[$sizekey] = $localUrl;
392  $this->wanCache->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
393 
394  return $localUrl;
395  }
396  /* There is a new Commons file, or existing thumbnail older than a month */
397  }
398 
399  $thumb = self::httpGet( $foreignUrl, 'default', [], $mtime );
400  if ( !$thumb ) {
401  wfDebug( __METHOD__ . " Could not download thumb" );
402 
403  return false;
404  }
405 
406  # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
407  $backend->prepare( [ 'dir' => dirname( $localFilename ) ] );
408  $params = [ 'dst' => $localFilename, 'content' => $thumb ];
409  if ( !$backend->quickCreate( $params )->isOK() ) {
410  wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'" );
411 
412  return $foreignUrl;
413  }
414  $knownThumbUrls[$sizekey] = $localUrl;
415 
416  $ttl = $mtime
417  ? $this->wanCache->adaptiveTTL( $mtime, $this->apiThumbCacheExpiry )
419  $this->wanCache->set( $key, $knownThumbUrls, $ttl );
420  wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache" );
421 
422  return $localUrl;
423  }
424 
431  public function getZoneUrl( $zone, $ext = null ) {
432  switch ( $zone ) {
433  case 'public':
434  return $this->url;
435  case 'thumb':
436  return $this->thumbUrl;
437  default:
438  return parent::getZoneUrl( $zone, $ext );
439  }
440  }
441 
447  public function getZonePath( $zone ) {
448  $supported = [ 'public', 'thumb' ];
449  if ( in_array( $zone, $supported ) ) {
450  return parent::getZonePath( $zone );
451  }
452 
453  return false;
454  }
455 
460  public function canCacheThumbs() {
461  return ( $this->apiThumbCacheExpiry > 0 );
462  }
463 
468  public static function getUserAgent() {
469  return Http::userAgent() . " ForeignAPIRepo/" . self::VERSION;
470  }
471 
478  public function getInfo() {
479  $info = parent::getInfo();
480  $info['apiurl'] = $this->getApiUrl();
481 
482  $query = [
483  'format' => 'json',
484  'action' => 'query',
485  'meta' => 'siteinfo',
486  'siprop' => 'general',
487  ];
488 
489  $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
490 
491  if ( $data ) {
492  $siteInfo = FormatJson::decode( $data, true );
493  $general = $siteInfo['query']['general'];
494 
495  $info['articlepath'] = $general['articlepath'];
496  $info['server'] = $general['server'];
497 
498  if ( isset( $general['favicon'] ) ) {
499  $info['favicon'] = $general['favicon'];
500  }
501  }
502 
503  return $info;
504  }
505 
516  public static function httpGet(
517  $url, $timeout = 'default', $options = [], &$mtime = false
518  ) {
519  $options['timeout'] = $timeout;
520  /* Http::get */
522  wfDebug( "ForeignAPIRepo: HTTP GET: $url" );
523  $options['method'] = "GET";
524 
525  if ( !isset( $options['timeout'] ) ) {
526  $options['timeout'] = 'default';
527  }
528 
529  $req = MWHttpRequest::factory( $url, $options, __METHOD__ );
530  $req->setUserAgent( self::getUserAgent() );
531  $status = $req->execute();
532 
533  if ( $status->isOK() ) {
534  $lmod = $req->getResponseHeader( 'Last-Modified' );
535  $mtime = $lmod ? wfTimestamp( TS_UNIX, $lmod ) : false;
536 
537  return $req->getContent();
538  } else {
539  $logger = LoggerFactory::getInstance( 'http' );
540  $logger->warning(
541  $status->getWikiText( false, false, 'en' ),
542  [ 'caller' => 'ForeignAPIRepo::httpGet' ]
543  );
544 
545  return false;
546  }
547  }
548 
553  protected static function getIIProps() {
554  return implode( '|', self::IMAGE_INFO_PROPS );
555  }
556 
564  public function httpGetCached( $attribute, $query, $cacheTTL = 3600 ) {
565  if ( $this->mApiBase ) {
566  $url = wfAppendQuery( $this->mApiBase, $query );
567  } else {
568  $url = $this->makeUrl( $query, 'api' );
569  }
570 
571  return $this->wanCache->getWithSetCallback(
572  $this->getLocalCacheKey( $attribute, sha1( $url ) ),
573  $cacheTTL,
574  function ( $curValue, &$ttl ) use ( $url ) {
575  $html = self::httpGet( $url, 'default', [], $mtime );
576  if ( $html !== false ) {
577  $ttl = $mtime ? $this->wanCache->adaptiveTTL( $mtime, $ttl ) : $ttl;
578  } else {
579  $ttl = $this->wanCache->adaptiveTTL( $mtime, $ttl );
580  $html = null; // caches negatives
581  }
582 
583  return $html;
584  },
585  [ 'pcGroup' => 'http-get:3', 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
586  );
587  }
588 
593  public function enumFiles( $callback ) {
594  throw new MWException( 'enumFiles is not supported by ' . static::class );
595  }
596 
600  protected function assertWritableRepo() {
601  throw new MWException( static::class . ': write operations are not supported.' );
602  }
603 }
ForeignAPIRepo\getApiUrl
getApiUrl()
Definition: ForeignAPIRepo.php:98
ForeignAPIRepo\findBySha1
findBySha1( $hash)
Definition: ForeignAPIRepo.php:232
ForeignAPIRepo\httpGet
static httpGet( $url, $timeout='default', $options=[], &$mtime=false)
Like a HttpRequestFactory::get request, but with custom User-Agent.
Definition: ForeignAPIRepo.php:516
MediaTransformError
Basic media transform error class.
Definition: MediaTransformError.php:31
ForeignAPIRepo\$apiThumbCacheExpiry
int $apiThumbCacheExpiry
Check back with Commons after this expiry.
Definition: ForeignAPIRepo.php:55
ForeignAPIRepo\getUserAgent
static getUserAgent()
The user agent the ForeignAPIRepo will use.
Definition: ForeignAPIRepo.php:468
FileRepo\validateFilename
validateFilename( $filename)
Determine if a relative path is valid, i.e.
Definition: FileRepo.php:1754
$lang
if(!isset( $args[0])) $lang
Definition: testCompression.php:37
FileRepo\makeUrl
makeUrl( $query='', $entry='index')
Make an url to this repo.
Definition: FileRepo.php:805
ForeignAPIRepo\$fileCacheExpiry
int $fileCacheExpiry
Redownload thumbnail files after this expiry.
Definition: ForeignAPIRepo.php:58
ForeignAPIRepo\$fileFactory
$fileFactory
Definition: ForeignAPIRepo.php:53
$wgLocalFileRepo
$wgLocalFileRepo
File repository structures.
Definition: DefaultSettings.php:660
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1832
Http\userAgent
static userAgent()
A standard user-agent we can use for external requests.
Definition: Http.php:97
FileRepo\$thumbUrl
string $thumbUrl
The base thumbnail URL.
Definition: FileRepo.php:108
ForeignAPIRepo\IMAGE_INFO_PROPS
const IMAGE_INFO_PROPS
List of iiprop values for the thumbnail fetch queries.
Definition: ForeignAPIRepo.php:48
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
ForeignAPIRepo\__construct
__construct( $info)
Definition: ForeignAPIRepo.php:69
ForeignAPIRepo\getThumbUrlFromCache
getThumbUrlFromCache( $name, $width, $height, $params="")
Return the imageurl from cache if possible.
Definition: ForeignAPIRepo.php:334
FileRepo\$backend
FileBackend $backend
Definition: FileRepo.php:64
ForeignAPIRepo\assertWritableRepo
assertWritableRepo()
Definition: ForeignAPIRepo.php:600
ForeignAPIRepo\getImageInfo
getImageInfo( $data)
Definition: ForeignAPIRepo.php:212
FileRepo
Base class for file repositories.
Definition: FileRepo.php:41
wfAppendQuery
wfAppendQuery( $url, $query)
Append a query string to an existing URL, which may or may not already have query string parameters a...
Definition: GlobalFunctions.php:443
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:174
MWException
MediaWiki exception.
Definition: MWException.php:29
MediaWiki\Logger\LoggerFactory
PSR-3 logger instance factory.
Definition: LoggerFactory.php:45
FileBackend\getFileTimestamp
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
FileBackend\isStoragePath
static isStoragePath( $path)
Check if a given path is a "mwstore://" path.
Definition: FileBackend.php:1528
ForeignAPIRepo\$mFileExists
array $mFileExists
Definition: ForeignAPIRepo.php:61
ForeignAPIRepo\getFileProps
getFileProps( $virtualUrl)
Definition: ForeignAPIRepo.php:177
$title
$title
Definition: testCompression.php:38
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:627
ForeignAPIRepo\httpGetCached
httpGetCached( $attribute, $query, $cacheTTL=3600)
HTTP GET request to a mediawiki API (with caching)
Definition: ForeignAPIRepo.php:564
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:915
FileRepo\getLocalCacheKey
getLocalCacheKey( $kClassSuffix,... $components)
Get a site-local, repository-qualified, WAN cache key.
Definition: FileRepo.php:1915
FileRepo\getDisplayName
getDisplayName()
Get the human-readable name of the repo.
Definition: FileRepo.php:1853
ForeignAPIRepo\getThumbUrl
getThumbUrl( $name, $width=-1, $height=-1, &$result=null, $otherParams='')
Definition: ForeignAPIRepo.php:261
$wgLanguageCode
$wgLanguageCode
Site language code.
Definition: DefaultSettings.php:3183
FileBackend\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
ForeignAPIRepo\fileExistsBatch
fileExistsBatch(array $files)
Definition: ForeignAPIRepo.php:122
FileBackend\quickCreate
quickCreate(array $params, array $opts=[])
Performs a single quick create operation.
Definition: FileBackend.php:765
FileBackend\prepare
prepare(array $params)
Prepare a storage directory for usage.
Definition: FileBackend.php:876
FileRepo\getBackend
getBackend()
Get the file backend instance.
Definition: FileRepo.php:247
ForeignAPIRepo\getIIProps
static getIIProps()
Definition: ForeignAPIRepo.php:553
ForeignAPIRepo\getZoneUrl
getZoneUrl( $zone, $ext=null)
Definition: ForeignAPIRepo.php:431
FileRepo\getHashPath
getHashPath( $name)
Get a relative path including trailing slash, e.g.
Definition: FileRepo.php:744
FileRepo\$name
string $name
Definition: FileRepo.php:155
ForeignAPIRepo\enumFiles
enumFiles( $callback)
Definition: ForeignAPIRepo.php:593
ForeignAPIRepo\VERSION
const VERSION
Definition: ForeignAPIRepo.php:43
ForeignAPIRepo
A foreign repository for a remote MediaWiki accessible through api.php requests.
Definition: ForeignAPIRepo.php:39
ForeignAPIRepo\newFile
newFile( $title, $time=false)
Per docs in FileRepo, this needs to return false if we don't support versioned files.
Definition: ForeignAPIRepo.php:110
ForeignAPIRepo\getZonePath
getZonePath( $zone)
Get the local directory corresponding to one of the basic zones.
Definition: ForeignAPIRepo.php:447
ForeignAPIFile\getProps
static getProps()
Get the property string for iiprop and aiprop.
Definition: ForeignAPIFile.php:93
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
wfWarn
wfWarn( $msg, $callerOffset=1, $level=E_USER_NOTICE)
Send a warning either to the debug log or in a PHP error depending on $wgDevelopmentWarnings.
Definition: GlobalFunctions.php:1081
ForeignAPIRepo\$mApiBase
string $mApiBase
Definition: ForeignAPIRepo.php:64
NS_FILE
const NS_FILE
Definition: Defines.php:70
ForeignAPIFile
Foreign file accessible through api.php requests.
Definition: ForeignAPIFile.php:29
ForeignAPIRepo\getThumbError
getThumbError( $name, $width=-1, $height=-1, $otherParams='', $lang=null)
Definition: ForeignAPIRepo.php:292
ForeignAPIRepo\getInfo
getInfo()
Get information about the repo - overrides/extends the parent class's information.
Definition: ForeignAPIRepo.php:478
ForeignAPIRepo\canCacheThumbs
canCacheThumbs()
Are we locally caching the thumbnails?
Definition: ForeignAPIRepo.php:460
FileRepo\$url
string false $url
Public zone URL.
Definition: FileRepo.php:105
PROTO_HTTP
const PROTO_HTTP
Definition: Defines.php:203
ForeignAPIRepo\fetchImageQuery
fetchImageQuery( $query)
Definition: ForeignAPIRepo.php:185
MWHttpRequest\factory
static factory( $url, array $options=null, $caller=__METHOD__)
Generate a new request object.
Definition: MWHttpRequest.php:195
wfExpandUrl
wfExpandUrl( $url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
Definition: GlobalFunctions.php:495