MediaWiki  master
ForeignResourceManager.php
Go to the documentation of this file.
1 <?php
21 namespace MediaWiki\ResourceLoader;
22 
23 use Composer\Spdx\SpdxLicenses;
24 use Exception;
27 use PharData;
28 use RecursiveDirectoryIterator;
29 use RecursiveIteratorIterator;
30 use SplFileInfo;
31 use Symfony\Component\Yaml\Yaml;
32 
42  private $defaultAlgo = 'sha384';
43 
45  private $hasErrors = false;
46 
48  private $registryFile;
49 
51  private $libDir;
52 
54  private $tmpParentDir;
55 
57  private $cacheDir;
58 
63  private $infoPrinter;
64 
69  private $errorPrinter;
74  private $verbosePrinter;
75 
77  private $action;
78 
80  private $registry;
81 
90  public function __construct(
91  $registryFile,
92  $libDir,
93  callable $infoPrinter = null,
94  callable $errorPrinter = null,
95  callable $verbosePrinter = null
96  ) {
97  $this->registryFile = $registryFile;
98  $this->libDir = $libDir;
99  $this->infoPrinter = $infoPrinter ?? static function ( $_ ) {
100  };
101  $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
102  $this->verbosePrinter = $verbosePrinter ?? static function ( $_ ) {
103  };
104 
105  // Support XDG_CACHE_HOME to speed up CI by avoiding repeated downloads.
106  $conf = MediaWikiServices::getInstance()->getMainConfig();
107  if ( ( $cacheHome = getenv( 'XDG_CACHE_HOME' ) ) !== false ) {
108  $this->cacheDir = realpath( $cacheHome ) . '/mw-foreign';
109  } elseif ( ( $cacheConf = $conf->get( MainConfigNames::CacheDirectory ) ) !== false ) {
110  $this->cacheDir = "$cacheConf/ForeignResourceManager";
111  } else {
112  $this->cacheDir = "{$this->libDir}/.foreign/cache";
113  }
114  }
115 
122  public function run( $action, $module ) {
123  $actions = [ 'update', 'verify', 'make-sri' ];
124  if ( !in_array( $action, $actions ) ) {
125  $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
126  return false;
127  }
128  $this->action = $action;
129  $this->setupTempDir( $action );
130 
131  $this->registry = Yaml::parseFile( $this->registryFile );
132  if ( $module === 'all' ) {
133  $modules = $this->registry;
134  } elseif ( isset( $this->registry[$module] ) ) {
135  $modules = [ $module => $this->registry[$module] ];
136  } else {
137  $this->error( "Unknown module name.\n\nMust be one of:\n" .
138  wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
139  '.'
140  );
141  return false;
142  }
143 
144  foreach ( $modules as $moduleName => $info ) {
145  $this->verbose( "\n### {$moduleName}\n\n" );
146 
147  if ( $this->action === 'update' ) {
148  $this->output( "... updating '{$moduleName}'\n" );
149  } elseif ( $this->action === 'verify' ) {
150  $this->output( "... verifying '{$moduleName}'\n" );
151  } else {
152  $this->output( "... checking '{$moduleName}'\n" );
153  }
154 
155  // Do checks on yaml content (such as license existence, validity and type keys)
156  // before doing any potentially destructive actions (potentially deleting directories,
157  // depending on action.
158 
159  if ( !isset( $info['type'] ) ) {
160  throw new Exception( "Module '$moduleName' must have a 'type' key." );
161  }
162 
163  $this->validateLicense( $moduleName, $info );
164 
165  if ( $info['type'] === 'doc-only' ) {
166  $this->output( "... {$moduleName} is documentation-only, skipping\n" );
167  continue;
168  }
169 
170  $destDir = "{$this->libDir}/$moduleName";
171 
172  if ( $this->action === 'update' ) {
173  $this->verbose( "... emptying directory for $moduleName\n" );
174  wfRecursiveRemoveDir( $destDir );
175  }
176 
177  $this->verbose( "... preparing {$this->tmpParentDir}\n" );
178  wfRecursiveRemoveDir( $this->tmpParentDir );
179  if ( !wfMkdirParents( $this->tmpParentDir ) ) {
180  throw new Exception( "Unable to create {$this->tmpParentDir}" );
181  }
182 
183  switch ( $info['type'] ) {
184  case 'tar':
185  case 'zip':
186  $this->handleTypeTar( $moduleName, $destDir, $info, $info['type'] );
187  break;
188  case 'file':
189  $this->handleTypeFile( $moduleName, $destDir, $info );
190  break;
191  case 'multi-file':
192  $this->handleTypeMultiFile( $moduleName, $destDir, $info );
193  break;
194  default:
195  throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
196  }
197 
198  if ( $this->action === 'update' ) {
199  foreach ( $info['transforms'] ?? [] as $file => $transforms ) {
200  $fullFilePath = "$destDir/$file";
201  if ( !file_exists( $fullFilePath ) ) {
202  throw new Exception( "$moduleName: invalid transform target $file" );
203  }
204  if ( !is_array( $transforms ) || !array_is_list( $transforms ) ) {
205  $transforms = [ $transforms ];
206  }
207  foreach ( $transforms as $transform ) {
208  if ( $transform === 'nomin' ) {
209  // not super efficient but these files aren't expected to be large
210  file_put_contents( $fullFilePath, "/*@nomin*/\n" . file_get_contents( $fullFilePath ) );
211  } else {
212  throw new Exception( "$moduleName: invalid transform $transform" );
213  }
214  }
215  }
216  }
217  }
218 
219  $this->cleanUp();
220  if ( $this->hasErrors ) {
221  // The "verify" action should check all modules and files and fail after, not during.
222  // We don't throw on the first issue so that developers enjoy access to all actionable
223  // information at once (given we can't have cascading errors).
224  // The "verify" action prints errors along the way and simply exits here.
225  return false;
226  }
227 
228  return true;
229  }
230 
236  private function setupTempDir( $action ) {
237  if ( $action === 'verify' ) {
238  $this->tmpParentDir = wfTempDir() . '/ForeignResourceManager';
239  } else {
240  // Use a temporary directory under the destination directory instead
241  // of wfTempDir() because PHP's rename() does not work across file
242  // systems, and the user's /tmp and $IP may be on different filesystems.
243  $this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
244  }
245  }
246 
253  private function cacheKey( $src, $integrity, $moduleName ) {
254  $key = $moduleName
255  . '_' . hash( 'fnv132', $integrity )
256  . '_' . hash( 'fnv132', $src )
257  // Append readable filename to aid cache inspection and debugging
258  . '_' . basename( $src );
259  $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
260  return rtrim( $key, '_' );
261  }
262 
267  private function cacheGet( $key ) {
268  // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
269  return @file_get_contents( "{$this->cacheDir}/$key.data" );
270  }
271 
276  private function cacheSet( $key, $data ) {
277  // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
278  @mkdir( $this->cacheDir, 0777, true );
279  file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
280  }
281 
288  private function fetch( string $src, $integrity, string $moduleName ) {
289  if ( $integrity !== null ) {
290  $key = $this->cacheKey( $src, $integrity, $moduleName );
291  $data = $this->cacheGet( $key );
292  if ( $data ) {
293  return $data;
294  }
295  }
296 
297  $req = MediaWikiServices::getInstance()->getHttpRequestFactory()
298  ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ );
299  if ( !$req->execute()->isOK() ) {
300  throw new Exception( "Failed to download resource at {$src}" );
301  }
302  if ( $req->getStatus() !== 200 ) {
303  throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
304  }
305  $data = $req->getContent();
306  $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
307  $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
308  if ( $integrity === $actualIntegrity ) {
309  $this->verbose( "... passed integrity check for {$src}\n" );
310  $key = $this->cacheKey( $src, $actualIntegrity, $moduleName );
311  $this->cacheSet( $key, $data );
312  } elseif ( $this->action === 'make-sri' ) {
313  $this->output( "Integrity for {$src}\n\tintegrity: {$actualIntegrity}\n" );
314  } else {
315  $expectedIntegrity = $integrity ?? 'null';
316  throw new Exception( "Integrity check failed for {$src}\n" .
317  "\tExpected: {$expectedIntegrity}\n" .
318  "\tActual: {$actualIntegrity}"
319  );
320  }
321  return $data;
322  }
323 
329  private function handleTypeFile( $moduleName, $destDir, array $info ) {
330  if ( !isset( $info['src'] ) ) {
331  throw new Exception( "Module '$moduleName' must have a 'src' key." );
332  }
333  $data = $this->fetch( $info['src'], $info['integrity'] ?? null, $moduleName );
334  $dest = $info['dest'] ?? basename( $info['src'] );
335  $path = "$destDir/$dest";
336  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
337  $this->error( "File for '$moduleName' is different.\n" );
338  }
339  if ( $this->action === 'update' ) {
340  wfMkdirParents( $destDir );
341  file_put_contents( "$destDir/$dest", $data );
342  }
343  }
344 
350  private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
351  if ( !isset( $info['files'] ) ) {
352  throw new Exception( "Module '$moduleName' must have a 'files' key." );
353  }
354  foreach ( $info['files'] as $dest => $file ) {
355  if ( !isset( $file['src'] ) ) {
356  throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
357  }
358  $data = $this->fetch( $file['src'], $file['integrity'] ?? null, $moduleName );
359  $path = "$destDir/$dest";
360  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
361  $this->error( "File '$dest' for '$moduleName' is different.\n" );
362  } elseif ( $this->action === 'update' ) {
363  wfMkdirParents( $destDir );
364  file_put_contents( "$destDir/$dest", $data );
365  }
366  }
367  }
368 
375  private function handleTypeTar( $moduleName, $destDir, array $info, string $fileType ) {
376  $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
377  if ( $info['src'] === null ) {
378  throw new Exception( "Module '$moduleName' must have a 'src' key." );
379  }
380  // Download the resource to a temporary file and open it
381  $data = $this->fetch( $info['src'], $info['integrity'], $moduleName );
382  $tmpFile = "{$this->tmpParentDir}/$moduleName." . $fileType;
383  $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
384  file_put_contents( $tmpFile, $data );
385  $p = new PharData( $tmpFile );
386  $tmpDir = "{$this->tmpParentDir}/$moduleName";
387  $p->extractTo( $tmpDir );
388  unset( $data, $p );
389 
390  if ( $info['dest'] === null ) {
391  // Default: Replace the entire directory
392  $toCopy = [ $tmpDir => $destDir ];
393  } else {
394  // Expand and normalise the 'dest' entries
395  $toCopy = [];
396  foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
397  // Use glob() to expand wildcards and check existence
398  $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
399  if ( !$fromPaths ) {
400  throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
401  }
402  foreach ( $fromPaths as $fromPath ) {
403  $toCopy[$fromPath] = $toSubPath === null
404  ? "$destDir/" . basename( $fromPath )
405  : "$destDir/$toSubPath/" . basename( $fromPath );
406  }
407  }
408  }
409  foreach ( $toCopy as $from => $to ) {
410  if ( $this->action === 'verify' ) {
411  $this->verbose( "... verifying $to\n" );
412  if ( is_dir( $from ) ) {
413  $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
414  $from,
415  RecursiveDirectoryIterator::SKIP_DOTS
416  ) );
418  foreach ( $rii as $file ) {
419  $remote = $file->getPathname();
420  $local = strtr( $remote, [ $from => $to ] );
421  if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
422  $this->error( "File '$local' is different.\n" );
423  }
424  }
425  } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
426  $this->error( "File '$to' is different.\n" );
427  }
428  } elseif ( $this->action === 'update' ) {
429  $this->verbose( "... moving $from to $to\n" );
430  wfMkdirParents( dirname( $to ) );
431  if ( !rename( $from, $to ) ) {
432  throw new Exception( "Could not move $from to $to." );
433  }
434  }
435  }
436  }
437 
441  private function verbose( $text ) {
442  ( $this->verbosePrinter )( $text );
443  }
444 
448  private function output( $text ) {
449  ( $this->infoPrinter )( $text );
450  }
451 
455  private function error( $text ) {
456  $this->hasErrors = true;
457  ( $this->errorPrinter )( $text );
458  }
459 
460  private function cleanUp() {
461  wfRecursiveRemoveDir( $this->tmpParentDir );
462 
463  // Prune the cache of files we don't recognise.
464  $knownKeys = [];
465  foreach ( $this->registry as $module => $info ) {
466  if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
467  $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'], $module );
468  } elseif ( $info['type'] === 'multi-file' ) {
469  foreach ( $info['files'] as $file ) {
470  $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'], $module );
471  }
472  }
473  }
474  foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
475  if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
476  unlink( $cacheFile );
477  }
478  }
479  }
480 
485  private function validateLicense( $moduleName, $info ) {
486  if ( !isset( $info['license'] ) || !is_string( $info['license'] ) ) {
487  throw new Exception( "Module '$moduleName' needs a valid SPDX license; no license is currently present" );
488  }
489  $licenses = new SpdxLicenses();
490  if ( !$licenses->validate( $info['license'] ) ) {
491  $this->error(
492  "Module '$moduleName' has an invalid SPDX license identifier '{$info['license']}', "
493  . 'see <https://spdx.org/licenses/>'
494  );
495  }
496  }
497 }
498 
500 class_alias( ForeignResourceManager::class, 'ForeignResourceManager' );
wfTempDir()
Tries to get the system directory for temporary files.
wfRecursiveRemoveDir( $dir)
Remove a directory and all its content.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
$modules
A class containing constants representing the names of configuration variables.
const CacheDirectory
Name constant for the CacheDirectory setting, for use with Config::get()
Service locator for MediaWiki core services.
static getInstance()
Returns the global default instance of the top level service locator.
Manage foreign resources registered with ResourceLoader.
__construct( $registryFile, $libDir, callable $infoPrinter=null, callable $errorPrinter=null, callable $verbosePrinter=null)
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42