MediaWiki  master
ForeignResourceManager.php
Go to the documentation of this file.
1 <?php
21 namespace MediaWiki\ResourceLoader;
22 
23 use Exception;
25 use PharData;
26 use RecursiveDirectoryIterator;
27 use RecursiveIteratorIterator;
28 use Symfony\Component\Yaml\Yaml;
29 
38  private $defaultAlgo = 'sha384';
39 
41  private $hasErrors = false;
42 
44  private $registryFile;
45 
47  private $libDir;
48 
50  private $tmpParentDir;
51 
53  private $cacheDir;
54 
59  private $infoPrinter;
60 
65  private $errorPrinter;
70  private $verbosePrinter;
71 
73  private $action;
74 
76  private $registry;
77 
86  public function __construct(
87  $registryFile,
88  $libDir,
89  callable $infoPrinter = null,
90  callable $errorPrinter = null,
91  callable $verbosePrinter = null
92  ) {
93  $this->registryFile = $registryFile;
94  $this->libDir = $libDir;
95  $this->infoPrinter = $infoPrinter ?? static function ( $_ ) {
96  };
97  $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
98  $this->verbosePrinter = $verbosePrinter ?? static function ( $_ ) {
99  };
100 
101  // Use a temporary directory under the destination directory instead
102  // of wfTempDir() because PHP's rename() does not work across file
103  // systems, and the user's /tmp and $IP may be on different filesystems.
104  $this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
105 
106  // Support XDG_CACHE_HOME to speed up CI by avoiding repeated downloads.
107  $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
108  $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
109  }
110 
117  public function run( $action, $module ) {
118  $actions = [ 'update', 'verify', 'make-sri' ];
119  if ( !in_array( $action, $actions ) ) {
120  $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
121  return false;
122  }
123  $this->action = $action;
124 
125  $this->registry = Yaml::parseFile( $this->registryFile );
126  if ( $module === 'all' ) {
127  $modules = $this->registry;
128  } elseif ( isset( $this->registry[$module] ) ) {
129  $modules = [ $module => $this->registry[$module] ];
130  } else {
131  $this->error( "Unknown module name.\n\nMust be one of:\n" .
132  wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
133  '.'
134  );
135  return false;
136  }
137 
138  foreach ( $modules as $moduleName => $info ) {
139  $this->verbose( "\n### {$moduleName}\n\n" );
140  $destDir = "{$this->libDir}/$moduleName";
141 
142  if ( $this->action === 'update' ) {
143  $this->output( "... updating '{$moduleName}'\n" );
144  $this->verbose( "... emptying directory for $moduleName\n" );
145  wfRecursiveRemoveDir( $destDir );
146  } elseif ( $this->action === 'verify' ) {
147  $this->output( "... verifying '{$moduleName}'\n" );
148  } else {
149  $this->output( "... checking '{$moduleName}'\n" );
150  }
151 
152  $this->verbose( "... preparing {$this->tmpParentDir}\n" );
153  wfRecursiveRemoveDir( $this->tmpParentDir );
154  if ( !wfMkdirParents( $this->tmpParentDir ) ) {
155  throw new Exception( "Unable to create {$this->tmpParentDir}" );
156  }
157 
158  if ( !isset( $info['type'] ) ) {
159  throw new Exception( "Module '$moduleName' must have a 'type' key." );
160  }
161  switch ( $info['type'] ) {
162  case 'tar':
163  $this->handleTypeTar( $moduleName, $destDir, $info );
164  break;
165  case 'file':
166  $this->handleTypeFile( $moduleName, $destDir, $info );
167  break;
168  case 'multi-file':
169  $this->handleTypeMultiFile( $moduleName, $destDir, $info );
170  break;
171  default:
172  throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
173  }
174  }
175 
176  $this->cleanUp();
177  if ( $this->hasErrors ) {
178  // The "verify" action should check all modules and files and fail after, not during.
179  // We don't throw on the first issue so that developers enjoy access to all actionable
180  // information at once (given we can't have cascading errors).
181  // The "verify" action prints errors along the way and simply exits here.
182  return false;
183  }
184 
185  return true;
186  }
187 
194  private function cacheKey( $src, $integrity, $moduleName ) {
195  $key = $moduleName . '_' . hash( 'fnv132', $integrity ) . '_' . basename( $src );
196  $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
197  return rtrim( $key, '_' );
198  }
199 
204  private function cacheGet( $key ) {
205  // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
206  return @file_get_contents( "{$this->cacheDir}/$key.data" );
207  }
208 
213  private function cacheSet( $key, $data ) {
214  // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
215  @mkdir( $this->cacheDir, 0777, true );
216  file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
217  }
218 
225  private function fetch( string $src, $integrity, string $moduleName ) {
226  if ( $integrity !== null ) {
227  $key = $this->cacheKey( $src, $integrity, $moduleName );
228  $data = $this->cacheGet( $key );
229  if ( $data ) {
230  return $data;
231  }
232  }
233 
234  $req = MediaWikiServices::getInstance()->getHttpRequestFactory()
235  ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ );
236  if ( !$req->execute()->isOK() ) {
237  throw new Exception( "Failed to download resource at {$src}" );
238  }
239  if ( $req->getStatus() !== 200 ) {
240  throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
241  }
242  $data = $req->getContent();
243  $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
244  $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
245  if ( $integrity === $actualIntegrity ) {
246  $this->verbose( "... passed integrity check for {$src}\n" );
247  $key = $this->cacheKey( $src, $actualIntegrity, $moduleName );
248  $this->cacheSet( $key, $data );
249  } elseif ( $this->action === 'make-sri' ) {
250  $this->output( "Integrity for {$src}\n\tintegrity: {$actualIntegrity}\n" );
251  } else {
252  $expectedIntegrity = $integrity ?? 'null';
253  throw new Exception( "Integrity check failed for {$src}\n" .
254  "\tExpected: {$expectedIntegrity}\n" .
255  "\tActual: {$actualIntegrity}"
256  );
257  }
258  return $data;
259  }
260 
266  private function handleTypeFile( $moduleName, $destDir, array $info ) {
267  if ( !isset( $info['src'] ) ) {
268  throw new Exception( "Module '$moduleName' must have a 'src' key." );
269  }
270  $data = $this->fetch( $info['src'], $info['integrity'] ?? null, $moduleName );
271  $dest = $info['dest'] ?? basename( $info['src'] );
272  $path = "$destDir/$dest";
273  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
274  $this->error( "File for '$moduleName' is different.\n" );
275  }
276  if ( $this->action === 'update' ) {
277  wfMkdirParents( $destDir );
278  file_put_contents( "$destDir/$dest", $data );
279  }
280  }
281 
287  private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
288  if ( !isset( $info['files'] ) ) {
289  throw new Exception( "Module '$moduleName' must have a 'files' key." );
290  }
291  foreach ( $info['files'] as $dest => $file ) {
292  if ( !isset( $file['src'] ) ) {
293  throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
294  }
295  $data = $this->fetch( $file['src'], $file['integrity'] ?? null, $moduleName );
296  $path = "$destDir/$dest";
297  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
298  $this->error( "File '$dest' for '$moduleName' is different.\n" );
299  } elseif ( $this->action === 'update' ) {
300  wfMkdirParents( $destDir );
301  file_put_contents( "$destDir/$dest", $data );
302  }
303  }
304  }
305 
311  private function handleTypeTar( $moduleName, $destDir, array $info ) {
312  $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
313  if ( $info['src'] === null ) {
314  throw new Exception( "Module '$moduleName' must have a 'src' key." );
315  }
316  // Download the resource to a temporary file and open it
317  $data = $this->fetch( $info['src'], $info['integrity'], $moduleName );
318  $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
319  $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
320  file_put_contents( $tmpFile, $data );
321  $p = new PharData( $tmpFile );
322  $tmpDir = "{$this->tmpParentDir}/$moduleName";
323  $p->extractTo( $tmpDir );
324  unset( $data, $p );
325 
326  if ( $info['dest'] === null ) {
327  // Default: Replace the entire directory
328  $toCopy = [ $tmpDir => $destDir ];
329  } else {
330  // Expand and normalise the 'dest' entries
331  $toCopy = [];
332  foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
333  // Use glob() to expand wildcards and check existence
334  $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
335  if ( !$fromPaths ) {
336  throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
337  }
338  foreach ( $fromPaths as $fromPath ) {
339  $toCopy[$fromPath] = $toSubPath === null
340  ? "$destDir/" . basename( $fromPath )
341  : "$destDir/$toSubPath/" . basename( $fromPath );
342  }
343  }
344  }
345  foreach ( $toCopy as $from => $to ) {
346  if ( $this->action === 'verify' ) {
347  $this->verbose( "... verifying $to\n" );
348  if ( is_dir( $from ) ) {
349  $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
350  $from,
351  RecursiveDirectoryIterator::SKIP_DOTS
352  ) );
354  foreach ( $rii as $file ) {
355  $remote = $file->getPathname();
356  $local = strtr( $remote, [ $from => $to ] );
357  if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
358  $this->error( "File '$local' is different.\n" );
359  }
360  }
361  } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
362  $this->error( "File '$to' is different.\n" );
363  }
364  } elseif ( $this->action === 'update' ) {
365  $this->verbose( "... moving $from to $to\n" );
366  wfMkdirParents( dirname( $to ) );
367  if ( !rename( $from, $to ) ) {
368  throw new Exception( "Could not move $from to $to." );
369  }
370  }
371  }
372  }
373 
377  private function verbose( $text ) {
378  ( $this->verbosePrinter )( $text );
379  }
380 
384  private function output( $text ) {
385  ( $this->infoPrinter )( $text );
386  }
387 
391  private function error( $text ) {
392  $this->hasErrors = true;
393  ( $this->errorPrinter )( $text );
394  }
395 
396  private function cleanUp() {
397  wfRecursiveRemoveDir( $this->tmpParentDir );
398 
399  // Prune the cache of files we don't recognise.
400  $knownKeys = [];
401  foreach ( $this->registry as $module => $info ) {
402  if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
403  $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'], $module );
404  } elseif ( $info['type'] === 'multi-file' ) {
405  foreach ( $info['files'] as $file ) {
406  $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'], $module );
407  }
408  }
409  }
410  foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
411  if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
412  unlink( $cacheFile );
413  }
414  }
415  }
416 }
418 class_alias( ForeignResourceManager::class, 'ForeignResourceManager' );
wfRecursiveRemoveDir( $dir)
Remove a directory and all its content.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
$modules
Service locator for MediaWiki core services.
static getInstance()
Returns the global default instance of the top level service locator.
Manage foreign resources registered with ResourceLoader.
__construct( $registryFile, $libDir, callable $infoPrinter=null, callable $errorPrinter=null, callable $verbosePrinter=null)
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42