MediaWiki  master
ForeignResourceManager.php
Go to the documentation of this file.
1 <?php
23 use Wikimedia\AtEase\AtEase;
24 
31  private $defaultAlgo = 'sha384';
32  private $hasErrors = false;
33  private $registryFile;
34  private $libDir;
35  private $tmpParentDir;
36  private $cacheDir;
41  private $infoPrinter;
46  private $errorPrinter;
51  private $verbosePrinter;
52  private $action;
54  private $registry;
55 
64  public function __construct(
66  $libDir,
67  callable $infoPrinter = null,
68  callable $errorPrinter = null,
69  callable $verbosePrinter = null
70  ) {
71  $this->registryFile = $registryFile;
72  $this->libDir = $libDir;
73  $this->infoPrinter = $infoPrinter ?? static function ( $_ ) {
74  };
75  $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
76  $this->verbosePrinter = $verbosePrinter ?? static function ( $_ ) {
77  };
78 
79  // Use a temporary directory under the destination directory instead
80  // of wfTempDir() because PHP's rename() does not work across file
81  // systems, and the user's /tmp and $IP may be on different filesystems.
82  $this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
83 
84  $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
85  $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
86  }
87 
94  public function run( $action, $module ) {
95  $actions = [ 'update', 'verify', 'make-sri' ];
96  if ( !in_array( $action, $actions ) ) {
97  $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
98  return false;
99  }
100  $this->action = $action;
101 
102  $this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
103  if ( $module === 'all' ) {
105  } elseif ( isset( $this->registry[ $module ] ) ) {
106  $modules = [ $module => $this->registry[ $module ] ];
107  } else {
108  $this->error( "Unknown module name.\n\nMust be one of:\n" .
109  wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
110  '.'
111  );
112  return false;
113  }
114 
115  foreach ( $modules as $moduleName => $info ) {
116  $this->verbose( "\n### {$moduleName}\n\n" );
117  $destDir = "{$this->libDir}/$moduleName";
118 
119  if ( $this->action === 'update' ) {
120  $this->output( "... updating '{$moduleName}'\n" );
121  $this->verbose( "... emptying directory for $moduleName\n" );
122  wfRecursiveRemoveDir( $destDir );
123  } elseif ( $this->action === 'verify' ) {
124  $this->output( "... verifying '{$moduleName}'\n" );
125  } else {
126  $this->output( "... checking '{$moduleName}'\n" );
127  }
128 
129  $this->verbose( "... preparing {$this->tmpParentDir}\n" );
130  wfRecursiveRemoveDir( $this->tmpParentDir );
131  if ( !wfMkdirParents( $this->tmpParentDir ) ) {
132  throw new Exception( "Unable to create {$this->tmpParentDir}" );
133  }
134 
135  if ( !isset( $info['type'] ) ) {
136  throw new Exception( "Module '$moduleName' must have a 'type' key." );
137  }
138  switch ( $info['type'] ) {
139  case 'tar':
140  $this->handleTypeTar( $moduleName, $destDir, $info );
141  break;
142  case 'file':
143  $this->handleTypeFile( $moduleName, $destDir, $info );
144  break;
145  case 'multi-file':
146  $this->handleTypeMultiFile( $moduleName, $destDir, $info );
147  break;
148  default:
149  throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
150  }
151  }
152 
153  $this->output( "\nDone!\n" );
154  $this->cleanUp();
155  if ( $this->hasErrors ) {
156  // The verify mode should check all modules/files and fail after, not during.
157  return false;
158  }
159 
160  return true;
161  }
162 
163  private function cacheKey( $src, $integrity ) {
164  $key = basename( $src ) . '_' . substr( $integrity, -12 );
165  $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
166  return rtrim( $key, '_' );
167  }
168 
173  private function cacheGet( $key ) {
174  return AtEase::quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
175  }
176 
177  private function cacheSet( $key, $data ) {
178  wfMkdirParents( $this->cacheDir );
179  file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
180  }
181 
182  private function fetch( $src, $integrity ) {
183  $key = $this->cacheKey( $src, $integrity );
184  $data = $this->cacheGet( $key );
185  if ( $data ) {
186  return $data;
187  }
188 
189  $req = MediaWikiServices::getInstance()->getHttpRequestFactory()
190  ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ );
191  if ( !$req->execute()->isOK() ) {
192  throw new Exception( "Failed to download resource at {$src}" );
193  }
194  if ( $req->getStatus() !== 200 ) {
195  throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
196  }
197  $data = $req->getContent();
198  $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
199  $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
200  if ( $integrity === $actualIntegrity ) {
201  $this->verbose( "... passed integrity check for {$src}\n" );
202  $this->cacheSet( $key, $data );
203  } elseif ( $this->action === 'make-sri' ) {
204  $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
205  } else {
206  $expectedIntegrity = $integrity ?? 'null';
207  throw new Exception( "Integrity check failed for {$src}\n" .
208  "\tExpected: {$expectedIntegrity}\n" .
209  "\tActual: {$actualIntegrity}"
210  );
211  }
212  return $data;
213  }
214 
215  private function handleTypeFile( $moduleName, $destDir, array $info ) {
216  if ( !isset( $info['src'] ) ) {
217  throw new Exception( "Module '$moduleName' must have a 'src' key." );
218  }
219  $data = $this->fetch( $info['src'], $info['integrity'] ?? null );
220  $dest = $info['dest'] ?? basename( $info['src'] );
221  $path = "$destDir/$dest";
222  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
223  throw new Exception( "File for '$moduleName' is different." );
224  }
225  if ( $this->action === 'update' ) {
226  wfMkdirParents( $destDir );
227  file_put_contents( "$destDir/$dest", $data );
228  }
229  }
230 
231  private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
232  if ( !isset( $info['files'] ) ) {
233  throw new Exception( "Module '$moduleName' must have a 'files' key." );
234  }
235  foreach ( $info['files'] as $dest => $file ) {
236  if ( !isset( $file['src'] ) ) {
237  throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
238  }
239  $data = $this->fetch( $file['src'], $file['integrity'] ?? null );
240  $path = "$destDir/$dest";
241  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
242  throw new Exception( "File '$dest' for '$moduleName' is different." );
243  } elseif ( $this->action === 'update' ) {
244  wfMkdirParents( $destDir );
245  file_put_contents( "$destDir/$dest", $data );
246  }
247  }
248  }
249 
250  private function handleTypeTar( $moduleName, $destDir, array $info ) {
251  $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
252  if ( $info['src'] === null ) {
253  throw new Exception( "Module '$moduleName' must have a 'src' key." );
254  }
255  // Download the resource to a temporary file and open it
256  $data = $this->fetch( $info['src'], $info['integrity' ] );
257  $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
258  $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
259  file_put_contents( $tmpFile, $data );
260  $p = new PharData( $tmpFile );
261  $tmpDir = "{$this->tmpParentDir}/$moduleName";
262  $p->extractTo( $tmpDir );
263  unset( $data, $p );
264 
265  if ( $info['dest'] === null ) {
266  // Default: Replace the entire directory
267  $toCopy = [ $tmpDir => $destDir ];
268  } else {
269  // Expand and normalise the 'dest' entries
270  $toCopy = [];
271  foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
272  // Use glob() to expand wildcards and check existence
273  $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
274  if ( !$fromPaths ) {
275  throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
276  }
277  foreach ( $fromPaths as $fromPath ) {
278  $toCopy[$fromPath] = $toSubPath === null
279  ? "$destDir/" . basename( $fromPath )
280  : "$destDir/$toSubPath/" . basename( $fromPath );
281  }
282  }
283  }
284  foreach ( $toCopy as $from => $to ) {
285  if ( $this->action === 'verify' ) {
286  $this->verbose( "... verifying $to\n" );
287  if ( is_dir( $from ) ) {
288  $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
289  $from,
290  RecursiveDirectoryIterator::SKIP_DOTS
291  ) );
293  foreach ( $rii as $file ) {
294  $remote = $file->getPathname();
295  $local = strtr( $remote, [ $from => $to ] );
296  if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
297  $this->error( "File '$local' is different." );
298  $this->hasErrors = true;
299  }
300  }
301  } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
302  $this->error( "File '$to' is different." );
303  $this->hasErrors = true;
304  }
305  } elseif ( $this->action === 'update' ) {
306  $this->verbose( "... moving $from to $to\n" );
307  wfMkdirParents( dirname( $to ) );
308  if ( !rename( $from, $to ) ) {
309  throw new Exception( "Could not move $from to $to." );
310  }
311  }
312  }
313  }
314 
315  private function verbose( $text ) {
316  ( $this->verbosePrinter )( $text );
317  }
318 
319  private function output( $text ) {
320  ( $this->infoPrinter )( $text );
321  }
322 
323  private function error( $text ) {
324  ( $this->errorPrinter )( $text );
325  }
326 
327  private function cleanUp() {
328  wfRecursiveRemoveDir( $this->tmpParentDir );
329 
330  // Prune the cache of files we don't recognise.
331  $knownKeys = [];
332  foreach ( $this->registry as $info ) {
333  if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
334  $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
335  } elseif ( $info['type'] === 'multi-file' ) {
336  foreach ( $info['files'] as $file ) {
337  $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
338  }
339  }
340  }
341  foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
342  if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
343  unlink( $cacheFile );
344  }
345  }
346  }
347 
357  private function parseBasicYaml( $input ) {
358  $lines = explode( "\n", $input );
359  $root = [];
360  $stack = [ &$root ];
361  $prev = 0;
362  foreach ( $lines as $i => $text ) {
363  $line = $i + 1;
364  $trimmed = ltrim( $text, ' ' );
365  if ( $trimmed === '' || $trimmed[0] === '#' ) {
366  continue;
367  }
368  $indent = strlen( $text ) - strlen( $trimmed );
369  if ( $indent % 2 !== 0 ) {
370  throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
371  }
372  $depth = $indent === 0 ? 0 : ( $indent / 2 );
373  if ( $depth < $prev ) {
374  // Close previous branches we can't re-enter
375  array_splice( $stack, $depth + 1 );
376  }
377  if ( !array_key_exists( $depth, $stack ) ) {
378  throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
379  }
380  if ( strpos( $trimmed, ':' ) === false ) {
381  throw new Exception( __METHOD__ . ": Missing colon on line $line." );
382  }
383  $dest =& $stack[ $depth ];
384  if ( $dest === null ) {
385  // Promote from null to object
386  $dest = [];
387  }
388  list( $key, $val ) = explode( ':', $trimmed, 2 );
389  $val = ltrim( $val, ' ' );
390  if ( $val !== '' ) {
391  // Add string
392  $dest[ $key ] = $val;
393  } else {
394  // Add null (may become an object later)
395  $val = null;
396  $stack[] = &$val;
397  $dest[ $key ] = &$val;
398  }
399  $prev = $depth;
400  unset( $dest, $val );
401  }
402  return $root;
403  }
404 }
wfRecursiveRemoveDir( $dir)
Remove a directory and all its content.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
$modules
Manage foreign resources registered with ResourceLoader.
parseBasicYaml( $input)
Basic YAML parser.
handleTypeTar( $moduleName, $destDir, array $info)
handleTypeMultiFile( $moduleName, $destDir, array $info)
__construct( $registryFile, $libDir, callable $infoPrinter=null, callable $errorPrinter=null, callable $verbosePrinter=null)
handleTypeFile( $moduleName, $destDir, array $info)
MediaWikiServices is the service locator for the application scope of MediaWiki.
$line
Definition: mcc.php:119
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
if(!file_exists( $CREDITS)) $lines