MediaWiki  master
ForeignResourceManager.php
Go to the documentation of this file.
1 <?php
23 use Wikimedia\AtEase\AtEase;
24 
31  private $defaultAlgo = 'sha384';
32  private $hasErrors = false;
33  private $registryFile;
34  private $libDir;
35  private $tmpParentDir;
36  private $cacheDir;
41  private $infoPrinter;
46  private $errorPrinter;
51  private $verbosePrinter;
52  private $action;
54  private $registry;
55 
64  public function __construct(
66  $libDir,
67  callable $infoPrinter = null,
68  callable $errorPrinter = null,
69  callable $verbosePrinter = null
70  ) {
71  $this->registryFile = $registryFile;
72  $this->libDir = $libDir;
73  $this->infoPrinter = $infoPrinter ?? function ( $_ ) {
74  };
75  $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
76  $this->verbosePrinter = $verbosePrinter ?? function ( $_ ) {
77  };
78 
79  // Use a temporary directory under the destination directory instead
80  // of wfTempDir() because PHP's rename() does not work across file
81  // systems, and the user's /tmp and $IP may be on different filesystems.
82  $this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
83 
84  $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
85  $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
86  }
87 
94  public function run( $action, $module ) {
95  $actions = [ 'update', 'verify', 'make-sri' ];
96  if ( !in_array( $action, $actions ) ) {
97  $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
98  return false;
99  }
100  $this->action = $action;
101 
102  $this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
103  if ( $module === 'all' ) {
105  } elseif ( isset( $this->registry[ $module ] ) ) {
106  $modules = [ $module => $this->registry[ $module ] ];
107  } else {
108  $this->error( "Unknown module name.\n\nMust be one of:\n" .
109  wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
110  '.'
111  );
112  return false;
113  }
114 
115  foreach ( $modules as $moduleName => $info ) {
116  $this->verbose( "\n### {$moduleName}\n\n" );
117  $destDir = "{$this->libDir}/$moduleName";
118 
119  if ( $this->action === 'update' ) {
120  $this->output( "... updating '{$moduleName}'\n" );
121  $this->verbose( "... emptying directory for $moduleName\n" );
122  wfRecursiveRemoveDir( $destDir );
123  } elseif ( $this->action === 'verify' ) {
124  $this->output( "... verifying '{$moduleName}'\n" );
125  } else {
126  $this->output( "... checking '{$moduleName}'\n" );
127  }
128 
129  $this->verbose( "... preparing {$this->tmpParentDir}\n" );
130  wfRecursiveRemoveDir( $this->tmpParentDir );
131  if ( !wfMkdirParents( $this->tmpParentDir ) ) {
132  throw new Exception( "Unable to create {$this->tmpParentDir}" );
133  }
134 
135  if ( !isset( $info['type'] ) ) {
136  throw new Exception( "Module '$moduleName' must have a 'type' key." );
137  }
138  switch ( $info['type'] ) {
139  case 'tar':
140  $this->handleTypeTar( $moduleName, $destDir, $info );
141  break;
142  case 'file':
143  $this->handleTypeFile( $moduleName, $destDir, $info );
144  break;
145  case 'multi-file':
146  $this->handleTypeMultiFile( $moduleName, $destDir, $info );
147  break;
148  default:
149  throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
150  }
151  }
152 
153  $this->output( "\nDone!\n" );
154  $this->cleanUp();
155  if ( $this->hasErrors ) {
156  // The verify mode should check all modules/files and fail after, not during.
157  return false;
158  }
159 
160  return true;
161  }
162 
163  private function cacheKey( $src, $integrity ) {
164  $key = basename( $src ) . '_' . substr( $integrity, -12 );
165  $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
166  return rtrim( $key, '_' );
167  }
168 
173  private function cacheGet( $key ) {
174  return AtEase::quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
175  }
176 
177  private function cacheSet( $key, $data ) {
178  wfMkdirParents( $this->cacheDir );
179  file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
180  }
181 
182  private function fetch( $src, $integrity ) {
183  $key = $this->cacheKey( $src, $integrity );
184  $data = $this->cacheGet( $key );
185  if ( $data ) {
186  return $data;
187  }
188 
189  $req = MediaWikiServices::getInstance()->getHttpRequestFactory()
190  ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ );
191  if ( !$req->execute()->isOK() ) {
192  throw new Exception( "Failed to download resource at {$src}" );
193  }
194  if ( $req->getStatus() !== 200 ) {
195  throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
196  }
197  $data = $req->getContent();
198  $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
199  $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
200  if ( $integrity === $actualIntegrity ) {
201  $this->verbose( "... passed integrity check for {$src}\n" );
202  $this->cacheSet( $key, $data );
203  } elseif ( $this->action === 'make-sri' ) {
204  $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
205  } else {
206  throw new Exception( "Integrity check failed for {$src}\n" .
207  "\tExpected: {$integrity}\n" .
208  "\tActual: {$actualIntegrity}"
209  );
210  }
211  return $data;
212  }
213 
214  private function handleTypeFile( $moduleName, $destDir, array $info ) {
215  if ( !isset( $info['src'] ) ) {
216  throw new Exception( "Module '$moduleName' must have a 'src' key." );
217  }
218  $data = $this->fetch( $info['src'], $info['integrity'] ?? null );
219  $dest = $info['dest'] ?? basename( $info['src'] );
220  $path = "$destDir/$dest";
221  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
222  throw new Exception( "File for '$moduleName' is different." );
223  }
224  if ( $this->action === 'update' ) {
225  wfMkdirParents( $destDir );
226  file_put_contents( "$destDir/$dest", $data );
227  }
228  }
229 
230  private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
231  if ( !isset( $info['files'] ) ) {
232  throw new Exception( "Module '$moduleName' must have a 'files' key." );
233  }
234  foreach ( $info['files'] as $dest => $file ) {
235  if ( !isset( $file['src'] ) ) {
236  throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
237  }
238  $data = $this->fetch( $file['src'], $file['integrity'] ?? null );
239  $path = "$destDir/$dest";
240  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
241  throw new Exception( "File '$dest' for '$moduleName' is different." );
242  } elseif ( $this->action === 'update' ) {
243  wfMkdirParents( $destDir );
244  file_put_contents( "$destDir/$dest", $data );
245  }
246  }
247  }
248 
249  private function handleTypeTar( $moduleName, $destDir, array $info ) {
250  $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
251  if ( $info['src'] === null ) {
252  throw new Exception( "Module '$moduleName' must have a 'src' key." );
253  }
254  // Download the resource to a temporary file and open it
255  $data = $this->fetch( $info['src'], $info['integrity' ] );
256  $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
257  $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
258  file_put_contents( $tmpFile, $data );
259  $p = new PharData( $tmpFile );
260  $tmpDir = "{$this->tmpParentDir}/$moduleName";
261  $p->extractTo( $tmpDir );
262  unset( $data, $p );
263 
264  if ( $info['dest'] === null ) {
265  // Default: Replace the entire directory
266  $toCopy = [ $tmpDir => $destDir ];
267  } else {
268  // Expand and normalise the 'dest' entries
269  $toCopy = [];
270  foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
271  // Use glob() to expand wildcards and check existence
272  $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
273  if ( !$fromPaths ) {
274  throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
275  }
276  foreach ( $fromPaths as $fromPath ) {
277  $toCopy[$fromPath] = $toSubPath === null
278  ? "$destDir/" . basename( $fromPath )
279  : "$destDir/$toSubPath/" . basename( $fromPath );
280  }
281  }
282  }
283  foreach ( $toCopy as $from => $to ) {
284  if ( $this->action === 'verify' ) {
285  $this->verbose( "... verifying $to\n" );
286  if ( is_dir( $from ) ) {
287  $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
288  $from,
289  RecursiveDirectoryIterator::SKIP_DOTS
290  ) );
292  foreach ( $rii as $file ) {
293  $remote = $file->getPathname();
294  $local = strtr( $remote, [ $from => $to ] );
295  if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
296  $this->error( "File '$local' is different." );
297  $this->hasErrors = true;
298  }
299  }
300  } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
301  $this->error( "File '$to' is different." );
302  $this->hasErrors = true;
303  }
304  } elseif ( $this->action === 'update' ) {
305  $this->verbose( "... moving $from to $to\n" );
306  wfMkdirParents( dirname( $to ) );
307  if ( !rename( $from, $to ) ) {
308  throw new Exception( "Could not move $from to $to." );
309  }
310  }
311  }
312  }
313 
314  private function verbose( $text ) {
315  ( $this->verbosePrinter )( $text );
316  }
317 
318  private function output( $text ) {
319  ( $this->infoPrinter )( $text );
320  }
321 
322  private function error( $text ) {
323  ( $this->errorPrinter )( $text );
324  }
325 
326  private function cleanUp() {
327  wfRecursiveRemoveDir( $this->tmpParentDir );
328 
329  // Prune the cache of files we don't recognise.
330  $knownKeys = [];
331  foreach ( $this->registry as $info ) {
332  if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
333  $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
334  } elseif ( $info['type'] === 'multi-file' ) {
335  foreach ( $info['files'] as $file ) {
336  $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
337  }
338  }
339  }
340  foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
341  if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
342  unlink( $cacheFile );
343  }
344  }
345  }
346 
356  private function parseBasicYaml( $input ) {
357  $lines = explode( "\n", $input );
358  $root = [];
359  $stack = [ &$root ];
360  $prev = 0;
361  foreach ( $lines as $i => $text ) {
362  $line = $i + 1;
363  $trimmed = ltrim( $text, ' ' );
364  if ( $trimmed === '' || $trimmed[0] === '#' ) {
365  continue;
366  }
367  $indent = strlen( $text ) - strlen( $trimmed );
368  if ( $indent % 2 !== 0 ) {
369  throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
370  }
371  $depth = $indent === 0 ? 0 : ( $indent / 2 );
372  if ( $depth < $prev ) {
373  // Close previous branches we can't re-enter
374  array_splice( $stack, $depth + 1 );
375  }
376  if ( !array_key_exists( $depth, $stack ) ) {
377  throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
378  }
379  if ( strpos( $trimmed, ':' ) === false ) {
380  throw new Exception( __METHOD__ . ": Missing colon on line $line." );
381  }
382  $dest =& $stack[ $depth ];
383  if ( $dest === null ) {
384  // Promote from null to object
385  $dest = [];
386  }
387  list( $key, $val ) = explode( ':', $trimmed, 2 );
388  $val = ltrim( $val, ' ' );
389  if ( $val !== '' ) {
390  // Add string
391  $dest[ $key ] = $val;
392  } else {
393  // Add null (may become an object later)
394  $val = null;
395  $stack[] = &$val;
396  $dest[ $key ] = &$val;
397  }
398  $prev = $depth;
399  unset( $dest, $val );
400  }
401  return $root;
402  }
403 }
ForeignResourceManager\$defaultAlgo
$defaultAlgo
Definition: ForeignResourceManager.php:31
ForeignResourceManager\__construct
__construct( $registryFile, $libDir, callable $infoPrinter=null, callable $errorPrinter=null, callable $verbosePrinter=null)
Definition: ForeignResourceManager.php:64
wfMkdirParents
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
Definition: GlobalFunctions.php:1890
ForeignResourceManager\verbose
verbose( $text)
Definition: ForeignResourceManager.php:314
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
ForeignResourceManager\cacheSet
cacheSet( $key, $data)
Definition: ForeignResourceManager.php:177
ForeignResourceManager\$libDir
$libDir
Definition: ForeignResourceManager.php:34
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
ForeignResourceManager\$errorPrinter
callable Closure $errorPrinter
-var callable(string):void
Definition: ForeignResourceManager.php:46
ForeignResourceManager\$action
$action
Definition: ForeignResourceManager.php:52
ForeignResourceManager\$cacheDir
$cacheDir
Definition: ForeignResourceManager.php:36
ForeignResourceManager\$hasErrors
$hasErrors
Definition: ForeignResourceManager.php:32
$modules
$modules
Definition: HTMLFormElement.php:15
ForeignResourceManager\cleanUp
cleanUp()
Definition: ForeignResourceManager.php:326
ForeignResourceManager
Manage foreign resources registered with ResourceLoader.
Definition: ForeignResourceManager.php:30
ForeignResourceManager\$verbosePrinter
callable Closure $verbosePrinter
-var callable(string):void
Definition: ForeignResourceManager.php:51
ForeignResourceManager\fetch
fetch( $src, $integrity)
Definition: ForeignResourceManager.php:182
ForeignResourceManager\run
run( $action, $module)
Definition: ForeignResourceManager.php:94
ForeignResourceManager\handleTypeFile
handleTypeFile( $moduleName, $destDir, array $info)
Definition: ForeignResourceManager.php:214
$line
$line
Definition: mcc.php:119
ForeignResourceManager\$infoPrinter
callable Closure $infoPrinter
-var callable(string):void
Definition: ForeignResourceManager.php:41
ForeignResourceManager\$registryFile
$registryFile
Definition: ForeignResourceManager.php:33
$lines
if(!file_exists( $CREDITS)) $lines
Definition: updateCredits.php:49
ForeignResourceManager\$registry
array[] $registry
Definition: ForeignResourceManager.php:54
wfRecursiveRemoveDir
wfRecursiveRemoveDir( $dir)
Remove a directory and all its content.
Definition: GlobalFunctions.php:1933
ForeignResourceManager\output
output( $text)
Definition: ForeignResourceManager.php:318
$path
$path
Definition: NoLocalSettings.php:25
ForeignResourceManager\cacheKey
cacheKey( $src, $integrity)
Definition: ForeignResourceManager.php:163
ForeignResourceManager\parseBasicYaml
parseBasicYaml( $input)
Basic YAML parser.
Definition: ForeignResourceManager.php:356
ForeignResourceManager\cacheGet
cacheGet( $key)
Definition: ForeignResourceManager.php:173
ForeignResourceManager\handleTypeMultiFile
handleTypeMultiFile( $moduleName, $destDir, array $info)
Definition: ForeignResourceManager.php:230
ForeignResourceManager\error
error( $text)
Definition: ForeignResourceManager.php:322
ForeignResourceManager\handleTypeTar
handleTypeTar( $moduleName, $destDir, array $info)
Definition: ForeignResourceManager.php:249
ForeignResourceManager\$tmpParentDir
$tmpParentDir
Definition: ForeignResourceManager.php:35