MediaWiki  master
ForeignResourceManager.php
Go to the documentation of this file.
1 <?php
28  private $defaultAlgo = 'sha384';
29  private $hasErrors = false;
30  private $registryFile;
31  private $libDir;
32  private $tmpParentDir;
33  private $cacheDir;
34  private $infoPrinter;
35  private $errorPrinter;
36  private $verbosePrinter;
37  private $action;
38  private $registry;
39 
48  public function __construct(
50  $libDir,
51  callable $infoPrinter = null,
52  callable $errorPrinter = null,
53  callable $verbosePrinter = null
54  ) {
55  $this->registryFile = $registryFile;
56  $this->libDir = $libDir;
57  $this->infoPrinter = $infoPrinter ?? function () {
58  };
59  $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
60  $this->verbosePrinter = $verbosePrinter ?? function () {
61  };
62 
63  // Use a temporary directory under the destination directory instead
64  // of wfTempDir() because PHP's rename() does not work across file
65  // systems, and the user's /tmp and $IP may be on different filesystems.
66  $this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
67 
68  $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
69  $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
70  }
71 
76  public function run( $action, $module ) {
77  $actions = [ 'update', 'verify', 'make-sri' ];
78  if ( !in_array( $action, $actions ) ) {
79  $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
80  return false;
81  }
82  $this->action = $action;
83 
84  $this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
85  if ( $module === 'all' ) {
87  } elseif ( isset( $this->registry[ $module ] ) ) {
88  $modules = [ $module => $this->registry[ $module ] ];
89  } else {
90  $this->error( "Unknown module name.\n\nMust be one of:\n" .
91  wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
92  '.'
93  );
94  return false;
95  }
96 
97  foreach ( $modules as $moduleName => $info ) {
98  $this->verbose( "\n### {$moduleName}\n\n" );
99  $destDir = "{$this->libDir}/$moduleName";
100 
101  if ( $this->action === 'update' ) {
102  $this->output( "... updating '{$moduleName}'\n" );
103  $this->verbose( "... emptying directory for $moduleName\n" );
104  wfRecursiveRemoveDir( $destDir );
105  } elseif ( $this->action === 'verify' ) {
106  $this->output( "... verifying '{$moduleName}'\n" );
107  } else {
108  $this->output( "... checking '{$moduleName}'\n" );
109  }
110 
111  $this->verbose( "... preparing {$this->tmpParentDir}\n" );
112  wfRecursiveRemoveDir( $this->tmpParentDir );
113  if ( !wfMkdirParents( $this->tmpParentDir ) ) {
114  throw new Exception( "Unable to create {$this->tmpParentDir}" );
115  }
116 
117  if ( !isset( $info['type'] ) ) {
118  throw new Exception( "Module '$moduleName' must have a 'type' key." );
119  }
120  switch ( $info['type'] ) {
121  case 'tar':
122  $this->handleTypeTar( $moduleName, $destDir, $info );
123  break;
124  case 'file':
125  $this->handleTypeFile( $moduleName, $destDir, $info );
126  break;
127  case 'multi-file':
128  $this->handleTypeMultiFile( $moduleName, $destDir, $info );
129  break;
130  default:
131  throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
132  }
133  }
134 
135  $this->output( "\nDone!\n" );
136  $this->cleanUp();
137  if ( $this->hasErrors ) {
138  // The verify mode should check all modules/files and fail after, not during.
139  return false;
140  }
141 
142  return true;
143  }
144 
145  private function cacheKey( $src, $integrity ) {
146  $key = basename( $src ) . '_' . substr( $integrity, -12 );
147  $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
148  return rtrim( $key, '_' );
149  }
150 
152  private function cacheGet( $key ) {
153  return Wikimedia\quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
154  }
155 
156  private function cacheSet( $key, $data ) {
157  wfMkdirParents( $this->cacheDir );
158  file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
159  }
160 
161  private function fetch( $src, $integrity ) {
162  $key = $this->cacheKey( $src, $integrity );
163  $data = $this->cacheGet( $key );
164  if ( $data ) {
165  return $data;
166  }
167 
168  $req = MWHttpRequest::factory( $src, [ 'method' => 'GET', 'followRedirects' => false ] );
169  if ( !$req->execute()->isOK() ) {
170  throw new Exception( "Failed to download resource at {$src}" );
171  }
172  if ( $req->getStatus() !== 200 ) {
173  throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
174  }
175  $data = $req->getContent();
176  $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
177  $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
178  if ( $integrity === $actualIntegrity ) {
179  $this->verbose( "... passed integrity check for {$src}\n" );
180  $this->cacheSet( $key, $data );
181  } elseif ( $this->action === 'make-sri' ) {
182  $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
183  } else {
184  throw new Exception( "Integrity check failed for {$src}\n" .
185  "\tExpected: {$integrity}\n" .
186  "\tActual: {$actualIntegrity}"
187  );
188  }
189  return $data;
190  }
191 
192  private function handleTypeFile( $moduleName, $destDir, array $info ) {
193  if ( !isset( $info['src'] ) ) {
194  throw new Exception( "Module '$moduleName' must have a 'src' key." );
195  }
196  $data = $this->fetch( $info['src'], $info['integrity'] ?? null );
197  $dest = $info['dest'] ?? basename( $info['src'] );
198  $path = "$destDir/$dest";
199  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
200  throw new Exception( "File for '$moduleName' is different." );
201  }
202  if ( $this->action === 'update' ) {
203  wfMkdirParents( $destDir );
204  file_put_contents( "$destDir/$dest", $data );
205  }
206  }
207 
208  private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
209  if ( !isset( $info['files'] ) ) {
210  throw new Exception( "Module '$moduleName' must have a 'files' key." );
211  }
212  foreach ( $info['files'] as $dest => $file ) {
213  if ( !isset( $file['src'] ) ) {
214  throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
215  }
216  $data = $this->fetch( $file['src'], $file['integrity'] ?? null );
217  $path = "$destDir/$dest";
218  if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
219  throw new Exception( "File '$dest' for '$moduleName' is different." );
220  } elseif ( $this->action === 'update' ) {
221  wfMkdirParents( $destDir );
222  file_put_contents( "$destDir/$dest", $data );
223  }
224  }
225  }
226 
227  private function handleTypeTar( $moduleName, $destDir, array $info ) {
228  $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
229  if ( $info['src'] === null ) {
230  throw new Exception( "Module '$moduleName' must have a 'src' key." );
231  }
232  // Download the resource to a temporary file and open it
233  $data = $this->fetch( $info['src'], $info['integrity' ] );
234  $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
235  $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
236  file_put_contents( $tmpFile, $data );
237  $p = new PharData( $tmpFile );
238  $tmpDir = "{$this->tmpParentDir}/$moduleName";
239  $p->extractTo( $tmpDir );
240  unset( $data, $p );
241 
242  if ( $info['dest'] === null ) {
243  // Default: Replace the entire directory
244  $toCopy = [ $tmpDir => $destDir ];
245  } else {
246  // Expand and normalise the 'dest' entries
247  $toCopy = [];
248  foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
249  // Use glob() to expand wildcards and check existence
250  $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
251  if ( !$fromPaths ) {
252  throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
253  }
254  foreach ( $fromPaths as $fromPath ) {
255  $toCopy[$fromPath] = $toSubPath === null
256  ? "$destDir/" . basename( $fromPath )
257  : "$destDir/$toSubPath/" . basename( $fromPath );
258  }
259  }
260  }
261  foreach ( $toCopy as $from => $to ) {
262  if ( $this->action === 'verify' ) {
263  $this->verbose( "... verifying $to\n" );
264  if ( is_dir( $from ) ) {
265  $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
266  $from,
267  RecursiveDirectoryIterator::SKIP_DOTS
268  ) );
270  foreach ( $rii as $file ) {
271  $remote = $file->getPathname();
272  $local = strtr( $remote, [ $from => $to ] );
273  if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
274  $this->error( "File '$local' is different." );
275  $this->hasErrors = true;
276  }
277  }
278  } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
279  $this->error( "File '$to' is different." );
280  $this->hasErrors = true;
281  }
282  } elseif ( $this->action === 'update' ) {
283  $this->verbose( "... moving $from to $to\n" );
284  wfMkdirParents( dirname( $to ) );
285  if ( !rename( $from, $to ) ) {
286  throw new Exception( "Could not move $from to $to." );
287  }
288  }
289  }
290  }
291 
292  private function verbose( $text ) {
293  ( $this->verbosePrinter )( $text );
294  }
295 
296  private function output( $text ) {
297  ( $this->infoPrinter )( $text );
298  }
299 
300  private function error( $text ) {
301  ( $this->errorPrinter )( $text );
302  }
303 
304  private function cleanUp() {
305  wfRecursiveRemoveDir( $this->tmpParentDir );
306 
307  // Prune the cache of files we don't recognise.
308  $knownKeys = [];
309  foreach ( $this->registry as $info ) {
310  if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
311  $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
312  } elseif ( $info['type'] === 'multi-file' ) {
313  foreach ( $info['files'] as $file ) {
314  $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
315  }
316  }
317  }
318  foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
319  if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
320  unlink( $cacheFile );
321  }
322  }
323  }
324 
334  private function parseBasicYaml( $input ) {
335  $lines = explode( "\n", $input );
336  $root = [];
337  $stack = [ &$root ];
338  $prev = 0;
339  foreach ( $lines as $i => $text ) {
340  $line = $i + 1;
341  $trimmed = ltrim( $text, ' ' );
342  if ( $trimmed === '' || $trimmed[0] === '#' ) {
343  continue;
344  }
345  $indent = strlen( $text ) - strlen( $trimmed );
346  if ( $indent % 2 !== 0 ) {
347  throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
348  }
349  $depth = $indent === 0 ? 0 : ( $indent / 2 );
350  if ( $depth < $prev ) {
351  // Close previous branches we can't re-enter
352  array_splice( $stack, $depth + 1 );
353  }
354  if ( !array_key_exists( $depth, $stack ) ) {
355  throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
356  }
357  if ( strpos( $trimmed, ':' ) === false ) {
358  throw new Exception( __METHOD__ . ": Missing colon on line $line." );
359  }
360  $dest =& $stack[ $depth ];
361  if ( $dest === null ) {
362  // Promote from null to object
363  $dest = [];
364  }
365  list( $key, $val ) = explode( ':', $trimmed, 2 );
366  $val = ltrim( $val, ' ' );
367  if ( $val !== '' ) {
368  // Add string
369  $dest[ $key ] = $val;
370  } else {
371  // Add null (may become an object later)
372  $val = null;
373  $stack[] = &$val;
374  $dest[ $key ] = &$val;
375  }
376  $prev = $depth;
377  unset( $dest, $val );
378  }
379  return $root;
380  }
381 }
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
static factory( $url, array $options=null, $caller=__METHOD__)
Generate a new request object Deprecated:
$data
Utility to generate mapping file used in mw.Title (phpCharToUpper.json)
if(is_array( $mode)) switch( $mode) $input
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Definition: router.php:42
wfRecursiveRemoveDir( $dir)
Remove a directory and all its content.
handleTypeMultiFile( $moduleName, $destDir, array $info)
__construct( $registryFile, $libDir, callable $infoPrinter=null, callable $errorPrinter=null, callable $verbosePrinter=null)
$modules
Manage foreign resources registered with ResourceLoader.
handleTypeTar( $moduleName, $destDir, array $info)
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
parseBasicYaml( $input)
Basic YAML parser.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
this hook is for auditing only $req
Definition: hooks.txt:979
$lines
Definition: router.php:61
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don&#39;t exist.
$line
Definition: cdb.php:59
handleTypeFile( $moduleName, $destDir, array $info)