Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 206 |
|
0.00% |
0 / 15 |
CRAP | |
0.00% |
0 / 1 |
ForeignResourceManager | |
0.00% |
0 / 205 |
|
0.00% |
0 / 15 |
6806 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
12 | |||
run | |
0.00% |
0 / 64 |
|
0.00% |
0 / 1 |
600 | |||
setupTempDir | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
6 | |||
cacheKey | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
cacheGet | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
cacheSet | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
fetch | |
0.00% |
0 / 26 |
|
0.00% |
0 / 1 |
72 | |||
handleTypeFile | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
30 | |||
handleTypeMultiFile | |
0.00% |
0 / 12 |
|
0.00% |
0 / 1 |
56 | |||
handleTypeTar | |
0.00% |
0 / 42 |
|
0.00% |
0 / 1 |
240 | |||
verbose | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
output | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
error | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
cleanUp | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
72 | |||
validateLicense | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
20 |
1 | <?php |
2 | /** |
3 | * This program is free software; you can redistribute it and/or modify |
4 | * it under the terms of the GNU General Public License as published by |
5 | * the Free Software Foundation; either version 2 of the License, or |
6 | * (at your option) any later version. |
7 | * |
8 | * This program is distributed in the hope that it will be useful, |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
11 | * GNU General Public License for more details. |
12 | * |
13 | * You should have received a copy of the GNU General Public License along |
14 | * with this program; if not, write to the Free Software Foundation, Inc., |
15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
16 | * http://www.gnu.org/copyleft/gpl.html |
17 | * |
18 | * @file |
19 | */ |
20 | |
21 | namespace MediaWiki\ResourceLoader; |
22 | |
23 | use Composer\Spdx\SpdxLicenses; |
24 | use LogicException; |
25 | use MediaWiki\MainConfigNames; |
26 | use MediaWiki\MediaWikiServices; |
27 | use PharData; |
28 | use RecursiveDirectoryIterator; |
29 | use RecursiveIteratorIterator; |
30 | use SplFileInfo; |
31 | use Symfony\Component\Yaml\Yaml; |
32 | |
33 | /** |
34 | * Manage foreign resources registered with ResourceLoader. |
35 | * |
36 | * @since 1.32 |
37 | * @ingroup ResourceLoader |
38 | * @see https://www.mediawiki.org/wiki/Foreign_resources |
39 | */ |
40 | class ForeignResourceManager { |
41 | /** @var string */ |
42 | private $defaultAlgo = 'sha384'; |
43 | |
44 | /** @var bool */ |
45 | private $hasErrors = false; |
46 | |
47 | /** @var string */ |
48 | private $registryFile; |
49 | |
50 | /** @var string */ |
51 | private $libDir; |
52 | |
53 | /** @var string */ |
54 | private $tmpParentDir; |
55 | |
56 | /** @var string */ |
57 | private $cacheDir; |
58 | |
59 | /** |
60 | * @var callable|Closure |
61 | * @phan-var callable(string):void |
62 | */ |
63 | private $infoPrinter; |
64 | |
65 | /** |
66 | * @var callable|Closure |
67 | * @phan-var callable(string):void |
68 | */ |
69 | private $errorPrinter; |
70 | /** |
71 | * @var callable|Closure |
72 | * @phan-var callable(string):void |
73 | */ |
74 | private $verbosePrinter; |
75 | |
76 | /** @var string */ |
77 | private $action; |
78 | |
79 | /** @var array[] */ |
80 | private $registry; |
81 | |
82 | /** |
83 | * @param string $registryFile Path to YAML file |
84 | * @param string $libDir Path to a modules directory |
85 | * @param callable|null $infoPrinter Callback for printing info about the run. |
86 | * @param callable|null $errorPrinter Callback for printing errors from the run. |
87 | * @param callable|null $verbosePrinter Callback for printing extra verbose |
88 | * progress information from the run. |
89 | */ |
90 | public function __construct( |
91 | $registryFile, |
92 | $libDir, |
93 | callable $infoPrinter = null, |
94 | callable $errorPrinter = null, |
95 | callable $verbosePrinter = null |
96 | ) { |
97 | $this->registryFile = $registryFile; |
98 | $this->libDir = $libDir; |
99 | $this->infoPrinter = $infoPrinter ?? static function ( $_ ) { |
100 | }; |
101 | $this->errorPrinter = $errorPrinter ?? $this->infoPrinter; |
102 | $this->verbosePrinter = $verbosePrinter ?? static function ( $_ ) { |
103 | }; |
104 | |
105 | // Support XDG_CACHE_HOME to speed up CI by avoiding repeated downloads. |
106 | $conf = MediaWikiServices::getInstance()->getMainConfig(); |
107 | if ( ( $cacheHome = getenv( 'XDG_CACHE_HOME' ) ) !== false ) { |
108 | $this->cacheDir = realpath( $cacheHome ) . '/mw-foreign'; |
109 | } elseif ( ( $cacheConf = $conf->get( MainConfigNames::CacheDirectory ) ) !== false ) { |
110 | $this->cacheDir = "$cacheConf/ForeignResourceManager"; |
111 | } else { |
112 | $this->cacheDir = "{$this->libDir}/.foreign/cache"; |
113 | } |
114 | } |
115 | |
116 | /** |
117 | * @param string $action |
118 | * @param string $module |
119 | * @return bool |
120 | * @throws LogicException |
121 | */ |
122 | public function run( $action, $module ) { |
123 | $actions = [ 'update', 'verify', 'make-sri' ]; |
124 | if ( !in_array( $action, $actions ) ) { |
125 | $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' ); |
126 | return false; |
127 | } |
128 | $this->action = $action; |
129 | $this->setupTempDir( $action ); |
130 | |
131 | $this->registry = Yaml::parseFile( $this->registryFile ); |
132 | if ( $module === 'all' ) { |
133 | $modules = $this->registry; |
134 | } elseif ( isset( $this->registry[$module] ) ) { |
135 | $modules = [ $module => $this->registry[$module] ]; |
136 | } else { |
137 | $this->error( "Unknown module name.\n\nMust be one of:\n" . |
138 | wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) . |
139 | '.' |
140 | ); |
141 | return false; |
142 | } |
143 | |
144 | foreach ( $modules as $moduleName => $info ) { |
145 | $this->verbose( "\n### {$moduleName}\n\n" ); |
146 | |
147 | if ( $this->action === 'update' ) { |
148 | $this->output( "... updating '{$moduleName}'\n" ); |
149 | } elseif ( $this->action === 'verify' ) { |
150 | $this->output( "... verifying '{$moduleName}'\n" ); |
151 | } else { |
152 | $this->output( "... checking '{$moduleName}'\n" ); |
153 | } |
154 | |
155 | // Do checks on yaml content (such as license existence, validity and type keys) |
156 | // before doing any potentially destructive actions (potentially deleting directories, |
157 | // depending on action. |
158 | |
159 | if ( !isset( $info['type'] ) ) { |
160 | throw new LogicException( "Module '$moduleName' must have a 'type' key." ); |
161 | } |
162 | |
163 | $this->validateLicense( $moduleName, $info ); |
164 | |
165 | if ( $info['type'] === 'doc-only' ) { |
166 | $this->output( "... {$moduleName} is documentation-only, skipping integrity checks.\n" ); |
167 | continue; |
168 | } |
169 | |
170 | $destDir = "{$this->libDir}/$moduleName"; |
171 | |
172 | if ( $this->action === 'update' ) { |
173 | $this->verbose( "... emptying directory for $moduleName\n" ); |
174 | wfRecursiveRemoveDir( $destDir ); |
175 | } |
176 | |
177 | $this->verbose( "... preparing {$this->tmpParentDir}\n" ); |
178 | wfRecursiveRemoveDir( $this->tmpParentDir ); |
179 | if ( !wfMkdirParents( $this->tmpParentDir ) ) { |
180 | throw new LogicException( "Unable to create {$this->tmpParentDir}" ); |
181 | } |
182 | |
183 | switch ( $info['type'] ) { |
184 | case 'tar': |
185 | case 'zip': |
186 | $this->handleTypeTar( $moduleName, $destDir, $info, $info['type'] ); |
187 | break; |
188 | case 'file': |
189 | $this->handleTypeFile( $moduleName, $destDir, $info ); |
190 | break; |
191 | case 'multi-file': |
192 | $this->handleTypeMultiFile( $moduleName, $destDir, $info ); |
193 | break; |
194 | default: |
195 | throw new LogicException( "Unknown type '{$info['type']}' for '$moduleName'" ); |
196 | } |
197 | |
198 | if ( $this->action === 'update' ) { |
199 | foreach ( $info['transforms'] ?? [] as $file => $transforms ) { |
200 | $fullFilePath = "$destDir/$file"; |
201 | if ( !file_exists( $fullFilePath ) ) { |
202 | throw new LogicException( "$moduleName: invalid transform target $file" ); |
203 | } |
204 | if ( !is_array( $transforms ) || !array_is_list( $transforms ) ) { |
205 | $transforms = [ $transforms ]; |
206 | } |
207 | foreach ( $transforms as $transform ) { |
208 | if ( $transform === 'nomin' ) { |
209 | // not super efficient but these files aren't expected to be large |
210 | file_put_contents( $fullFilePath, "/*@nomin*/\n" . file_get_contents( $fullFilePath ) ); |
211 | } else { |
212 | throw new LogicException( "$moduleName: invalid transform $transform" ); |
213 | } |
214 | } |
215 | } |
216 | } |
217 | } |
218 | |
219 | $this->cleanUp(); |
220 | if ( $this->hasErrors ) { |
221 | // The "verify" action should check all modules and files and fail after, not during. |
222 | // We don't throw on the first issue so that developers enjoy access to all actionable |
223 | // information at once (given we can't have cascading errors). |
224 | // The "verify" action prints errors along the way and simply exits here. |
225 | return false; |
226 | } |
227 | |
228 | return true; |
229 | } |
230 | |
231 | /** |
232 | * Choose the temp parent directory |
233 | * |
234 | * @param string $action |
235 | */ |
236 | private function setupTempDir( $action ) { |
237 | if ( $action === 'verify' ) { |
238 | $this->tmpParentDir = wfTempDir() . '/ForeignResourceManager'; |
239 | } else { |
240 | // Use a temporary directory under the destination directory instead |
241 | // of wfTempDir() because PHP's rename() does not work across file |
242 | // systems, and the user's /tmp and $IP may be on different filesystems. |
243 | $this->tmpParentDir = "{$this->libDir}/.foreign/tmp"; |
244 | } |
245 | } |
246 | |
247 | /** |
248 | * @param string $src |
249 | * @param string $integrity |
250 | * @param string $moduleName |
251 | * @return string |
252 | */ |
253 | private function cacheKey( $src, $integrity, $moduleName ) { |
254 | $key = $moduleName |
255 | . '_' . hash( 'fnv132', $integrity ) |
256 | . '_' . hash( 'fnv132', $src ) |
257 | // Append readable filename to aid cache inspection and debugging |
258 | . '_' . basename( $src ); |
259 | $key = preg_replace( '/[.\/+?=_-]+/', '_', $key ); |
260 | return rtrim( $key, '_' ); |
261 | } |
262 | |
263 | /** |
264 | * @param string $key |
265 | * @return string|false |
266 | */ |
267 | private function cacheGet( $key ) { |
268 | // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged |
269 | return @file_get_contents( "{$this->cacheDir}/$key.data" ); |
270 | } |
271 | |
272 | /** |
273 | * @param string $key |
274 | * @param mixed $data |
275 | */ |
276 | private function cacheSet( $key, $data ) { |
277 | // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged |
278 | @mkdir( $this->cacheDir, 0777, true ); |
279 | file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX ); |
280 | } |
281 | |
282 | /** |
283 | * @param string $src |
284 | * @param string|null $integrity |
285 | * @param string $moduleName |
286 | * @return string |
287 | */ |
288 | private function fetch( string $src, $integrity, string $moduleName ) { |
289 | if ( $integrity !== null ) { |
290 | $key = $this->cacheKey( $src, $integrity, $moduleName ); |
291 | $data = $this->cacheGet( $key ); |
292 | if ( $data ) { |
293 | return $data; |
294 | } |
295 | } |
296 | |
297 | $req = MediaWikiServices::getInstance()->getHttpRequestFactory() |
298 | ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ ); |
299 | if ( !$req->execute()->isOK() ) { |
300 | throw new LogicException( "Failed to download resource at {$src}" ); |
301 | } |
302 | if ( $req->getStatus() !== 200 ) { |
303 | throw new LogicException( "Unexpected HTTP {$req->getStatus()} response from {$src}" ); |
304 | } |
305 | $data = $req->getContent(); |
306 | $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0]; |
307 | $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); |
308 | if ( $integrity === $actualIntegrity ) { |
309 | $this->verbose( "... passed integrity check for {$src}\n" ); |
310 | $key = $this->cacheKey( $src, $actualIntegrity, $moduleName ); |
311 | $this->cacheSet( $key, $data ); |
312 | } elseif ( $this->action === 'make-sri' ) { |
313 | $this->output( "Integrity for {$src}\n\tintegrity: {$actualIntegrity}\n" ); |
314 | } else { |
315 | $expectedIntegrity = $integrity ?? 'null'; |
316 | throw new LogicException( "Integrity check failed for {$src}\n" . |
317 | "\tExpected: {$expectedIntegrity}\n" . |
318 | "\tActual: {$actualIntegrity}" |
319 | ); |
320 | } |
321 | return $data; |
322 | } |
323 | |
324 | /** |
325 | * @param string $moduleName |
326 | * @param string $destDir |
327 | * @param array $info |
328 | */ |
329 | private function handleTypeFile( $moduleName, $destDir, array $info ) { |
330 | if ( !isset( $info['src'] ) ) { |
331 | throw new LogicException( "Module '$moduleName' must have a 'src' key." ); |
332 | } |
333 | $data = $this->fetch( $info['src'], $info['integrity'] ?? null, $moduleName ); |
334 | $dest = $info['dest'] ?? basename( $info['src'] ); |
335 | $path = "$destDir/$dest"; |
336 | if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { |
337 | $this->error( "File for '$moduleName' is different.\n" ); |
338 | } |
339 | if ( $this->action === 'update' ) { |
340 | wfMkdirParents( $destDir ); |
341 | file_put_contents( "$destDir/$dest", $data ); |
342 | } |
343 | } |
344 | |
345 | /** |
346 | * @param string $moduleName |
347 | * @param string $destDir |
348 | * @param array $info |
349 | */ |
350 | private function handleTypeMultiFile( $moduleName, $destDir, array $info ) { |
351 | if ( !isset( $info['files'] ) ) { |
352 | throw new LogicException( "Module '$moduleName' must have a 'files' key." ); |
353 | } |
354 | foreach ( $info['files'] as $dest => $file ) { |
355 | if ( !isset( $file['src'] ) ) { |
356 | throw new LogicException( "Module '$moduleName' file '$dest' must have a 'src' key." ); |
357 | } |
358 | $data = $this->fetch( $file['src'], $file['integrity'] ?? null, $moduleName ); |
359 | $path = "$destDir/$dest"; |
360 | if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { |
361 | $this->error( "File '$dest' for '$moduleName' is different.\n" ); |
362 | } elseif ( $this->action === 'update' ) { |
363 | wfMkdirParents( $destDir ); |
364 | file_put_contents( "$destDir/$dest", $data ); |
365 | } |
366 | } |
367 | } |
368 | |
369 | /** |
370 | * @param string $moduleName |
371 | * @param string $destDir |
372 | * @param array $info |
373 | * @param string $fileType |
374 | */ |
375 | private function handleTypeTar( $moduleName, $destDir, array $info, string $fileType ) { |
376 | $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; |
377 | if ( $info['src'] === null ) { |
378 | throw new LogicException( "Module '$moduleName' must have a 'src' key." ); |
379 | } |
380 | // Download the resource to a temporary file and open it |
381 | $data = $this->fetch( $info['src'], $info['integrity'], $moduleName ); |
382 | $tmpFile = "{$this->tmpParentDir}/$moduleName." . $fileType; |
383 | $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); |
384 | file_put_contents( $tmpFile, $data ); |
385 | $p = new PharData( $tmpFile ); |
386 | $tmpDir = "{$this->tmpParentDir}/$moduleName"; |
387 | $p->extractTo( $tmpDir ); |
388 | unset( $data, $p ); |
389 | |
390 | if ( $info['dest'] === null ) { |
391 | // Default: Replace the entire directory |
392 | $toCopy = [ $tmpDir => $destDir ]; |
393 | } else { |
394 | // Expand and normalise the 'dest' entries |
395 | $toCopy = []; |
396 | foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { |
397 | // Use glob() to expand wildcards and check existence |
398 | $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); |
399 | if ( !$fromPaths ) { |
400 | throw new LogicException( "Path '$fromSubPath' of '$moduleName' not found." ); |
401 | } |
402 | foreach ( $fromPaths as $fromPath ) { |
403 | $toCopy[$fromPath] = $toSubPath === null |
404 | ? "$destDir/" . basename( $fromPath ) |
405 | : "$destDir/$toSubPath/" . basename( $fromPath ); |
406 | } |
407 | } |
408 | } |
409 | foreach ( $toCopy as $from => $to ) { |
410 | if ( $this->action === 'verify' ) { |
411 | $this->verbose( "... verifying $to\n" ); |
412 | if ( is_dir( $from ) ) { |
413 | $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( |
414 | $from, |
415 | RecursiveDirectoryIterator::SKIP_DOTS |
416 | ) ); |
417 | /** @var SplFileInfo $file */ |
418 | foreach ( $rii as $file ) { |
419 | $remote = $file->getPathname(); |
420 | $local = strtr( $remote, [ $from => $to ] ); |
421 | if ( sha1_file( $remote ) !== sha1_file( $local ) ) { |
422 | $this->error( "File '$local' is different.\n" ); |
423 | } |
424 | } |
425 | } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) { |
426 | $this->error( "File '$to' is different.\n" ); |
427 | } |
428 | } elseif ( $this->action === 'update' ) { |
429 | $this->verbose( "... moving $from to $to\n" ); |
430 | wfMkdirParents( dirname( $to ) ); |
431 | if ( !rename( $from, $to ) ) { |
432 | throw new LogicException( "Could not move $from to $to." ); |
433 | } |
434 | } |
435 | } |
436 | } |
437 | |
438 | /** |
439 | * @param string $text |
440 | */ |
441 | private function verbose( $text ) { |
442 | ( $this->verbosePrinter )( $text ); |
443 | } |
444 | |
445 | /** |
446 | * @param string $text |
447 | */ |
448 | private function output( $text ) { |
449 | ( $this->infoPrinter )( $text ); |
450 | } |
451 | |
452 | /** |
453 | * @param string $text |
454 | */ |
455 | private function error( $text ) { |
456 | $this->hasErrors = true; |
457 | ( $this->errorPrinter )( $text ); |
458 | } |
459 | |
460 | private function cleanUp() { |
461 | wfRecursiveRemoveDir( $this->tmpParentDir ); |
462 | |
463 | // Prune the cache of files we don't recognise. |
464 | $knownKeys = []; |
465 | foreach ( $this->registry as $module => $info ) { |
466 | if ( $info['type'] === 'file' || $info['type'] === 'tar' ) { |
467 | $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'], $module ); |
468 | } elseif ( $info['type'] === 'multi-file' ) { |
469 | foreach ( $info['files'] as $file ) { |
470 | $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'], $module ); |
471 | } |
472 | } |
473 | } |
474 | foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) { |
475 | if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) { |
476 | unlink( $cacheFile ); |
477 | } |
478 | } |
479 | } |
480 | |
481 | /** |
482 | * @param string $moduleName |
483 | * @param array $info |
484 | */ |
485 | private function validateLicense( $moduleName, $info ) { |
486 | if ( !isset( $info['license'] ) || !is_string( $info['license'] ) ) { |
487 | throw new LogicException( |
488 | "Module '$moduleName' needs a valid SPDX license; no license is currently present" |
489 | ); |
490 | } |
491 | $licenses = new SpdxLicenses(); |
492 | if ( !$licenses->validate( $info['license'] ) ) { |
493 | $this->error( |
494 | "Module '$moduleName' has an invalid SPDX license identifier '{$info['license']}', " |
495 | . "see <https://spdx.org/licenses/>.\n" |
496 | ); |
497 | } |
498 | } |
499 | } |
500 | |
501 | /** @deprecated class alias since 1.40 */ |
502 | class_alias( ForeignResourceManager::class, 'ForeignResourceManager' ); |