44 private $defaultAlgo =
'sha384';
47 private $hasErrors =
false;
50 private $registryFile;
56 private $tmpParentDir;
71 private $errorPrinter;
76 private $verbosePrinter;
97 callable $infoPrinter =
null,
98 callable $errorPrinter =
null,
99 callable $verbosePrinter =
null
102 $this->registryFile = $registryFile;
103 $this->libDir = $libDir;
104 $this->infoPrinter = $infoPrinter ??
static function ( $_ ) {
106 $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
107 $this->verbosePrinter = $verbosePrinter ??
static function ( $_ ) {
112 if ( ( $cacheHome = getenv(
'XDG_CACHE_HOME' ) ) !==
false ) {
113 $this->cacheDir = realpath( $cacheHome ) .
'/mw-foreign';
115 $this->cacheDir =
"$cacheConf/ForeignResourceManager";
117 $this->cacheDir =
"{$this->libDir}/.foreign/cache";
127 public function run( $action, $module ) {
128 $actions = [
'update',
'verify',
'make-sri',
'make-cdx' ];
129 if ( !in_array( $action, $actions ) ) {
130 $this->error(
"Invalid action.\n\nMust be one of " . implode(
', ', $actions ) .
'.' );
133 $this->action = $action;
134 $this->setupTempDir( $action );
136 $this->registry = Yaml::parseFile( $this->registryFile );
137 if ( $module ===
'all' ) {
138 $modules = $this->registry;
139 } elseif ( isset( $this->registry[$module] ) ) {
140 $modules = [ $module => $this->registry[$module] ];
142 $this->error(
"Unknown module name.\n\nMust be one of:\n" .
143 wordwrap( implode(
', ', array_keys( $this->registry ) ), 80 ) .
149 if ( $this->action ===
'make-cdx' ) {
150 $this->output( json_encode(
151 $this->generateCdx( $modules ),
152 JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR
157 foreach ( $modules as $moduleName => $info ) {
158 $this->verbose(
"\n### {$moduleName}\n\n" );
160 if ( $this->action ===
'update' ) {
161 $this->output(
"... updating '{$moduleName}'\n" );
162 } elseif ( $this->action ===
'verify' ) {
163 $this->output(
"... verifying '{$moduleName}'\n" );
165 $this->output(
"... checking '{$moduleName}'\n" );
172 if ( !isset( $info[
'type'] ) ) {
173 throw new LogicException(
"Module '$moduleName' must have a 'type' key." );
176 $this->validateLicense( $moduleName, $info );
178 if ( $info[
'type'] ===
'doc-only' ) {
179 $this->output(
"... {$moduleName} is documentation-only, skipping integrity checks.\n" );
183 $destDir =
"{$this->libDir}/$moduleName";
185 if ( $this->action ===
'update' ) {
186 $this->verbose(
"... emptying directory for $moduleName\n" );
190 $this->verbose(
"... preparing {$this->tmpParentDir}\n" );
193 throw new LogicException(
"Unable to create {$this->tmpParentDir}" );
196 switch ( $info[
'type'] ) {
199 $this->handleTypeTar( $moduleName, $destDir, $info, $info[
'type'] );
202 $this->handleTypeFile( $moduleName, $destDir, $info );
205 $this->handleTypeMultiFile( $moduleName, $destDir, $info );
208 throw new LogicException(
"Unknown type '{$info['type']}' for '$moduleName'" );
213 if ( $this->hasErrors ) {
229 private function setupTempDir( $action ) {
230 if ( $action ===
'verify' ) {
231 $this->tmpParentDir =
wfTempDir() .
'/ForeignResourceManager';
236 $this->tmpParentDir =
"{$this->libDir}/.foreign/tmp";
246 private function cacheKey( $src, $integrity, $moduleName ) {
248 .
'_' . hash(
'fnv132', $integrity )
249 .
'_' . hash(
'fnv132', $src )
251 .
'_' . basename( $src );
252 $key = preg_replace(
'/[.\/+?=_-]+/',
'_', $key );
253 return rtrim( $key,
'_' );
260 private function cacheGet( $key ) {
262 return @file_get_contents(
"{$this->cacheDir}/$key.data" );
269 private function cacheSet( $key, $data ) {
271 @mkdir( $this->cacheDir, 0777,
true );
272 file_put_contents(
"{$this->cacheDir}/$key.data", $data, LOCK_EX );
281 private function fetch(
string $src, $integrity,
string $moduleName ) {
282 if ( $integrity !==
null ) {
283 $key = $this->cacheKey( $src, $integrity, $moduleName );
284 $data = $this->cacheGet( $key );
291 $req = $services->getHttpRequestFactory()
292 ->create( $src, [
'method' =>
'GET',
'followRedirects' =>
false ], __METHOD__ );
293 $reqStatusValue = $req->execute();
294 if ( !$reqStatusValue->isOK() ) {
295 $message =
"Failed to download resource at {$src}";
296 $reqError = $reqStatusValue->getMessages(
'error' )[0] ??
null;
297 if ( $reqError !==
null ) {
300 throw new LogicException( $message );
302 if ( $req->getStatus() !== 200 ) {
303 throw new LogicException(
"Unexpected HTTP {$req->getStatus()} response from {$src}" );
305 $data = $req->getContent();
306 $algo = $integrity ===
null ? $this->defaultAlgo : explode(
'-', $integrity )[0];
307 $actualIntegrity = $algo .
'-' . base64_encode( hash( $algo, $data,
true ) );
308 if ( $integrity === $actualIntegrity ) {
309 $this->verbose(
"... passed integrity check for {$src}\n" );
310 $key = $this->cacheKey( $src, $actualIntegrity, $moduleName );
311 $this->cacheSet( $key, $data );
312 } elseif ( $this->action ===
'make-sri' ) {
313 $this->output(
"Integrity for {$src}\n\tintegrity: {$actualIntegrity}\n" );
315 $expectedIntegrity = $integrity ??
'null';
316 throw new LogicException(
"Integrity check failed for {$src}\n" .
317 "\tExpected: {$expectedIntegrity}\n" .
318 "\tActual: {$actualIntegrity}"
329 private function handleTypeFile( $moduleName, $destDir, array $info ) {
330 if ( !isset( $info[
'src'] ) ) {
331 throw new LogicException(
"Module '$moduleName' must have a 'src' key." );
333 $data = $this->fetch( $info[
'src'], $info[
'integrity'] ??
null, $moduleName );
334 $dest = $info[
'dest'] ?? basename( $info[
'src'] );
335 $path =
"$destDir/$dest";
336 if ( $this->action ===
'verify' && sha1_file(
$path ) !== sha1( $data ) ) {
337 $this->error(
"File for '$moduleName' is different.\n" );
339 if ( $this->action ===
'update' ) {
341 file_put_contents(
"$destDir/$dest", $data );
350 private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
351 if ( !isset( $info[
'files'] ) ) {
352 throw new LogicException(
"Module '$moduleName' must have a 'files' key." );
354 foreach ( $info[
'files'] as $dest => $file ) {
355 if ( !isset( $file[
'src'] ) ) {
356 throw new LogicException(
"Module '$moduleName' file '$dest' must have a 'src' key." );
358 $data = $this->fetch( $file[
'src'], $file[
'integrity'] ??
null, $moduleName );
359 $path =
"$destDir/$dest";
360 if ( $this->action ===
'verify' && sha1_file(
$path ) !== sha1( $data ) ) {
361 $this->error(
"File '$dest' for '$moduleName' is different.\n" );
362 } elseif ( $this->action ===
'update' ) {
364 file_put_contents(
"$destDir/$dest", $data );
375 private function handleTypeTar( $moduleName, $destDir, array $info,
string $fileType ) {
376 $info += [
'src' =>
null,
'integrity' =>
null,
'dest' => null ];
377 if ( $info[
'src'] ===
null ) {
378 throw new LogicException(
"Module '$moduleName' must have a 'src' key." );
381 $data = $this->fetch( $info[
'src'], $info[
'integrity'], $moduleName );
382 $tmpFile =
"{$this->tmpParentDir}/$moduleName." . $fileType;
383 $this->verbose(
"... writing '$moduleName' src to $tmpFile\n" );
384 file_put_contents( $tmpFile, $data );
385 $p =
new PharData( $tmpFile );
386 $tmpDir =
"{$this->tmpParentDir}/$moduleName";
387 $p->extractTo( $tmpDir );
390 if ( $info[
'dest'] ===
null ) {
392 $toCopy = [ $tmpDir => $destDir ];
396 foreach ( $info[
'dest'] as $fromSubPath => $toSubPath ) {
398 $fromPaths = glob(
"{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
400 throw new LogicException(
"Path '$fromSubPath' of '$moduleName' not found." );
402 foreach ( $fromPaths as $fromPath ) {
403 $toCopy[$fromPath] = $toSubPath ===
null
404 ?
"$destDir/" . basename( $fromPath )
405 :
"$destDir/$toSubPath/" . basename( $fromPath );
409 foreach ( $toCopy as $from => $to ) {
410 if ( $this->action ===
'verify' ) {
411 $this->verbose(
"... verifying $to\n" );
412 if ( is_dir( $from ) ) {
413 $rii =
new RecursiveIteratorIterator(
new RecursiveDirectoryIterator(
415 RecursiveDirectoryIterator::SKIP_DOTS
418 foreach ( $rii as $file ) {
419 $remote = $file->getPathname();
420 $local = strtr( $remote, [ $from => $to ] );
421 if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
422 $this->error(
"File '$local' is different.\n" );
425 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
426 $this->error(
"File '$to' is different.\n" );
428 } elseif ( $this->action ===
'update' ) {
429 $this->verbose(
"... moving $from to $to\n" );
431 if ( !rename( $from, $to ) ) {
432 throw new LogicException(
"Could not move $from to $to." );
441 private function verbose( $text ) {
442 ( $this->verbosePrinter )( $text );
448 private function output( $text ) {
449 ( $this->infoPrinter )( $text );
455 private function error( $text ) {
456 $this->hasErrors =
true;
457 ( $this->errorPrinter )( $text );
460 private function cleanUp() {
465 foreach ( $this->registry as $module => $info ) {
466 if ( $info[
'type'] ===
'file' || $info[
'type'] ===
'tar' ) {
467 $knownKeys[] = $this->cacheKey( $info[
'src'], $info[
'integrity'], $module );
468 } elseif ( $info[
'type'] ===
'multi-file' ) {
469 foreach ( $info[
'files'] as $file ) {
470 $knownKeys[] = $this->cacheKey( $file[
'src'], $file[
'integrity'], $module );
474 foreach ( glob(
"{$this->cacheDir}/*" ) as $cacheFile ) {
475 if ( !in_array( basename( $cacheFile,
'.data' ), $knownKeys ) ) {
476 unlink( $cacheFile );
485 private function validateLicense( $moduleName, $info ) {
486 if ( !isset( $info[
'license'] ) || !is_string( $info[
'license'] ) ) {
487 throw new LogicException(
488 "Module '$moduleName' needs a valid SPDX license; no license is currently present"
491 $licenses =
new SpdxLicenses();
492 if ( !$licenses->validate( $info[
'license'] ) ) {
494 "Module '$moduleName' has an invalid SPDX license identifier '{$info['license']}', "
495 .
"see <https://spdx.org/licenses/>.\n"
500 private function generateCdx( array $modules ): array {
502 '$schema' =>
'http://cyclonedx.org/schema/bom-1.6.schema.json',
503 'bomFormat' =>
'CycloneDX',
504 'specVersion' =>
'1.6',
505 'serialNumber' =>
'urn:uuid:' . $this->globalIdGenerator->newUUIDv4(),
509 foreach ( $modules as $moduleName => $module ) {
512 'name' => $moduleName,
513 'version' => $module[
'version'],
515 if ( preg_match(
'/ (AND|OR|WITH) /', $module[
'license'] ) ) {
516 $moduleCdx[
'licenses'][] = [
'expression' => $module[
'license'] ];
518 $moduleCdx[
'licenses'][] = [
'license' => [
'id' => $module[
'license'] ] ];
520 if ( $module[
'purl'] ??
false ) {
521 $moduleCdx[
'purl'] = $module[
'purl'];
523 if ( $module[
'version'] ??
false ) {
524 $moduleCdx[
'version'] = $module[
'version'];
526 if ( $module[
'authors'] ??
false ) {
527 $moduleCdx[
'authors'] = array_map(
528 fn ( $author ) => [
'name' => $author ],
529 preg_split(
'/,( and)? /', $module[
'authors'] )
532 if ( $module[
'homepage'] ??
false ) {
533 $moduleCdx[
'externalReferences'] = [ [
'url' => $module[
'homepage'],
'type' =>
'website' ] ];
535 $cdx[
'components'][] = $moduleCdx;