23 define(
'MW_ENTRY_POINT',
'cli' );
27 require_once __DIR__ .
'/../includes/PHPVersionCheck.php';
39 define(
'RUN_MAINTENANCE_IF_MAIN', __DIR__ .
'/doMaintenance.php' );
51 if ( strval( getenv(
'MW_INSTALL_PATH' ) ) ===
'' ) {
52 putenv(
'MW_INSTALL_PATH=' . realpath( __DIR__ .
'/..' ) );
197 $IP = getenv(
'MW_INSTALL_PATH' );
200 register_shutdown_function( [ $this,
'outputChanneled' ],
false );
213 if ( !function_exists(
'debug_backtrace' ) ) {
218 $bt = debug_backtrace();
219 $count = count( $bt );
223 if ( $bt[0][
'class'] !== self::class || $bt[0][
'function'] !==
'shouldExecute' ) {
226 $includeFuncs = [
'require_once',
'require',
'include',
'include_once' ];
227 for ( $i = 1; $i < $count; $i++ ) {
228 if ( !in_array( $bt[$i][
'function'], $includeFuncs ) ) {
244 abstract public function execute();
253 return isset( $this->mParams[$name] );
267 protected function addOption( $name, $description, $required =
false,
268 $withArg =
false, $shortName =
false, $multiOccurrence =
false
270 $this->mParams[$name] = [
271 'desc' => $description,
272 'require' => $required,
273 'withArg' => $withArg,
274 'shortName' => $shortName,
275 'multiOccurrence' => $multiOccurrence
278 if ( $shortName !==
false ) {
279 $this->mShortParamsMap[$shortName] = $name;
289 return isset( $this->mOptions[$name] );
302 protected function getOption( $name, $default =
null ) {
304 return $this->mOptions[$name];
307 $this->mOptions[$name] = $default;
309 return $this->mOptions[$name];
319 protected function addArg( $arg, $description, $required =
true ) {
320 $this->mArgList[] = [
322 'desc' => $description,
323 'require' => $required
332 unset( $this->mParams[$name] );
341 $this->mAllowUnregisteredOptions = $allow;
349 $this->mDescription = $text;
357 protected function hasArg( $argId = 0 ) {
358 if ( func_num_args() === 0 ) {
359 wfDeprecated( __METHOD__ .
' without an $argId',
'1.33' );
362 return isset( $this->mArgs[$argId] );
371 protected function getArg( $argId = 0, $default =
null ) {
372 if ( func_num_args() === 0 ) {
373 wfDeprecated( __METHOD__ .
' without an $argId',
'1.33' );
376 return $this->mArgs[$argId] ?? $default;
395 $this->mBatchSize =
$s;
402 if ( $this->mBatchSize ) {
403 $this->
addOption(
'batch-size',
'Run this many operations ' .
404 'per batch, default: ' . $this->mBatchSize,
false,
true );
405 if ( isset( $this->mParams[
'batch-size'] ) ) {
407 $this->mDependantParameters[
'batch-size'] = $this->mParams[
'batch-size'];
427 if ( $len == self::STDIN_ALL ) {
428 return file_get_contents(
'php://stdin' );
430 $f = fopen(
'php://stdin',
'rt' );
434 $input = fgets( $f, $len );
437 return rtrim( $input );
453 protected function output( $out, $channel =
null ) {
455 if ( class_exists( MediaWikiServices::class ) ) {
457 $stats = MediaWikiServices::getInstance()->getStatsdDataFactory();
458 if ( $stats->getDataCount() > 1000 ) {
463 if ( $this->mQuiet ) {
466 if ( $channel ===
null ) {
470 $out = preg_replace(
'/\n\z/',
'', $out );
481 protected function error( $err, $die = 0 ) {
482 if ( intval( $die ) !== 0 ) {
488 ( PHP_SAPI ==
'cli' || PHP_SAPI ==
'phpdbg' ) &&
489 !defined(
'MW_PHPUNIT_TEST' )
491 fwrite( STDERR, $err .
"\n" );
505 $this->
error( $msg );
516 if ( !$this->atLineStart ) {
518 $this->atLineStart =
true;
531 if ( $msg ===
false ) {
538 if ( !$this->atLineStart && $channel !== $this->lastChannel ) {
544 $this->atLineStart =
false;
545 if ( $channel ===
null ) {
548 $this->atLineStart =
true;
550 $this->lastChannel = $channel;
571 # Generic (non script dependant) options:
573 $this->
addOption(
'help',
'Display this help message',
false,
false,
'h' );
574 $this->
addOption(
'quiet',
'Whether to suppress non-error output',
false,
false,
'q' );
575 $this->
addOption(
'conf',
'Location of LocalSettings.php, if not default',
false,
true );
576 $this->
addOption(
'wiki',
'For specifying the wiki ID',
false,
true );
577 $this->
addOption(
'globals',
'Output globals at the end of processing for debugging' );
580 'Set a specific memory limit for the script, '
581 .
'"max" for no limit or "default" to avoid changing it',
585 $this->
addOption(
'server',
"The protocol and server name to use in URLs, e.g. " .
586 "http://en.wikipedia.org. This is sometimes necessary because " .
587 "server name detection may fail in command line scripts.",
false,
true );
588 $this->
addOption(
'profiler',
'Profiler output format (usually "text")',
false,
true );
590 $this->
addOption(
'mwdebug',
'Enable built-in MediaWiki development settings',
false,
false );
592 # Save generic options to display them separately in help
595 # Script dependant options:
599 $this->
addOption(
'dbuser',
'The DB user to use for this script',
false,
true );
600 $this->
addOption(
'dbpass',
'The password to use for this script',
false,
true );
601 $this->
addOption(
'dbgroupdefault',
'The default DB group to use.',
false,
true );
604 # Save additional script dependant options to display
605 # Â them separately in help
606 $this->mDependantParameters = array_diff_key( $this->mParams, $this->mGenericParameters );
614 if ( $this->config ===
null ) {
615 $this->config = MediaWikiServices::getInstance()->getMainConfig();
639 $this->requiredExtensions[] = $name;
650 foreach ( $this->requiredExtensions as $name ) {
651 if ( !$registry->isLoaded( $name ) ) {
657 $joined = implode(
', ', $missing );
658 $msg =
"The following extensions are required to be installed "
659 .
"for this script to run: $joined. Please enable them and then try again.";
669 if ( function_exists(
'posix_getpwuid' ) ) {
670 $agent = posix_getpwuid( posix_geteuid() )[
'name'];
676 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
678 $lbFactory->setAgentName(
679 mb_strlen( $agent ) > 15 ? mb_substr( $agent, 0, 15 ) .
'...' : $agent
690 $services = MediaWikiServices::getInstance();
691 $stats = $services->getStatsdDataFactory();
693 $lbFactory = $services->getDBLoadBalancerFactory();
694 $lbFactory->setWaitForReplicationListener(
696 function () use ( $stats,
$config ) {
707 $lbFactory->getMainLB()->setTransactionListener(
709 function ( $trigger ) use ( $stats,
$config ) {
711 if (
$config->
get(
'CommandLineMode' ) && $trigger === IDatabase::TRIGGER_COMMIT ) {
731 require_once $classFile;
734 $this->
error(
"Cannot spawn child: $maintClass" );
742 $child->loadParamsAndArgs( $this->mSelf, $this->mOptions, $this->mArgs );
743 if ( !is_null( $this->mDb ) ) {
744 $child->setDB( $this->mDb );
756 # Abort if called from a web server
757 # wfIsCLI() is not available yet
758 if ( PHP_SAPI !==
'cli' && PHP_SAPI !==
'phpdbg' ) {
759 $this->
fatalError(
'This script must be run from the command line' );
762 if ( $IP ===
null ) {
763 $this->
fatalError(
"\$IP not set, aborting!\n" .
764 '(Did you forget to call parent::__construct() in your maintenance script?)' );
767 # Make sure we can handle script parameters
768 if ( !defined(
'HPHP_VERSION' ) && !ini_get(
'register_argc_argv' ) ) {
769 $this->
fatalError(
'Cannot get command line arguments, register_argc_argv is set to false' );
775 if ( ini_get(
'display_errors' ) ) {
776 ini_set(
'display_errors',
'stderr' );
781 # Set the memory limit
782 # Note we need to set it again later in cache LocalSettings changed it
785 # Set max execution time to 0 (no limit). PHP.net says that
786 # "When running PHP from the command line the default setting is 0."
787 # But sometimes this doesn't seem to be the case.
788 ini_set(
'max_execution_time', 0 );
790 # Define us as being in MediaWiki
791 define(
'MEDIAWIKI',
true );
795 # Turn off output buffering if it's on
796 while ( ob_get_level() > 0 ) {
811 $limit = $this->
getOption(
'memory-limit',
'max' );
812 $limit = trim( $limit,
"\" '" );
821 if ( $limit ==
'max' ) {
824 if ( $limit !=
'default' ) {
825 ini_set(
'memory_limit', $limit );
843 $profiler =
new $class(
844 [
'sampling' => 1,
'output' => [
$output ] ]
848 $profiler->setAllowOutput();
853 $trxProfiler->setLogger( LoggerFactory::getInstance(
'DBPerformance' ) );
861 $this->mOptions = [];
863 $this->mInputLoaded =
false;
876 $this->orderedOptions = [];
879 for ( $arg = reset( $argv ); $arg !==
false; $arg = next( $argv ) ) {
880 if ( $arg ==
'--' ) {
881 # End of options, remainder should be considered arguments
882 $arg = next( $argv );
883 while ( $arg !==
false ) {
885 $arg = next( $argv );
888 } elseif ( substr( $arg, 0, 2 ) ==
'--' ) {
890 $option = substr( $arg, 2 );
891 if ( isset( $this->mParams[$option] ) && $this->mParams[$option][
'withArg'] ) {
892 $param = next( $argv );
893 if ( $param ===
false ) {
894 $this->
error(
"\nERROR: $option parameter needs a value after it\n" );
898 $this->
setParam( $options, $option, $param );
900 $bits = explode(
'=', $option, 2 );
901 $this->
setParam( $options, $bits[0], $bits[1] ?? 1 );
903 } elseif ( $arg ==
'-' ) {
904 # Lonely "-", often used to indicate stdin or stdout.
906 } elseif ( substr( $arg, 0, 1 ) ==
'-' ) {
908 $argLength = strlen( $arg );
909 for ( $p = 1; $p < $argLength; $p++ ) {
911 if ( !isset( $this->mParams[$option] ) && isset( $this->mShortParamsMap[$option] ) ) {
912 $option = $this->mShortParamsMap[$option];
915 if ( isset( $this->mParams[$option][
'withArg'] ) && $this->mParams[$option][
'withArg'] ) {
916 $param = next( $argv );
917 if ( $param ===
false ) {
918 $this->
error(
"\nERROR: $option parameter needs a value after it\n" );
921 $this->
setParam( $options, $option, $param );
923 $this->
setParam( $options, $option, 1 );
931 $this->mOptions = $options;
932 $this->mArgs =
$args;
934 $this->mInputLoaded =
true;
949 private function setParam( &$options, $option, $value ) {
950 $this->orderedOptions[] = [ $option, $value ];
952 if ( isset( $this->mParams[$option] ) ) {
953 $multi = $this->mParams[$option][
'multiOccurrence'];
957 $exists = array_key_exists( $option, $options );
958 if ( $multi && $exists ) {
959 $options[$option][] = $value;
960 } elseif ( $multi ) {
961 $options[$option] = [ $value ];
962 } elseif ( !$exists ) {
963 $options[$option] = $value;
965 $this->
error(
"\nERROR: $option parameter given twice\n" );
980 # If we were given opts or args, set those and return early
982 $this->mSelf =
$self;
983 $this->mInputLoaded =
true;
986 $this->mOptions = $opts;
987 $this->mInputLoaded =
true;
990 $this->mArgs =
$args;
991 $this->mInputLoaded =
true;
994 # If we've already loaded input (either by user values or from $argv)
995 # skip on loading it again. The array_shift() will corrupt values if
996 # it's run again and again
997 if ( $this->mInputLoaded ) {
1004 $this->mSelf = $argv[0];
1013 # Check to make sure we've got all the required options
1014 foreach ( $this->mParams as $opt => $info ) {
1015 if ( $info[
'require'] && !$this->
hasOption( $opt ) ) {
1016 $this->
error(
"Param $opt required!" );
1020 # Check arg list too
1021 foreach ( $this->mArgList as $k => $info ) {
1022 if ( $info[
'require'] && !$this->
hasArg( $k ) ) {
1023 $this->
error(
'Argument <' . $info[
'name'] .
'> required!' );
1027 if ( !$this->mAllowUnregisteredOptions ) {
1028 # Check for unexpected options
1029 foreach ( $this->mOptions as $opt => $val ) {
1031 $this->
error(
"Unexpected option $opt!" );
1045 $this->mDbUser = $this->
getOption(
'dbuser' );
1048 $this->mDbPass = $this->
getOption(
'dbpass' );
1051 $this->mQuiet =
true;
1053 if ( $this->
hasOption(
'batch-size' ) ) {
1054 $this->mBatchSize = intval( $this->
getOption(
'batch-size' ) );
1063 if ( !$force && !$this->
hasOption(
'help' ) ) {
1069 $descWidth = $screenWidth - ( 2 * strlen( $tab ) );
1071 ksort( $this->mParams );
1072 $this->mQuiet =
false;
1075 if ( $this->mDescription ) {
1076 $this->
output(
"\n" . wordwrap( $this->mDescription, $screenWidth ) .
"\n" );
1078 $output =
"\nUsage: php " . basename( $this->mSelf );
1081 if ( $this->mParams ) {
1082 $output .=
" [--" . implode(
"|--", array_keys( $this->mParams ) ) .
"]";
1086 if ( $this->mArgList ) {
1088 foreach ( $this->mArgList as $k => $arg ) {
1089 if ( $arg[
'require'] ) {
1090 $output .=
'<' . $arg[
'name'] .
'>';
1092 $output .=
'[' . $arg[
'name'] .
']';
1094 if ( $k < count( $this->mArgList ) - 1 ) {
1099 $this->
output(
"$output\n\n" );
1101 # TODO abstract some repetitive code below
1104 $this->
output(
"Generic maintenance parameters:\n" );
1105 foreach ( $this->mGenericParameters as $par => $info ) {
1106 if ( $info[
'shortName'] !==
false ) {
1107 $par .=
" (-{$info['shortName']})";
1110 wordwrap(
"$tab--$par: " . $info[
'desc'], $descWidth,
1111 "\n$tab$tab" ) .
"\n"
1117 if ( count( $scriptDependantParams ) > 0 ) {
1118 $this->
output(
"Script dependant parameters:\n" );
1120 foreach ( $scriptDependantParams as $par => $info ) {
1121 if ( $info[
'shortName'] !==
false ) {
1122 $par .=
" (-{$info['shortName']})";
1125 wordwrap(
"$tab--$par: " . $info[
'desc'], $descWidth,
1126 "\n$tab$tab" ) .
"\n"
1134 $scriptSpecificParams = array_diff_key(
1135 # all script parameters:
1138 $this->mGenericParameters,
1139 $this->mDependantParameters
1141 if ( count( $scriptSpecificParams ) > 0 ) {
1142 $this->
output(
"Script specific parameters:\n" );
1144 foreach ( $scriptSpecificParams as $par => $info ) {
1145 if ( $info[
'shortName'] !==
false ) {
1146 $par .=
" (-{$info['shortName']})";
1149 wordwrap(
"$tab--$par: " . $info[
'desc'], $descWidth,
1150 "\n$tab$tab" ) .
"\n"
1157 if ( count( $this->mArgList ) > 0 ) {
1158 $this->
output(
"Arguments:\n" );
1160 foreach ( $this->mArgList as $info ) {
1161 $openChar = $info[
'require'] ?
'<' :
'[';
1162 $closeChar = $info[
'require'] ?
'>' :
']';
1164 wordwrap(
"$tab$openChar" . $info[
'name'] .
"$closeChar: " .
1165 $info[
'desc'], $descWidth,
"\n$tab$tab" ) .
"\n"
1182 # Turn off output buffering again, it might have been turned on in the settings files
1183 if ( ob_get_level() ) {
1189 # Override $wgServer
1191 $wgServer = $this->
getOption(
'server', $wgServer );
1194 # If these were passed, use them
1195 if ( $this->mDbUser ) {
1198 if ( $this->mDbPass ) {
1201 if ( $this->
hasOption(
'dbgroupdefault' ) ) {
1202 $wgDBDefaultGroup = $this->
getOption(
'dbgroupdefault',
null );
1204 MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->destroy();
1224 MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->destroy();
1227 # Apply debug settings
1229 require __DIR__ .
'/../includes/DevelopmentSettings.php';
1237 $wgShowExceptionDetails =
true;
1240 Wikimedia\suppressWarnings();
1241 set_time_limit( 0 );
1242 Wikimedia\restoreWarnings();
1251 if ( defined(
'MW_CMDLINE_CALLBACK' ) ) {
1253 call_user_func( MW_CMDLINE_CALLBACK );
1274 if ( isset( $this->mOptions[
'conf'] ) ) {
1275 $settingsFile = $this->mOptions[
'conf'];
1276 } elseif ( defined(
"MW_CONFIG_FILE" ) ) {
1277 $settingsFile = MW_CONFIG_FILE;
1279 $settingsFile =
"$IP/LocalSettings.php";
1281 if ( isset( $this->mOptions[
'wiki'] ) ) {
1282 $bits = explode(
'-', $this->mOptions[
'wiki'], 2 );
1283 define(
'MW_DB', $bits[0] );
1284 define(
'MW_PREFIX', $bits[1] ??
'' );
1285 } elseif ( isset( $this->mOptions[
'server'] ) ) {
1290 $_SERVER[
'SERVER_NAME'] = $this->mOptions[
'server'];
1293 if ( !is_readable( $settingsFile ) ) {
1294 $this->
fatalError(
"A copy of your installation's LocalSettings.php\n" .
1295 "must exist and be readable in the source directory.\n" .
1296 "Use --conf to specify it." );
1298 $wgCommandLineMode =
true;
1300 return $settingsFile;
1311 # Data should come off the master, wrapped in a transaction
1316 # Get "active" text records from the revisions table
1318 $this->
output(
'Searching for active text records in revisions table...' );
1319 $res = $dbw->select(
'revision',
'rev_text_id', [], __METHOD__, [
'DISTINCT' ] );
1320 foreach (
$res as $row ) {
1321 $cur[] = $row->rev_text_id;
1323 $this->
output(
"done.\n" );
1325 # Get "active" text records from the archive table
1326 $this->
output(
'Searching for active text records in archive table...' );
1327 $res = $dbw->select(
'archive',
'ar_text_id', [], __METHOD__, [
'DISTINCT' ] );
1328 foreach (
$res as $row ) {
1329 # old pre-MW 1.5 records can have null ar_text_id's.
1330 if ( $row->ar_text_id !==
null ) {
1331 $cur[] = $row->ar_text_id;
1334 $this->
output(
"done.\n" );
1336 # Get "active" text records via the content table
1338 $this->
output(
'Searching for active text records via contents table...' );
1339 $res = $dbw->select(
'content',
'content_address', [], __METHOD__, [
'DISTINCT' ] );
1340 $blobStore = MediaWikiServices::getInstance()->getBlobStore();
1341 foreach (
$res as $row ) {
1343 $textId = $blobStore->getTextIdFromAddress( $row->content_address );
1348 $this->
output(
"done.\n" );
1350 $this->
output(
"done.\n" );
1352 # Get the IDs of all text records not in these sets
1353 $this->
output(
'Searching for inactive text records...' );
1354 $cond =
'old_id NOT IN ( ' . $dbw->makeList( $cur ) .
' )';
1355 $res = $dbw->select(
'text',
'old_id', [ $cond ], __METHOD__, [
'DISTINCT' ] );
1357 foreach (
$res as $row ) {
1358 $old[] = $row->old_id;
1360 $this->
output(
"done.\n" );
1362 # Inform the user of what we're going to do
1363 $count = count( $old );
1364 $this->
output(
"$count inactive items found.\n" );
1366 # Delete as appropriate
1367 if ( $delete && $count ) {
1368 $this->
output(
'Deleting...' );
1369 $dbw->delete(
'text', [
'old_id' => $old ], __METHOD__ );
1370 $this->
output(
"done.\n" );
1396 protected function getDB( $db, $groups = [], $dbDomain =
false ) {
1397 if ( $this->mDb ===
null ) {
1398 return MediaWikiServices::getInstance()
1399 ->getDBLoadBalancerFactory()
1400 ->getMainLB( $dbDomain )
1401 ->getMaintenanceConnectionRef( $db, $groups, $dbDomain );
1427 $dbw->
begin( $fname );
1443 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
1444 $waitSucceeded = $lbFactory->waitForReplication(
1445 [
'timeout' => 30,
'ifWritesSince' => $this->lastReplicationWait ]
1447 $this->lastReplicationWait = microtime(
true );
1448 return $waitSucceeded;
1470 $write = [
'searchindex' ];
1480 $db->lockTables( $read, $write, __CLASS__ .
'-searchIndexLock' );
1488 $db->unlockTables( __CLASS__ .
'-searchIndexLock' );
1512 if ( $maxLockTime ) {
1513 $this->
output(
" --- Waiting for lock ---" );
1519 # Loop through the results and do a search update
1520 foreach ( $results as $row ) {
1521 # Allow reads to be processed
1522 if ( $maxLockTime && time() > $lockTime + $maxLockTime ) {
1523 $this->
output(
" --- Relocking ---" );
1528 call_user_func( $callback, $dbw, $row );
1531 # Unlock searchindex
1532 if ( $maxLockTime ) {
1533 $this->
output(
" --- Unlocking --" );
1550 $titleObj = $rev->getTitle();
1551 $title = $titleObj->getPrefixedDBkey();
1552 $this->
output(
"$title..." );
1553 # Update searchindex
1554 $u =
new SearchUpdate( $pageId, $titleObj, $rev->getContent() );
1576 for ( $i = $seconds; $i >= 0; $i-- ) {
1577 if ( $i != $seconds ) {
1578 $this->
output( str_repeat(
"\x08", strlen( $i + 1 ) ) );
1597 if ( !function_exists(
'posix_isatty' ) ) {
1610 static $isatty =
null;
1611 if ( is_null( $isatty ) ) {
1615 if ( $isatty && function_exists(
'readline' ) ) {
1616 return readline( $prompt );
1621 if ( feof( STDIN ) ) {
1624 $st = fgets( STDIN, 1024 );
1627 if ( $st ===
false ) {
1630 $resp = trim( $st );
1645 $encPrompt = Shell::escape( $prompt );
1646 $command =
"read -er -p $encPrompt && echo \"\$REPLY\"";
1647 $encCommand = Shell::escape(
$command );
1648 $line = Shell::escape(
"$bash -c $encCommand", $retval, [], [
'walltime' => 0 ] );
1650 if ( $retval == 0 ) {
1652 } elseif ( $retval == 127 ) {
1663 if ( feof( STDIN ) ) {
1668 return fgets( STDIN, 1024 );
1679 $default = [ 80, 50 ];
1683 if ( Shell::isDisabled() ) {
1693 $result = Shell::command(
'stty',
'size' )
1695 if ( $result->getExitCode() !== 0 ) {
1698 if ( !preg_match(
'/^(\d+) (\d+)$/', $result->getStdout(), $m ) ) {
1701 return [ intval( $m[2] ), intval( $m[1] ) ];
1709 require_once __DIR__ .
'/../tests/common/TestsAutoLoader.php';
1729 parent::__construct();
1730 $this->
addOption(
'force',
'Run the update even if it was completed already' );
1739 && $db->selectRow(
'updatelog',
'1', [
'ul_key' => $key ], __METHOD__ )
1750 $db->insert(
'updatelog', [
'ul_key' => $key ], __METHOD__, [
'IGNORE' ] );
1762 return "Update '{$key}' already logged as completed. Use --force to run it again.";