MediaWiki  master
rebuildLocalisationCache.php
Go to the documentation of this file.
1 <?php
2 
36 
37 require_once __DIR__ . '/Maintenance.php';
38 
45  public function __construct() {
46  parent::__construct();
47  $this->addDescription( 'Rebuild the localisation cache' );
48  $this->addOption( 'force', 'Rebuild all files, even ones not out of date' );
49  $this->addOption( 'threads', 'Fork more than one thread', false, true );
50  $this->addOption( 'outdir', 'Override the output directory (normally $wgCacheDirectory)',
51  false, true );
52  $this->addOption( 'lang', 'Only rebuild these languages, comma separated.',
53  false, true );
54  $this->addOption(
55  'store-class',
56  'Override the LC store class (normally $wgLocalisationCacheConf[\'storeClass\'])',
57  false,
58  true
59  );
60  $this->addOption(
61  'no-database',
62  'EXPERIMENTAL: Disable the database backend. Setting this option will result in an error ' .
63  'if you have extensions or use site configuration that need the database. This is an ' .
64  'experimental feature to allow offline building of the localisation cache. Known limitations:' .
65  "\n" .
66  '* Incompatible with LCStoreDB, which always requires a database. ' . "\n" .
67  '* The message purge may require a database. See --skip-message-purge.'
68  );
69  // T237148: The Gadget extension (bundled with MediaWiki by default) requires a database`
70  // connection to register its modules for MessageBlobStore.
71  $this->addOption(
72  'skip-message-purge',
73  'Skip purging of MessageBlobStore. The purge operation may require a database, depending ' .
74  'on the configuration and extensions on this wiki. If skipping the purge now, you need to ' .
75  'run purgeMessageBlobStore.php shortly after deployment.'
76  );
77  }
78 
79  public function finalSetup() {
80  # This script needs to be run to build the initial l10n cache. But if
81  # $wgLanguageCode is not 'en', it won't be able to run because there is
82  # no l10n cache. Break the cycle by forcing $wgLanguageCode = 'en'.
83  global $wgLanguageCode;
84  $wgLanguageCode = 'en';
85  parent::finalSetup();
86  }
87 
88  public function execute() {
90 
91  $force = $this->hasOption( 'force' );
92  $threads = $this->getOption( 'threads', 1 );
93  if ( $threads < 1 || $threads != intval( $threads ) ) {
94  $this->output( "Invalid thread count specified; running single-threaded.\n" );
95  $threads = 1;
96  }
97  if ( $threads > 1 && wfIsWindows() ) {
98  $this->output( "Threaded rebuild is not supported on Windows; running single-threaded.\n" );
99  $threads = 1;
100  }
101  if ( $threads > 1 && !function_exists( 'pcntl_fork' ) ) {
102  $this->output( "PHP pcntl extension is not present; running single-threaded.\n" );
103  $threads = 1;
104  }
105 
106  $conf = $wgLocalisationCacheConf;
107  // Allow fallbacks to create CDB files
108  $conf['manualRecache'] = false;
109  $conf['forceRecache'] = $force || !empty( $conf['forceRecache'] );
110  if ( $this->hasOption( 'outdir' ) ) {
111  $conf['storeDirectory'] = $this->getOption( 'outdir' );
112  }
113 
114  if ( $this->hasOption( 'store-class' ) ) {
115  $conf['storeClass'] = $this->getOption( 'store-class' );
116  }
117 
118  // XXX Copy-pasted from ServiceWiring.php. Do we need a factory for this one caller?
119  $services = MediaWikiServices::getInstance();
120  $lc = new LocalisationCacheBulkLoad(
121  new ServiceOptions(
123  $conf,
124  $services->getMainConfig()
125  ),
127  LoggerFactory::getInstance( 'localisation' ),
128  $this->hasOption( 'skip-message-purge' ) ? [] :
129  [ static function () use ( $services ) {
130  MessageBlobStore::clearGlobalCacheEntry( $services->getMainWANObjectCache() );
131  } ],
132  $services->getLanguageNameUtils(),
133  $services->getHookContainer()
134  );
135 
136  $allCodes = array_keys( $services
137  ->getLanguageNameUtils()
138  ->getLanguageNames( LanguageNameUtils::AUTONYMS, LanguageNameUtils::SUPPORTED ) );
139  if ( $this->hasOption( 'lang' ) ) {
140  # Validate requested languages
141  $codes = array_intersect( $allCodes,
142  explode( ',', $this->getOption( 'lang' ) ) );
143  # Bailed out if nothing is left
144  if ( count( $codes ) == 0 ) {
145  $this->fatalError( 'None of the languages specified exists.' );
146  }
147  } else {
148  # By default get all languages
149  $codes = $allCodes;
150  }
151  sort( $codes );
152 
153  // Initialise and split into chunks
154  $numRebuilt = 0;
155  $total = count( $codes );
156  $chunks = array_chunk( $codes, ceil( count( $codes ) / $threads ) );
157  $pids = [];
158  $parentStatus = 0;
159  foreach ( $chunks as $codes ) {
160  // Do not fork for only one thread
161  $pid = ( $threads > 1 ) ? pcntl_fork() : -1;
162 
163  if ( $pid === 0 ) {
164  // Child, reseed because there is no bug in PHP:
165  // https://bugs.php.net/bug.php?id=42465
166  mt_srand( getmypid() );
167 
168  $this->doRebuild( $codes, $lc, $force );
169  return;
170  } elseif ( $pid === -1 ) {
171  // Fork failed or one thread, do it serialized
172  $numRebuilt += $this->doRebuild( $codes, $lc, $force );
173  } else {
174  // Main thread
175  $pids[] = $pid;
176  }
177  }
178  // Wait for all children
179  foreach ( $pids as $pid ) {
180  $status = 0;
181  pcntl_waitpid( $pid, $status );
182 
183  if ( pcntl_wifexited( $status ) ) {
184  $code = pcntl_wexitstatus( $status );
185  if ( $code ) {
186  $this->output( "Pid $pid exited with status $code !!\n" );
187  }
188  // Mush all child statuses into a single value in the parent.
189  $parentStatus |= $code;
190  } elseif ( pcntl_wifsignaled( $status ) ) {
191  $signum = pcntl_wtermsig( $status );
192  $this->output( "Pid $pid terminated by signal $signum !!\n" );
193  $parentStatus |= 1;
194  }
195  }
196 
197  if ( !$pids ) {
198  $this->output( "$numRebuilt languages rebuilt out of $total\n" );
199  if ( $numRebuilt === 0 ) {
200  $this->output( "Use --force to rebuild the caches which are still fresh.\n" );
201  }
202  }
203  if ( $parentStatus ) {
204  $this->fatalError( 'Failed.', $parentStatus );
205  }
206  }
207 
216  private function doRebuild( $codes, $lc, $force ) {
217  $numRebuilt = 0;
218  foreach ( $codes as $code ) {
219  if ( $force || $lc->isExpired( $code ) ) {
220  $this->output( "Rebuilding $code...\n" );
221  $lc->recache( $code );
222  $numRebuilt++;
223  }
224  }
225 
226  return $numRebuilt;
227  }
228 
230  public function getDbType() {
231  if ( $this->hasOption( 'no-database' ) ) {
232  return Maintenance::DB_NONE;
233  }
234 
235  return parent::getDbType();
236  }
237 
243  public function setForce( $forced = true ) {
244  $this->mOptions['force'] = $forced;
245  }
246 }
247 
248 $maintClass = RebuildLocalisationCache::class;
249 require_once RUN_MAINTENANCE_IF_MAIN;
LocalisationCache\CONSTRUCTOR_OPTIONS
const CONSTRUCTOR_OPTIONS
Definition: LocalisationCache.php:235
RebuildLocalisationCache\setForce
setForce( $forced=true)
Sets whether a run of this maintenance script has the force parameter set.
Definition: rebuildLocalisationCache.php:243
RebuildLocalisationCache\__construct
__construct()
Default constructor.
Definition: rebuildLocalisationCache.php:45
MessageBlobStore\clearGlobalCacheEntry
static clearGlobalCacheEntry(WANObjectCache $cache)
Invalidate cache keys for all known modules.
Definition: MessageBlobStore.php:189
RebuildLocalisationCache\execute
execute()
Do the actual work.
Definition: rebuildLocalisationCache.php:88
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:203
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:489
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:329
RebuildLocalisationCache\finalSetup
finalSetup()
Handle some last-minute setup here.
Definition: rebuildLocalisationCache.php:79
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:59
$wgCacheDirectory
$wgCacheDirectory
Directory for caching data in the local filesystem.
Definition: DefaultSettings.php:2757
LocalisationCacheBulkLoad
A localisation cache optimised for loading large amounts of data for many languages.
Definition: LocalisationCacheBulkLoad.php:25
MediaWiki\Languages\LanguageNameUtils
A service that provides utilities to do with language names and codes.
Definition: LanguageNameUtils.php:43
RebuildLocalisationCache\getDbType
getDbType()
@inheritDoc
Definition: rebuildLocalisationCache.php:230
MediaWiki\Config\ServiceOptions
A class for passing options to services.
Definition: ServiceOptions.php:27
MediaWiki\Logger\LoggerFactory
PSR-3 logger instance factory.
Definition: LoggerFactory.php:45
LocalisationCache\getStoreFromConf
static getStoreFromConf(array $conf, $fallbackCacheDir)
Return a suitable LCStore as specified by the given configuration.
Definition: LocalisationCache.php:207
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:249
$wgLocalisationCacheConf
$wgLocalisationCacheConf
Localisation cache configuration.
Definition: DefaultSettings.php:3098
$maintClass
$maintClass
Definition: rebuildLocalisationCache.php:248
RebuildLocalisationCache
Maintenance script to rebuild the localisation cache.
Definition: rebuildLocalisationCache.php:44
wfIsWindows
wfIsWindows()
Check if the operating system is Windows.
Definition: GlobalFunctions.php:1687
$wgLanguageCode
$wgLanguageCode
Site language code.
Definition: DefaultSettings.php:3451
Maintenance\DB_NONE
const DB_NONE
Constants for DB access type.
Definition: Maintenance.php:64
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:286
RebuildLocalisationCache\doRebuild
doRebuild( $codes, $lc, $force)
Helper function to rebuild list of languages codes.
Definition: rebuildLocalisationCache.php:216
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:435
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option was set.
Definition: Maintenance.php:271