MediaWiki  master
rebuildLocalisationCache.php
Go to the documentation of this file.
1 <?php
2 
38 
39 require_once __DIR__ . '/Maintenance.php';
40 
47  public function __construct() {
48  parent::__construct();
49  $this->addDescription( 'Rebuild the localisation cache' );
50  $this->addOption( 'dry-run', 'Determine what languages need to be rebuilt without changing anything' );
51  $this->addOption( 'force', 'Rebuild all files, even ones not out of date' );
52  $this->addOption( 'threads', 'Fork more than one thread', false, true );
53  $this->addOption( 'outdir', 'Override the output directory (normally $wgCacheDirectory)',
54  false, true );
55  $this->addOption( 'lang', 'Only rebuild these languages, comma separated.',
56  false, true );
57  $this->addOption(
58  'store-class',
59  'Override the LC store class (normally $wgLocalisationCacheConf[\'storeClass\'])',
60  false,
61  true
62  );
63  $this->addOption(
64  'no-database',
65  'EXPERIMENTAL: Disable the database backend. Setting this option will result in an error ' .
66  'if you have extensions or use site configuration that need the database. This is an ' .
67  'experimental feature to allow offline building of the localisation cache. Known limitations:' .
68  "\n" .
69  '* Incompatible with LCStoreDB, which always requires a database. ' . "\n" .
70  '* The message purge may require a database. See --skip-message-purge.'
71  );
72  // T237148: The Gadget extension (bundled with MediaWiki by default) requires a database`
73  // connection to register its modules for MessageBlobStore.
74  $this->addOption(
75  'skip-message-purge',
76  'Skip purging of MessageBlobStore. The purge operation may require a database, depending ' .
77  'on the configuration and extensions on this wiki. If skipping the purge now, you need to ' .
78  'run purgeMessageBlobStore.php shortly after deployment.'
79  );
80  $this->addOption(
81  'no-progress',
82  "Don't print a message for each rebuilt language file. Use this instead of " .
83  "--quiet to get a brief summary of the operation."
84  );
85  }
86 
87  public function finalSetup( SettingsBuilder $settingsBuilder = null ) {
88  # This script needs to be run to build the initial l10n cache. But if
89  # LanguageCode is not 'en', it won't be able to run because there is
90  # no l10n cache. Break the cycle by forcing the LanguageCode setting to 'en'.
91  $settingsBuilder->putConfigValue( MainConfigNames::LanguageCode, 'en' );
92  parent::finalSetup( $settingsBuilder );
93  }
94 
95  public function execute() {
96  $force = $this->hasOption( 'force' );
97  $threads = $this->getOption( 'threads', 1 );
98  if ( $threads < 1 || $threads != intval( $threads ) ) {
99  $this->output( "Invalid thread count specified; running single-threaded.\n" );
100  $threads = 1;
101  }
102  if ( $threads > 1 && wfIsWindows() ) {
103  $this->output( "Threaded rebuild is not supported on Windows; running single-threaded.\n" );
104  $threads = 1;
105  }
106  if ( $threads > 1 && ( !extension_loaded( 'sockets' ) || !function_exists( 'pcntl_fork' ) ) ) {
107  $this->output( "Threaded rebuild requires ext-pcntl and ext-sockets; running single-threaded.\n" );
108  $threads = 1;
109  }
110 
111  $conf = $this->getConfig()->get( MainConfigNames::LocalisationCacheConf );
112  // Allow fallbacks to create CDB files
113  $conf['manualRecache'] = false;
114  $conf['forceRecache'] = $force || !empty( $conf['forceRecache'] );
115  if ( $this->hasOption( 'outdir' ) ) {
116  $conf['storeDirectory'] = $this->getOption( 'outdir' );
117  }
118 
119  if ( $this->hasOption( 'store-class' ) ) {
120  $conf['storeClass'] = $this->getOption( 'store-class' );
121  }
122 
123  // XXX Copy-pasted from ServiceWiring.php. Do we need a factory for this one caller?
124  $services = $this->getServiceContainer();
125  $lc = new LocalisationCacheBulkLoad(
126  new ServiceOptions(
128  $conf,
129  $services->getMainConfig()
130  ),
131  LocalisationCache::getStoreFromConf( $conf, $this->getConfig()->get( MainConfigNames::CacheDirectory ) ),
132  LoggerFactory::getInstance( 'localisation' ),
133  $this->hasOption( 'skip-message-purge' ) ? [] :
134  [ static function () use ( $services ) {
135  MessageBlobStore::clearGlobalCacheEntry( $services->getMainWANObjectCache() );
136  } ],
137  $services->getLanguageNameUtils(),
138  $services->getHookContainer()
139  );
140 
141  $allCodes = array_keys( $services
142  ->getLanguageNameUtils()
143  ->getLanguageNames( LanguageNameUtils::AUTONYMS, LanguageNameUtils::SUPPORTED ) );
144  if ( $this->hasOption( 'lang' ) ) {
145  # Validate requested languages
146  $codes = array_intersect( $allCodes,
147  explode( ',', $this->getOption( 'lang' ) ) );
148  # Bailed out if nothing is left
149  if ( count( $codes ) == 0 ) {
150  $this->fatalError( 'None of the languages specified exists.' );
151  }
152  } else {
153  # By default get all languages
154  $codes = $allCodes;
155  }
156  sort( $codes );
157 
158  $numRebuilt = 0;
159  $total = count( $codes );
160  $parentStatus = 0;
161 
162  if ( $threads <= 1 ) {
163  // Single-threaded implementation
164  $numRebuilt += $this->doRebuild( $codes, $lc, $force );
165  } else {
166  // Multi-threaded implementation
167  $chunks = array_chunk( $codes, ceil( count( $codes ) / $threads ) );
168  // Map from PID to readable socket
169  $sockets = [];
170 
171  foreach ( $chunks as $codes ) {
172  $socketpair = [];
173  // Create a pair of sockets so that the child can communicate
174  // the number of rebuilt langs to the parent.
175  if ( !socket_create_pair( AF_UNIX, SOCK_STREAM, 0, $socketpair ) ) {
176  $this->fatalError( 'socket_create_pair failed' );
177  }
178 
179  $pid = pcntl_fork();
180 
181  if ( $pid === -1 ) {
182  $this->fatalError( ' pcntl_fork failed' );
183  } elseif ( $pid === 0 ) {
184  // Child, reseed because there is no bug in PHP:
185  // https://bugs.php.net/bug.php?id=42465
186  mt_srand( getmypid() );
187 
188  $numRebuilt = $this->doRebuild( $codes, $lc, $force );
189  // Report the number of rebuilt langs to the parent.
190  $msg = "$numRebuilt\n";
191  socket_write( $socketpair[1], $msg, strlen( $msg ) );
192  // Child exits.
193  return;
194  } else {
195  // Main thread
196  $sockets[$pid] = $socketpair[0];
197  }
198  }
199 
200  // Wait for all children
201  foreach ( $sockets as $pid => $socket ) {
202  $status = 0;
203  pcntl_waitpid( $pid, $status );
204 
205  if ( pcntl_wifexited( $status ) ) {
206  $code = pcntl_wexitstatus( $status );
207  if ( $code ) {
208  $this->output( "Pid $pid exited with status $code !!\n" );
209  } else {
210  // Good exit status from child. Read the number of rebuilt langs from it.
211  $res = socket_read( $socket, 512, PHP_NORMAL_READ );
212  if ( $res === false ) {
213  $this->output( "socket_read failed in parent\n" );
214  } else {
215  $numRebuilt += intval( $res );
216  }
217  }
218 
219  // Mush all child statuses into a single value in the parent.
220  $parentStatus |= $code;
221  } elseif ( pcntl_wifsignaled( $status ) ) {
222  $signum = pcntl_wtermsig( $status );
223  $this->output( "Pid $pid terminated by signal $signum !!\n" );
224  $parentStatus |= 1;
225  }
226  }
227  }
228 
229  $this->output( "$numRebuilt languages rebuilt out of $total\n" );
230  if ( $numRebuilt === 0 ) {
231  $this->output( "Use --force to rebuild the caches which are still fresh.\n" );
232  }
233  if ( $parentStatus ) {
234  $this->fatalError( 'Failed.', $parentStatus );
235  }
236  }
237 
246  private function doRebuild( $codes, $lc, $force ) {
247  $numRebuilt = 0;
248  $operation = $this->hasOption( 'dry-run' ) ? "Would rebuild" : "Rebuilding";
249 
250  foreach ( $codes as $code ) {
251  if ( $force || $lc->isExpired( $code ) ) {
252  if ( !$this->hasOption( 'no-progress' ) ) {
253  $this->output( "$operation $code...\n" );
254  }
255  if ( !$this->hasOption( 'dry-run' ) ) {
256  $lc->recache( $code );
257  }
258  $numRebuilt++;
259  }
260  }
261 
262  return $numRebuilt;
263  }
264 
266  public function getDbType() {
267  if ( $this->hasOption( 'no-database' ) ) {
268  return Maintenance::DB_NONE;
269  }
270 
271  return parent::getDbType();
272  }
273 
279  public function setForce( $forced = true ) {
280  $this->mOptions['force'] = $forced;
281  }
282 }
283 
284 $maintClass = RebuildLocalisationCache::class;
285 require_once RUN_MAINTENANCE_IF_MAIN;
wfIsWindows()
Check if the operating system is Windows.
LocalisationCache optimised for loading many languages at once.
static getStoreFromConf(array $conf, $fallbackCacheDir)
Return a suitable LCStore as specified by the given configuration.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
const DB_NONE
Constants for DB access type.
Definition: Maintenance.php:71
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class for passing options to services.
A service that provides utilities to do with language names and codes.
PSR-3 logger instance factory.
A class containing constants representing the names of configuration variables.
This class generates message blobs for use by ResourceLoader.
Builder class for constructing a Config object from a set of sources during bootstrap.
Maintenance script to rebuild the localisation cache.
__construct()
Default constructor.
setForce( $forced=true)
Sets whether a run of this maintenance script has the force parameter set.
finalSetup(SettingsBuilder $settingsBuilder=null)
Handle some last-minute setup here.