MediaWiki master
rebuildLocalisationCache.php
Go to the documentation of this file.
1<?php
2
39
40// @codeCoverageIgnoreStart
41require_once __DIR__ . '/Maintenance.php';
42// @codeCoverageIgnoreEnd
43
50 public function __construct() {
51 parent::__construct();
52 $this->addDescription( 'Rebuild the localisation cache' );
53 $this->addOption( 'dry-run', 'Determine what languages need to be rebuilt without changing anything' );
54 $this->addOption( 'force', 'Rebuild all files, even ones not out of date' );
55 $this->addOption( 'threads', 'Fork more than one thread', false, true );
56 $this->addOption( 'outdir', 'Override the output directory (normally $wgCacheDirectory)',
57 false, true );
58 $this->addOption( 'lang', 'Only rebuild these languages, comma separated.',
59 false, true );
60 $this->addOption(
61 'store-class',
62 'Override the LC store class (normally $wgLocalisationCacheConf[\'storeClass\'])',
63 false,
64 true
65 );
66 $this->addOption(
67 'no-database',
68 'EXPERIMENTAL: Disable the database backend. Setting this option will result in an error ' .
69 'if you have extensions or use site configuration that need the database. This is an ' .
70 'experimental feature to allow offline building of the localisation cache. Known limitations:' .
71 "\n" .
72 '* Incompatible with LCStoreDB, which always requires a database. ' . "\n" .
73 '* The message purge may require a database. See --skip-message-purge.'
74 );
75 // T237148: The Gadget extension (bundled with MediaWiki by default) requires a database`
76 // connection to register its modules for MessageBlobStore.
77 $this->addOption(
78 'skip-message-purge',
79 'Skip purging of MessageBlobStore. The purge operation may require a database, depending ' .
80 'on the configuration and extensions on this wiki. If skipping the purge now, you need to ' .
81 'run purgeMessageBlobStore.php shortly after deployment.'
82 );
83 $this->addOption(
84 'no-progress',
85 "Don't print a message for each rebuilt language file. Use this instead of " .
86 "--quiet to get a brief summary of the operation."
87 );
88 }
89
90 public function finalSetup( SettingsBuilder $settingsBuilder ) {
91 # This script needs to be run to build the initial l10n cache. But if
92 # LanguageCode is not 'en', it won't be able to run because there is
93 # no l10n cache. Break the cycle by forcing the LanguageCode setting to 'en'.
94 $settingsBuilder->putConfigValue( MainConfigNames::LanguageCode, 'en' );
95 parent::finalSetup( $settingsBuilder );
96 }
97
98 public function execute() {
99 $force = $this->hasOption( 'force' );
100 $threads = $this->getOption( 'threads', 1 );
101 if ( $threads < 1 || $threads != intval( $threads ) ) {
102 $this->output( "Invalid thread count specified; running single-threaded.\n" );
103 $threads = 1;
104 }
105 if ( $threads > 1 && wfIsWindows() ) {
106 $this->output( "Threaded rebuild is not supported on Windows; running single-threaded.\n" );
107 $threads = 1;
108 }
109 if ( $threads > 1 && ( !extension_loaded( 'sockets' ) || !function_exists( 'pcntl_fork' ) ) ) {
110 $this->output( "Threaded rebuild requires ext-pcntl and ext-sockets; running single-threaded.\n" );
111 $threads = 1;
112 }
113
114 $conf = $this->getConfig()->get( MainConfigNames::LocalisationCacheConf );
115 // Allow fallbacks to create CDB files
116 $conf['manualRecache'] = false;
117 $conf['forceRecache'] = $force || !empty( $conf['forceRecache'] );
118 if ( $this->hasOption( 'outdir' ) ) {
119 $conf['storeDirectory'] = $this->getOption( 'outdir' );
120 }
121
122 if ( $this->hasOption( 'store-class' ) ) {
123 $conf['storeClass'] = $this->getOption( 'store-class' );
124 }
125
126 // XXX Copy-pasted from ServiceWiring.php. Do we need a factory for this one caller?
127 $services = $this->getServiceContainer();
129 new ServiceOptions(
130 LocalisationCache::CONSTRUCTOR_OPTIONS,
131 $conf,
132 $services->getMainConfig()
133 ),
134 LocalisationCache::getStoreFromConf( $conf, $this->getConfig()->get( MainConfigNames::CacheDirectory ) ),
135 LoggerFactory::getInstance( 'localisation' ),
136 $this->hasOption( 'skip-message-purge' ) ? [] :
137 [ static function () use ( $services ) {
138 MessageBlobStore::clearGlobalCacheEntry( $services->getMainWANObjectCache() );
139 } ],
140 $services->getLanguageNameUtils(),
141 $services->getHookContainer()
142 );
143
144 $allCodes = array_keys( $services
145 ->getLanguageNameUtils()
146 ->getLanguageNames( LanguageNameUtils::AUTONYMS, LanguageNameUtils::SUPPORTED ) );
147 if ( $this->hasOption( 'lang' ) ) {
148 # Validate requested languages
149 $codes = array_intersect( $allCodes,
150 explode( ',', $this->getOption( 'lang' ) ) );
151 # Bailed out if nothing is left
152 if ( count( $codes ) == 0 ) {
153 $this->fatalError( 'None of the languages specified exists.' );
154 }
155 } else {
156 # By default get all languages
157 $codes = $allCodes;
158 }
159 sort( $codes );
160
161 $numRebuilt = 0;
162 $total = count( $codes );
163 $parentStatus = 0;
164
165 if ( $threads <= 1 ) {
166 // Single-threaded implementation
167 $numRebuilt += $this->doRebuild( $codes, $lc, $force );
168 } else {
169 // Multi-threaded implementation
170 $chunks = array_chunk( $codes, ceil( count( $codes ) / $threads ) );
171 // Map from PID to readable socket
172 $sockets = [];
173
174 foreach ( $chunks as $codes ) {
175 $socketpair = [];
176 // Create a pair of sockets so that the child can communicate
177 // the number of rebuilt langs to the parent.
178 if ( !socket_create_pair( AF_UNIX, SOCK_STREAM, 0, $socketpair ) ) {
179 $this->fatalError( 'socket_create_pair failed' );
180 }
181
182 $pid = pcntl_fork();
183
184 if ( $pid === -1 ) {
185 $this->fatalError( ' pcntl_fork failed' );
186 } elseif ( $pid === 0 ) {
187 // Child, reseed because there is no bug in PHP:
188 // https://bugs.php.net/bug.php?id=42465
189 mt_srand( getmypid() );
190
191 $numRebuilt = $this->doRebuild( $codes, $lc, $force );
192 // Report the number of rebuilt langs to the parent.
193 $msg = "$numRebuilt\n";
194 socket_write( $socketpair[1], $msg, strlen( $msg ) );
195 // Child exits.
196 return;
197 } else {
198 // Main thread
199 $sockets[$pid] = $socketpair[0];
200 }
201 }
202
203 // Wait for all children
204 foreach ( $sockets as $pid => $socket ) {
205 $status = 0;
206 pcntl_waitpid( $pid, $status );
207
208 if ( pcntl_wifexited( $status ) ) {
209 $code = pcntl_wexitstatus( $status );
210 if ( $code ) {
211 $this->output( "Pid $pid exited with status $code !!\n" );
212 } else {
213 // Good exit status from child. Read the number of rebuilt langs from it.
214 $res = socket_read( $socket, 512, PHP_NORMAL_READ );
215 if ( $res === false ) {
216 $this->output( "socket_read failed in parent\n" );
217 } else {
218 $numRebuilt += intval( $res );
219 }
220 }
221
222 // Mush all child statuses into a single value in the parent.
223 $parentStatus |= $code;
224 } elseif ( pcntl_wifsignaled( $status ) ) {
225 $signum = pcntl_wtermsig( $status );
226 $this->output( "Pid $pid terminated by signal $signum !!\n" );
227 $parentStatus |= 1;
228 }
229 }
230 }
231
232 $this->output( "$numRebuilt languages rebuilt out of $total\n" );
233 if ( $numRebuilt === 0 ) {
234 $this->output( "Use --force to rebuild the caches which are still fresh.\n" );
235 }
236 if ( $parentStatus ) {
237 $this->fatalError( 'Failed.', $parentStatus );
238 }
239 }
240
249 private function doRebuild( $codes, $lc, $force ) {
250 $numRebuilt = 0;
251 $operation = $this->hasOption( 'dry-run' ) ? "Would rebuild" : "Rebuilding";
252
253 foreach ( $codes as $code ) {
254 if ( $force || $lc->isExpired( $code ) ) {
255 if ( !$this->hasOption( 'no-progress' ) ) {
256 $this->output( "$operation $code...\n" );
257 }
258 if ( !$this->hasOption( 'dry-run' ) ) {
259 $lc->recache( $code );
260 }
261 $numRebuilt++;
262 }
263 }
264
265 return $numRebuilt;
266 }
267
269 public function getDbType() {
270 if ( $this->hasOption( 'no-database' ) ) {
271 return Maintenance::DB_NONE;
272 }
273
274 return parent::getDbType();
275 }
276
282 public function setForce( $forced = true ) {
283 $this->setOption( 'force', $forced );
284 }
285}
286
287// @codeCoverageIgnoreStart
288$maintClass = RebuildLocalisationCache::class;
289require_once RUN_MAINTENANCE_IF_MAIN;
290// @codeCoverageIgnoreEnd
wfIsWindows()
Check if the operating system is Windows.
LocalisationCache optimised for loading many languages at once.
A class for passing options to services.
A service that provides utilities to do with language names and codes.
Create PSR-3 logger objects.
A class containing constants representing the names of configuration variables.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
output( $out, $channel=null)
Throw some output to the user.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
hasOption( $name)
Checks to see if a particular option was set.
setOption(string $name, $value)
Programmatically set the value of the given option.
getOption( $name, $default=null)
Get an option, or return the default.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
This class generates message blobs for use by ResourceLoader.
Builder class for constructing a Config object from a set of sources during bootstrap.
putConfigValue(string $key, $value)
Puts a value into a config variable.
Maintenance script to rebuild the localisation cache.
finalSetup(SettingsBuilder $settingsBuilder)
Handle some last-minute setup here.
getDbType()
Does the script need different DB access? By default, we give Maintenance scripts normal rights to th...
setForce( $forced=true)
Sets whether a run of this maintenance script has the force parameter set.