Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
59.64% |
99 / 166 |
|
11.11% |
1 / 9 |
CRAP | |
0.00% |
0 / 1 |
| SchemaDump | |
59.64% |
99 / 166 |
|
11.11% |
1 / 9 |
84.30 | |
0.00% |
0 / 1 |
| execute | |
66.67% |
6 / 9 |
|
0.00% |
0 / 1 |
3.33 | |||
| fetchFromCluster | |
0.00% |
0 / 27 |
|
0.00% |
0 / 1 |
30 | |||
| buildFromCode | |
66.67% |
6 / 9 |
|
0.00% |
0 / 1 |
3.33 | |||
| getBuildContext | |
79.31% |
23 / 29 |
|
0.00% |
0 / 1 |
5.22 | |||
| buildSchemaForIndex | |
55.26% |
21 / 38 |
|
0.00% |
0 / 1 |
3.81 | |||
| buildCompleteSettings | |
93.94% |
31 / 33 |
|
0.00% |
0 / 1 |
7.01 | |||
| getAllowedParams | |
100.00% |
12 / 12 |
|
100.00% |
1 / 1 |
1 | |||
| isInternal | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| getExamplesMessages | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
2 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch\Api; |
| 4 | |
| 5 | use CirrusSearch\CirrusConfigNames; |
| 6 | use CirrusSearch\Connection; |
| 7 | use CirrusSearch\Maintenance\AnalysisConfigBuilder; |
| 8 | use CirrusSearch\Maintenance\ArchiveMappingConfigBuilder; |
| 9 | use CirrusSearch\Maintenance\ConfigUtils; |
| 10 | use CirrusSearch\Maintenance\MappingConfigBuilder; |
| 11 | use CirrusSearch\Maintenance\NullPrinter; |
| 12 | use CirrusSearch\Maintenance\SuggesterAnalysisConfigBuilder; |
| 13 | use CirrusSearch\Maintenance\SuggesterMappingConfigBuilder; |
| 14 | use CirrusSearch\ReplicaCount; |
| 15 | use CirrusSearch\SearchConfig; |
| 16 | use MediaWiki\Api\ApiBase; |
| 17 | use MediaWiki\MainConfigNames; |
| 18 | use Wikimedia\ParamValidator\ParamValidator; |
| 19 | |
| 20 | /** |
| 21 | * Dumps CirrusSearch mappings for easy viewing. |
| 22 | * |
| 23 | * @license GPL-2.0-or-later |
| 24 | */ |
| 25 | class SchemaDump extends ApiBase { |
| 26 | use ApiTrait; |
| 27 | |
| 28 | public function execute() { |
| 29 | $build = $this->getParameter( 'build' ); |
| 30 | $conn = $this->getCirrusConnection(); |
| 31 | $indexPrefix = $this->getSearchConfig()->get( SearchConfig::INDEX_BASE_NAME ); |
| 32 | |
| 33 | // Get all index suffixes to process |
| 34 | $indexSuffixes = $conn->getAllIndexSuffixes( null ); |
| 35 | |
| 36 | if ( $build ) { |
| 37 | try { |
| 38 | $this->buildFromCode( $conn, $indexPrefix, $indexSuffixes ); |
| 39 | } catch ( \InvalidArgumentException $e ) { |
| 40 | $this->dieWithException( $e ); |
| 41 | } |
| 42 | } else { |
| 43 | $this->fetchFromCluster( $conn, $indexPrefix, $indexSuffixes ); |
| 44 | } |
| 45 | } |
| 46 | |
| 47 | /** |
| 48 | * Fetch schema (settings and mappings) from live cluster indices |
| 49 | * |
| 50 | * @param Connection $conn CirrusSearch connection |
| 51 | * @param string $indexPrefix Index prefix (typically wiki ID) |
| 52 | * @param array $indexSuffixes Index suffixes to process |
| 53 | */ |
| 54 | private function fetchFromCluster( Connection $conn, string $indexPrefix, array $indexSuffixes ): void { |
| 55 | foreach ( $indexSuffixes as $suffix ) { |
| 56 | $index = $conn->getIndex( $indexPrefix, $suffix ); |
| 57 | |
| 58 | if ( !$index->exists() ) { |
| 59 | continue; |
| 60 | } |
| 61 | |
| 62 | $settings = $index->getSettings()->get(); |
| 63 | $mappings = $index->getMapping(); |
| 64 | |
| 65 | $this->getResult()->addValue( |
| 66 | null, |
| 67 | $suffix, |
| 68 | [ |
| 69 | 'settings' => [ 'index' => $settings ], |
| 70 | 'mappings' => $mappings |
| 71 | ] |
| 72 | ); |
| 73 | } |
| 74 | |
| 75 | // Handle completion suggester if enabled |
| 76 | if ( $this->getSearchConfig()->isCompletionSuggesterEnabled() ) { |
| 77 | $index = $conn->getIndex( $indexPrefix, Connection::TITLE_SUGGEST_INDEX_SUFFIX ); |
| 78 | if ( $index->exists() ) { |
| 79 | $settings = $index->getSettings()->get(); |
| 80 | $mappings = $index->getMapping(); |
| 81 | |
| 82 | $this->getResult()->addValue( |
| 83 | null, |
| 84 | Connection::TITLE_SUGGEST_INDEX_SUFFIX, |
| 85 | [ |
| 86 | 'settings' => [ 'index' => $settings ], |
| 87 | 'mappings' => $mappings |
| 88 | ] |
| 89 | ); |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | /** |
| 95 | * Build schema (settings and mappings) from code and configuration |
| 96 | * |
| 97 | * @param Connection $conn CirrusSearch connection |
| 98 | * @param string $indexPrefix Index prefix (typically wiki ID) |
| 99 | * @param array $indexSuffixes Index suffixes to process |
| 100 | */ |
| 101 | private function buildFromCode( Connection $conn, string $indexPrefix, array $indexSuffixes ): void { |
| 102 | $buildContext = $this->getBuildContext( $conn ); |
| 103 | |
| 104 | foreach ( $indexSuffixes as $suffix ) { |
| 105 | $schema = $this->buildSchemaForIndex( $suffix, $buildContext ); |
| 106 | $indexName = $indexPrefix . '_' . $suffix; |
| 107 | |
| 108 | $this->getResult()->addValue( null, $suffix, $schema ); |
| 109 | } |
| 110 | |
| 111 | // Handle completion suggester if enabled |
| 112 | if ( $this->getSearchConfig()->isCompletionSuggesterEnabled() ) { |
| 113 | $suffix = Connection::TITLE_SUGGEST_INDEX_SUFFIX; |
| 114 | $schema = $this->buildSchemaForIndex( $suffix, $buildContext ); |
| 115 | |
| 116 | $this->getResult()->addValue( null, $suffix, $schema ); |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | /** |
| 121 | * Gather all context needed for building schema from code |
| 122 | * |
| 123 | * @param Connection $conn CirrusSearch connection |
| 124 | * @return array Build context with language code, plugins, flags, settings |
| 125 | */ |
| 126 | private function getBuildContext( Connection $conn ): array { |
| 127 | $config = $this->getSearchConfig(); |
| 128 | |
| 129 | // Get language code |
| 130 | $langCode = $config->get( MainConfigNames::LanguageCode ); |
| 131 | |
| 132 | // Check if plugins were provided via API parameter |
| 133 | $pluginsParam = $this->getParameter( 'plugins' ); |
| 134 | |
| 135 | $utils = new ConfigUtils( $conn->getClient(), new NullPrinter() ); |
| 136 | $bannedPlugins = $config->get( CirrusConfigNames::BannedPlugins ); |
| 137 | if ( $pluginsParam !== null ) { |
| 138 | $plugins = $utils->removeBannedPlugins( $pluginsParam, $bannedPlugins ); |
| 139 | } else { |
| 140 | // Fall back to scanning cluster |
| 141 | $pluginStatus = $utils->scanAvailablePlugins( $bannedPlugins ); |
| 142 | if ( !$pluginStatus->isOK() ) { |
| 143 | $this->dieStatus( $pluginStatus ); |
| 144 | } |
| 145 | $plugins = $pluginStatus->getValue(); |
| 146 | } |
| 147 | |
| 148 | // Get config flags |
| 149 | $flags = 0; |
| 150 | if ( $config->get( CirrusConfigNames::PrefixSearchStartsWithAnyWord ) ) { |
| 151 | $flags |= MappingConfigBuilder::PREFIX_START_WITH_ANY; |
| 152 | } |
| 153 | if ( $config->get( CirrusConfigNames::PhraseSuggestUseText ) ) { |
| 154 | $flags |= MappingConfigBuilder::PHRASE_SUGGEST_USE_TEXT; |
| 155 | } |
| 156 | |
| 157 | $optimizeForHighlighter = $config->get( CirrusConfigNames::OptimizeIndexForExperimentalHighlighter ); |
| 158 | |
| 159 | // Get replica and refresh settings |
| 160 | $replicas = $config->get( CirrusConfigNames::Replicas ); |
| 161 | $refreshInterval = $config->get( CirrusConfigNames::RefreshInterval ); |
| 162 | |
| 163 | return [ |
| 164 | 'langCode' => $langCode, |
| 165 | 'plugins' => $plugins, |
| 166 | 'flags' => $flags, |
| 167 | 'optimizeForHighlighter' => $optimizeForHighlighter, |
| 168 | 'replicas' => $replicas, |
| 169 | 'refreshInterval' => $refreshInterval, |
| 170 | 'config' => $config, |
| 171 | 'conn' => $conn, |
| 172 | ]; |
| 173 | } |
| 174 | |
| 175 | /** |
| 176 | * Build complete schema for a specific index type |
| 177 | * |
| 178 | * @param string $indexSuffix Index suffix (content, general, archive, titlesuggest) |
| 179 | * @param array $context Build context from getBuildContext() |
| 180 | * @return array Schema with 'settings' and 'mappings' keys |
| 181 | */ |
| 182 | private function buildSchemaForIndex( string $indexSuffix, array $context ): array { |
| 183 | $config = $context['config']; |
| 184 | |
| 185 | // Select appropriate builders based on index type |
| 186 | if ( $indexSuffix === Connection::TITLE_SUGGEST_INDEX_SUFFIX ) { |
| 187 | $analysisBuilder = new SuggesterAnalysisConfigBuilder( |
| 188 | $context['langCode'], |
| 189 | $context['plugins'], |
| 190 | $config |
| 191 | ); |
| 192 | $mappingBuilder = new SuggesterMappingConfigBuilder( $config ); |
| 193 | } elseif ( $indexSuffix === Connection::ARCHIVE_INDEX_SUFFIX ) { |
| 194 | $analysisBuilder = new AnalysisConfigBuilder( |
| 195 | $context['langCode'], |
| 196 | $context['plugins'], |
| 197 | $config |
| 198 | ); |
| 199 | $mappingBuilder = new ArchiveMappingConfigBuilder( |
| 200 | $context['optimizeForHighlighter'], |
| 201 | $context['plugins'], |
| 202 | $context['flags'], |
| 203 | $config |
| 204 | ); |
| 205 | } else { |
| 206 | // Content or general index |
| 207 | $analysisBuilder = new AnalysisConfigBuilder( |
| 208 | $context['langCode'], |
| 209 | $context['plugins'], |
| 210 | $config |
| 211 | ); |
| 212 | $mappingBuilder = new MappingConfigBuilder( |
| 213 | $context['optimizeForHighlighter'], |
| 214 | $context['plugins'], |
| 215 | $context['flags'], |
| 216 | $config |
| 217 | ); |
| 218 | } |
| 219 | |
| 220 | // Build analysis and mappings |
| 221 | $analysisConfig = $analysisBuilder->buildConfig(); |
| 222 | $mappings = $mappingBuilder->buildConfig(); |
| 223 | |
| 224 | // Build complete settings structure |
| 225 | $settings = $this->buildCompleteSettings( $analysisConfig, $indexSuffix, $context ); |
| 226 | |
| 227 | return [ |
| 228 | 'settings' => $settings, |
| 229 | 'mappings' => $mappings |
| 230 | ]; |
| 231 | } |
| 232 | |
| 233 | /** |
| 234 | * Construct complete settings structure matching IndexCreator |
| 235 | * |
| 236 | * @param array $analysisConfig Analysis configuration (analyzers, tokenizers, filters) |
| 237 | * @param string $indexSuffix Index suffix for getting type-specific settings |
| 238 | * @param array $context Build context from getBuildContext() |
| 239 | * @return array Complete settings structure |
| 240 | */ |
| 241 | private function buildCompleteSettings( array $analysisConfig, string $indexSuffix, array $context ): array { |
| 242 | $config = $context['config']; |
| 243 | $conn = $context['conn']; |
| 244 | |
| 245 | // Get shard count for this specific index type |
| 246 | $shardCount = $conn->getSettings()->getShardCount( $indexSuffix ); |
| 247 | |
| 248 | // Get replica count for this index type |
| 249 | $replicas = $context['replicas']; |
| 250 | $replicaCount = ReplicaCount::fromConfigValue( |
| 251 | is_array( $replicas ) && isset( $replicas[$indexSuffix] ) |
| 252 | ? $replicas[$indexSuffix] |
| 253 | : '0-2' |
| 254 | ); |
| 255 | |
| 256 | // Build similarity config |
| 257 | $analysisBuilder = new AnalysisConfigBuilder( |
| 258 | $context['langCode'], |
| 259 | $context['plugins'], |
| 260 | $config |
| 261 | ); |
| 262 | $similarityConfig = $analysisBuilder->buildSimilarityConfig(); |
| 263 | |
| 264 | // Base settings structure |
| 265 | $indexSettings = [ |
| 266 | 'number_of_shards' => $shardCount, |
| 267 | 'refresh_interval' => $context['refreshInterval'] . 's', |
| 268 | 'analysis' => $analysisConfig, |
| 269 | 'query' => [ |
| 270 | 'default_field' => 'all' |
| 271 | ], |
| 272 | ] + $replicaCount->toCreateSettings(); |
| 273 | |
| 274 | // Add similarity if present |
| 275 | if ( $similarityConfig ) { |
| 276 | $indexSettings['similarity'] = $similarityConfig; |
| 277 | } |
| 278 | |
| 279 | // Add merge settings if configured |
| 280 | $mergeSettings = $config->get( CirrusConfigNames::MergeSettings ); |
| 281 | if ( is_array( $mergeSettings ) && isset( $mergeSettings[$indexSuffix] ) ) { |
| 282 | $indexSettings['merge'] = [ 'policy' => $mergeSettings[$indexSuffix] ]; |
| 283 | } |
| 284 | |
| 285 | // Wrap in 'index' key |
| 286 | $settings = [ 'index' => $indexSettings ]; |
| 287 | |
| 288 | // Add extra settings if configured |
| 289 | $extraSettings = $config->get( CirrusConfigNames::ExtraIndexSettings ); |
| 290 | if ( is_array( $extraSettings ) ) { |
| 291 | $settings = array_merge( $settings, $extraSettings ); |
| 292 | } |
| 293 | |
| 294 | return $settings; |
| 295 | } |
| 296 | |
| 297 | /** @inheritDoc */ |
| 298 | public function getAllowedParams() { |
| 299 | return [ |
| 300 | 'build' => [ |
| 301 | ParamValidator::PARAM_DEFAULT => false, |
| 302 | ParamValidator::PARAM_TYPE => 'boolean', |
| 303 | ], |
| 304 | 'plugins' => [ |
| 305 | ParamValidator::PARAM_TYPE => 'string', |
| 306 | ParamValidator::PARAM_ISMULTI => true, |
| 307 | ParamValidator::PARAM_DEFAULT => null, |
| 308 | ParamValidator::PARAM_ALLOW_DUPLICATES => false, |
| 309 | ], |
| 310 | ]; |
| 311 | } |
| 312 | |
| 313 | /** |
| 314 | * Mark as internal. This isn't meant to be used by normal api users |
| 315 | * @return bool |
| 316 | */ |
| 317 | public function isInternal() { |
| 318 | return true; |
| 319 | } |
| 320 | |
| 321 | /** |
| 322 | * @see ApiBase::getExamplesMessages |
| 323 | * @return array |
| 324 | */ |
| 325 | protected function getExamplesMessages() { |
| 326 | return [ |
| 327 | 'action=cirrus-schema-dump' => |
| 328 | 'apihelp-cirrus-schema-dump-example', |
| 329 | 'action=cirrus-schema-dump&build=true' => |
| 330 | 'apihelp-cirrus-schema-dump-example-build', |
| 331 | 'action=cirrus-schema-dump&build=true&plugins=analysis-icu|extra-analysis-textify' => |
| 332 | 'apihelp-cirrus-schema-dump-example-build-plugins', |
| 333 | ]; |
| 334 | } |
| 335 | |
| 336 | } |