Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
74.23% |
72 / 97 |
|
70.00% |
7 / 10 |
CRAP | |
0.00% |
0 / 1 |
ImportConstraintEntities | |
74.23% |
72 / 97 |
|
70.00% |
7 / 10 |
35.70 | |
0.00% |
0 / 1 |
__construct | |
100.00% |
16 / 16 |
|
100.00% |
1 / 1 |
1 | |||
setupServices | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
2 | |||
execute | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
6 | |||
getEntitiesToImport | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
4 | |||
importEntityFromWikidata | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
importEntityFromJson | |
90.00% |
18 / 20 |
|
0.00% |
0 / 1 |
5.03 | |||
storageExceptionToEntityId | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
2 | |||
outputConfigUpdates | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
20 | |||
outputConfigUpdatesGlobals | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
2 | |||
outputConfigUpdatesWgConf | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
2 |
1 | <?php |
2 | |
3 | namespace WikibaseQuality\ConstraintReport\Maintenance; |
4 | |
5 | use Deserializers\Deserializer; |
6 | use Maintenance; |
7 | use MediaWiki\Config\Config; |
8 | use MediaWiki\Http\HttpRequestFactory; |
9 | use MediaWiki\MediaWikiServices; |
10 | use MediaWiki\User\User; |
11 | use MediaWiki\WikiMap\WikiMap; |
12 | use Serializers\Serializer; |
13 | use Wikibase\DataModel\Entity\Item; |
14 | use Wikibase\DataModel\SiteLinkList; |
15 | use Wikibase\DataModel\Statement\StatementListProvider; |
16 | use Wikibase\Lib\Store\EntityStore; |
17 | use Wikibase\Lib\Store\StorageException; |
18 | use Wikibase\Repo\WikibaseRepo; |
19 | |
20 | // @codeCoverageIgnoreStart |
21 | $basePath = getenv( "MW_INSTALL_PATH" ) !== false |
22 | ? getenv( "MW_INSTALL_PATH" ) : __DIR__ . "/../../.."; |
23 | |
24 | require_once $basePath . "/maintenance/Maintenance.php"; |
25 | // @codeCoverageIgnoreEnd |
26 | |
27 | /** |
28 | * Imports entities needed for constraint checks from Wikidata into the local repository. |
29 | * |
30 | * @license GPL-2.0-or-later |
31 | */ |
32 | class ImportConstraintEntities extends Maintenance { |
33 | |
34 | /** |
35 | * @var Serializer |
36 | */ |
37 | private $entitySerializer; |
38 | |
39 | /** |
40 | * @var Deserializer |
41 | */ |
42 | private $entityDeserializer; |
43 | |
44 | /** |
45 | * @var EntityStore |
46 | */ |
47 | private $entityStore; |
48 | |
49 | /** |
50 | * @var HttpRequestFactory |
51 | */ |
52 | private $httpRequestFactory; |
53 | |
54 | /** |
55 | * @var User|null (null in dry-run mode, non-null otherwise) |
56 | */ |
57 | private $user; |
58 | |
59 | public function __construct() { |
60 | parent::__construct(); |
61 | |
62 | $this->addDescription( |
63 | 'Import entities needed for constraint checks ' . |
64 | 'from Wikidata into the local repository.' |
65 | ); |
66 | $this->addOption( |
67 | 'config-format', |
68 | 'The format in which the resulting configuration will be omitted: ' . |
69 | '"globals" for directly settings global variables, suitable for inclusion in LocalSettings.php (default), ' . |
70 | 'or "wgConf" for printing parts of arrays suitable for inclusion in $wgConf->settings.' |
71 | ); |
72 | $this->addOption( |
73 | 'dry-run', |
74 | 'Don’t actually import entities, just print which ones would be imported.' |
75 | ); |
76 | $this->requireExtension( 'WikibaseQualityConstraints' ); |
77 | } |
78 | |
79 | /** |
80 | * (This cannot happen in the constructor because the autoloader is not yet initialized there.) |
81 | */ |
82 | private function setupServices() { |
83 | $services = MediaWikiServices::getInstance(); |
84 | $this->entitySerializer = WikibaseRepo::getAllTypesEntitySerializer( $services ); |
85 | $this->entityDeserializer = WikibaseRepo::getInternalFormatEntityDeserializer( $services ); |
86 | $this->entityStore = WikibaseRepo::getEntityStore( $services ); |
87 | $this->httpRequestFactory = $services->getHttpRequestFactory(); |
88 | if ( !$this->getOption( 'dry-run', false ) ) { |
89 | $this->user = User::newSystemUser( 'WikibaseQualityConstraints importer' ); |
90 | } |
91 | } |
92 | |
93 | public function execute() { |
94 | $this->setupServices(); |
95 | |
96 | $configUpdates = []; |
97 | |
98 | $extensionJsonFile = __DIR__ . '/../extension.json'; |
99 | $extensionJsonText = file_get_contents( $extensionJsonFile ); |
100 | $extensionJson = json_decode( $extensionJsonText, /* assoc = */ true ); |
101 | // @phan-suppress-next-line PhanTypeArraySuspiciousNullable |
102 | $wikidataEntityIds = $this->getEntitiesToImport( $extensionJson['config'], $this->getConfig() ); |
103 | |
104 | foreach ( $wikidataEntityIds as $key => $wikidataEntityId ) { |
105 | $localEntityId = $this->importEntityFromWikidata( $wikidataEntityId ); |
106 | $configUpdates[$key] = [ |
107 | 'wikidata' => $wikidataEntityId, |
108 | 'local' => $localEntityId, |
109 | ]; |
110 | } |
111 | |
112 | $this->outputConfigUpdates( $configUpdates ); |
113 | } |
114 | |
115 | /** |
116 | * @param array[] $extensionJsonConfig |
117 | * @param Config $wikiConfig |
118 | * @return string[] |
119 | */ |
120 | private function getEntitiesToImport( array $extensionJsonConfig, Config $wikiConfig ) { |
121 | $wikidataEntityIds = []; |
122 | |
123 | foreach ( $extensionJsonConfig as $key => $value ) { |
124 | if ( !preg_match( '/Id$/', $key ) ) { |
125 | continue; |
126 | } |
127 | |
128 | $wikidataEntityId = $value['value']; |
129 | $localEntityId = $wikiConfig->get( $key ); |
130 | |
131 | if ( $localEntityId === $wikidataEntityId ) { |
132 | $wikidataEntityIds[$key] = $wikidataEntityId; |
133 | } |
134 | } |
135 | |
136 | return $wikidataEntityIds; |
137 | } |
138 | |
139 | /** |
140 | * @param string $wikidataEntityId |
141 | * @return string local entity ID |
142 | */ |
143 | private function importEntityFromWikidata( $wikidataEntityId ) { |
144 | $wikidataEntityUrl = "https://www.wikidata.org/wiki/Special:EntityData/$wikidataEntityId.json"; |
145 | $wikidataEntitiesJson = $this->httpRequestFactory->get( $wikidataEntityUrl, [], __METHOD__ ); |
146 | return $this->importEntityFromJson( $wikidataEntityId, $wikidataEntitiesJson ); |
147 | } |
148 | |
149 | /** |
150 | * @param string $wikidataEntityId |
151 | * @param string $wikidataEntitiesJson |
152 | * @return string local entity ID |
153 | */ |
154 | private function importEntityFromJson( $wikidataEntityId, $wikidataEntitiesJson ) { |
155 | // @phan-suppress-next-line PhanTypeArraySuspiciousNullable |
156 | $wikidataEntityArray = json_decode( $wikidataEntitiesJson, true )['entities'][$wikidataEntityId]; |
157 | $wikidataEntity = $this->entityDeserializer->deserialize( $wikidataEntityArray ); |
158 | |
159 | $wikidataEntity->setId( null ); |
160 | |
161 | if ( $wikidataEntity instanceof StatementListProvider ) { |
162 | $wikidataEntity->getStatements()->clear(); |
163 | } |
164 | |
165 | if ( $wikidataEntity instanceof Item ) { |
166 | $wikidataEntity->setSiteLinkList( new SiteLinkList() ); |
167 | } |
168 | |
169 | if ( $this->getOption( 'dry-run', false ) ) { |
170 | $wikidataEntityJson = json_encode( $this->entitySerializer->serialize( $wikidataEntity ) ); |
171 | $this->output( $wikidataEntityJson . "\n" ); |
172 | return "-$wikidataEntityId"; |
173 | } |
174 | |
175 | try { |
176 | $localEntity = $this->entityStore->saveEntity( |
177 | $wikidataEntity, |
178 | "imported from [[wikidata:$wikidataEntityId]]", |
179 | $this->user, |
180 | EDIT_NEW | EDIT_FORCE_BOT |
181 | )->getEntity(); |
182 | |
183 | return $localEntity->getId()->getSerialization(); |
184 | } catch ( StorageException $storageException ) { |
185 | return $this->storageExceptionToEntityId( $storageException ); |
186 | } |
187 | } |
188 | |
189 | private function storageExceptionToEntityId( StorageException $storageException ) { |
190 | $message = $storageException->getMessage(); |
191 | // example messages: |
192 | // * Item [[Item:Q475|Q475]] already has label "as references" |
193 | // associated with language code en, using the same description text. |
194 | // * Item [[Q475]] already has label "as references" |
195 | // associated with language code en, using the same description text. |
196 | // * Property [[Property:P694|P694]] already has label "instance of" |
197 | // associated with language code en. |
198 | $pattern = '/[[|]([^][|]*)]] already has label .* associated with language code/'; |
199 | if ( preg_match( $pattern, $message, $matches ) ) { |
200 | return $matches[1]; |
201 | } else { |
202 | throw $storageException; |
203 | } |
204 | } |
205 | |
206 | private function outputConfigUpdates( array $configUpdates ) { |
207 | $configFormat = $this->getOption( 'config-format', 'globals' ); |
208 | switch ( $configFormat ) { |
209 | case 'globals': |
210 | $this->outputConfigUpdatesGlobals( $configUpdates ); |
211 | break; |
212 | case 'wgConf': |
213 | $this->outputConfigUpdatesWgConf( $configUpdates ); |
214 | break; |
215 | default: |
216 | $this->error( "Invalid config format \"$configFormat\", using \"globals\"" ); |
217 | $this->outputConfigUpdatesGlobals( $configUpdates ); |
218 | break; |
219 | } |
220 | } |
221 | |
222 | /** |
223 | * @param array[] $configUpdates |
224 | */ |
225 | private function outputConfigUpdatesGlobals( array $configUpdates ) { |
226 | foreach ( $configUpdates as $key => $value ) { |
227 | $localValueCode = var_export( $value['local'], true ); |
228 | $this->output( "\$wg$key = $localValueCode;\n" ); |
229 | } |
230 | } |
231 | |
232 | /** |
233 | * @param array[] $configUpdates |
234 | */ |
235 | private function outputConfigUpdatesWgConf( array $configUpdates ) { |
236 | $wikiIdCode = var_export( WikiMap::getCurrentWikiId(), true ); |
237 | foreach ( $configUpdates as $key => $value ) { |
238 | $keyCode = var_export( "wg$key", true ); |
239 | $wikidataValueCode = var_export( $value['wikidata'], true ); |
240 | $localValueCode = var_export( $value['local'], true ); |
241 | $block = <<< EOF |
242 | $keyCode => [ |
243 | 'default' => $wikidataValueCode, |
244 | $wikiIdCode => $localValueCode, |
245 | ], |
246 | |
247 | |
248 | EOF; |
249 | $this->output( $block ); |
250 | } |
251 | } |
252 | |
253 | } |
254 | |
255 | // @codeCoverageIgnoreStart |
256 | $maintClass = ImportConstraintEntities::class; |
257 | require_once RUN_MAINTENANCE_IF_MAIN; |
258 | // @codeCoverageIgnoreEnd |