Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
89.47% |
153 / 171 |
|
50.00% |
5 / 10 |
CRAP | |
0.00% |
0 / 1 |
Importer | |
89.47% |
153 / 171 |
|
50.00% |
5 / 10 |
26.79 | |
0.00% |
0 / 1 |
__construct | |
100.00% |
12 / 12 |
|
100.00% |
1 / 1 |
1 | |||
import | |
100.00% |
48 / 48 |
|
100.00% |
1 / 1 |
1 | |||
buildImportOperations | |
100.00% |
40 / 40 |
|
100.00% |
1 / 1 |
3 | |||
prepareImportOperations | |
33.33% |
2 / 6 |
|
0.00% |
0 / 1 |
3.19 | |||
validateImportOperations | |
100.00% |
17 / 17 |
|
100.00% |
1 / 1 |
10 | |||
commitImportOperations | |
33.33% |
2 / 6 |
|
0.00% |
0 / 1 |
3.19 | |||
validateFileInfoText | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
1 | |||
getPageFromImportPlan | |
57.14% |
4 / 7 |
|
0.00% |
0 / 1 |
2.31 | |||
createPostImportNullRevision | |
78.57% |
11 / 14 |
|
0.00% |
0 / 1 |
2.04 | |||
createPostImportEdit | |
69.23% |
9 / 13 |
|
0.00% |
0 / 1 |
2.12 |
1 | <?php |
2 | |
3 | namespace FileImporter\Services; |
4 | |
5 | use FileImporter\Data\ImportDetails; |
6 | use FileImporter\Data\ImportOperations; |
7 | use FileImporter\Data\ImportPlan; |
8 | use FileImporter\Exceptions\AbuseFilterWarningsException; |
9 | use FileImporter\Exceptions\ImportException; |
10 | use FileImporter\Exceptions\LocalizedImportException; |
11 | use FileImporter\Operations\FileRevisionFromRemoteUrl; |
12 | use FileImporter\Operations\TextRevisionFromTextRevision; |
13 | use FileImporter\Services\Http\HttpRequestExecutor; |
14 | use FileImporter\Services\UploadBase\UploadBaseFactory; |
15 | use IApiMessage; |
16 | use IDBAccessObject; |
17 | use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface; |
18 | use MediaWiki\MediaWikiServices; |
19 | use MediaWiki\Page\WikiPageFactory; |
20 | use MediaWiki\Permissions\Authority; |
21 | use MediaWiki\Permissions\RestrictionStore; |
22 | use MediaWiki\Status\Status; |
23 | use MediaWiki\Title\Title; |
24 | use MediaWiki\User\User; |
25 | use MediaWiki\User\UserIdentity; |
26 | use MediaWiki\User\UserIdentityLookup; |
27 | use NullStatsdDataFactory; |
28 | use OldRevisionImporter; |
29 | use Psr\Log\LoggerInterface; |
30 | use Psr\Log\NullLogger; |
31 | use RuntimeException; |
32 | use StatusValue; |
33 | use UploadRevisionImporter; |
34 | use WikiPage; |
35 | |
36 | /** |
37 | * Performs an import of a file to the local wiki based on an ImportPlan object for a given User. |
38 | * |
39 | * @license GPL-2.0-or-later |
40 | * @author Addshore |
41 | */ |
42 | class Importer { |
43 | |
44 | private const ERROR_NO_NEW_PAGE = 'noPageCreated'; |
45 | |
46 | private WikiPageFactory $wikiPageFactory; |
47 | private WikiRevisionFactory $wikiRevisionFactory; |
48 | private NullRevisionCreator $nullRevisionCreator; |
49 | private UserIdentityLookup $userLookup; |
50 | private HttpRequestExecutor $httpRequestExecutor; |
51 | private UploadBaseFactory $uploadBaseFactory; |
52 | private OldRevisionImporter $oldRevisionImporter; |
53 | private UploadRevisionImporter $uploadRevisionImporter; |
54 | private FileTextRevisionValidator $textRevisionValidator; |
55 | private RestrictionStore $restrictionStore; |
56 | private LoggerInterface $logger; |
57 | private StatsdDataFactoryInterface $stats; |
58 | |
59 | public function __construct( |
60 | WikiPageFactory $wikiPageFactory, |
61 | WikiRevisionFactory $wikiRevisionFactory, |
62 | NullRevisionCreator $nullRevisionCreator, |
63 | UserIdentityLookup $userLookup, |
64 | HttpRequestExecutor $httpRequestExecutor, |
65 | UploadBaseFactory $uploadBaseFactory, |
66 | OldRevisionImporter $oldRevisionImporter, |
67 | UploadRevisionImporter $uploadRevisionImporter, |
68 | FileTextRevisionValidator $textRevisionValidator, |
69 | RestrictionStore $restrictionStore, |
70 | LoggerInterface $logger = null, |
71 | StatsdDataFactoryInterface $statsdDataFactory = null |
72 | ) { |
73 | $this->wikiPageFactory = $wikiPageFactory; |
74 | $this->wikiRevisionFactory = $wikiRevisionFactory; |
75 | $this->nullRevisionCreator = $nullRevisionCreator; |
76 | $this->userLookup = $userLookup; |
77 | $this->httpRequestExecutor = $httpRequestExecutor; |
78 | $this->uploadBaseFactory = $uploadBaseFactory; |
79 | $this->oldRevisionImporter = $oldRevisionImporter; |
80 | $this->uploadRevisionImporter = $uploadRevisionImporter; |
81 | $this->textRevisionValidator = $textRevisionValidator; |
82 | $this->restrictionStore = $restrictionStore; |
83 | $this->logger = $logger ?? new NullLogger(); |
84 | $this->stats = $statsdDataFactory ?? new NullStatsdDataFactory(); |
85 | } |
86 | |
87 | /** |
88 | * @param User $user user to use for the import |
89 | * @param ImportPlan $importPlan A valid ImportPlan object. |
90 | * |
91 | * @throws ImportException|RuntimeException |
92 | */ |
93 | public function import( User $user, ImportPlan $importPlan ) { |
94 | $this->wikiRevisionFactory->setInterWikiPrefix( $importPlan->getInterWikiPrefix() ); |
95 | |
96 | $importStart = microtime( true ); |
97 | $this->logger->info( __METHOD__ . ' started' ); |
98 | |
99 | $validationStatus = $this->validateFileInfoText( |
100 | $user, |
101 | $importPlan |
102 | ); |
103 | |
104 | // TODO the type of ImportOperation created should be decided somewhere |
105 | |
106 | $operationBuildingStart = microtime( true ); |
107 | $importOperations = $this->buildImportOperations( |
108 | $user, |
109 | $importPlan->getTitle(), |
110 | $importPlan->getDetails() |
111 | ); |
112 | $this->stats->timing( |
113 | 'FileImporter.import.timing.buildOperations', |
114 | ( microtime( true ) - $operationBuildingStart ) * 1000 |
115 | ); |
116 | |
117 | $operationPrepareStart = microtime( true ); |
118 | $this->prepareImportOperations( $importOperations ); |
119 | $this->stats->timing( |
120 | 'FileImporter.import.timing.prepareOperations', |
121 | ( microtime( true ) - $operationPrepareStart ) * 1000 |
122 | ); |
123 | |
124 | $operationValidateStart = microtime( true ); |
125 | $validationStatus->merge( $importOperations->validate() ); |
126 | $this->validateImportOperations( $validationStatus, $importPlan ); |
127 | $this->stats->timing( |
128 | 'FileImporter.import.timing.validateOperations', |
129 | ( microtime( true ) - $operationValidateStart ) * 1000 |
130 | ); |
131 | |
132 | $operationCommitStart = microtime( true ); |
133 | $this->commitImportOperations( $importOperations ); |
134 | $this->stats->timing( |
135 | 'FileImporter.import.timing.commitOperations', |
136 | ( microtime( true ) - $operationCommitStart ) * 1000 |
137 | ); |
138 | |
139 | // TODO the below should be an ImportOperation |
140 | $miscActionsStart = microtime( true ); |
141 | $page = $this->getPageFromImportPlan( $importPlan ); |
142 | $this->createPostImportNullRevision( $importPlan, $user ); |
143 | $this->createPostImportEdit( $importPlan, $page, $user ); |
144 | $this->stats->timing( |
145 | 'FileImporter.import.timing.miscActions', |
146 | ( microtime( true ) - $miscActionsStart ) * 1000 |
147 | ); |
148 | |
149 | // TODO do we need to call WikiImporter::finishImportPage?? |
150 | // TODO factor logic in WikiImporter::finishImportPage out so we can call it |
151 | |
152 | $this->stats->timing( |
153 | 'FileImporter.import.timing.wholeImport', |
154 | ( microtime( true ) - $importStart ) * 1000 |
155 | ); |
156 | } |
157 | |
158 | /** |
159 | * @return ImportOperations |
160 | */ |
161 | private function buildImportOperations( |
162 | User $user, |
163 | Title $plannedTitle, |
164 | ImportDetails $importDetails |
165 | ) { |
166 | $textRevisions = $importDetails->getTextRevisions()->toArray(); |
167 | $fileRevisions = $importDetails->getFileRevisions()->toArray(); |
168 | $importOperations = new ImportOperations(); |
169 | |
170 | /** |
171 | * Text revisions should be added first. See T147451. |
172 | * This ensures that the page entry is created and if something fails it can thus be deleted. |
173 | */ |
174 | foreach ( $textRevisions as $textRevision ) { |
175 | $importOperations->add( new TextRevisionFromTextRevision( |
176 | $plannedTitle, |
177 | $user, |
178 | $textRevision, |
179 | $this->wikiRevisionFactory, |
180 | $this->oldRevisionImporter, |
181 | $this->textRevisionValidator, |
182 | $this->restrictionStore, |
183 | $this->logger |
184 | ) ); |
185 | } |
186 | |
187 | $totalFileSizes = 0; |
188 | $initialTextRevision = $textRevisions[0] ?? null; |
189 | |
190 | foreach ( $fileRevisions as $fileRevision ) { |
191 | $totalFileSizes += $fileRevision->getField( 'size' ); |
192 | $this->stats->gauge( |
193 | 'FileImporter.import.details.individualFileSizes', |
194 | $fileRevision->getField( 'size' ) |
195 | ); |
196 | $importOperations->add( new FileRevisionFromRemoteUrl( |
197 | $plannedTitle, |
198 | $user, |
199 | $fileRevision, |
200 | $initialTextRevision, |
201 | $this->userLookup, |
202 | $this->httpRequestExecutor, |
203 | $this->wikiRevisionFactory, |
204 | $this->uploadBaseFactory, |
205 | $this->uploadRevisionImporter, |
206 | $this->restrictionStore, |
207 | $this->logger |
208 | ) ); |
209 | |
210 | // only include the initial text revision in the first upload |
211 | $initialTextRevision = null; |
212 | } |
213 | |
214 | $this->stats->gauge( 'FileImporter.import.details.textRevisions', count( $textRevisions ) ); |
215 | $this->stats->gauge( 'FileImporter.import.details.fileRevisions', count( $fileRevisions ) ); |
216 | $this->stats->gauge( 'FileImporter.import.details.totalFileSizes', $totalFileSizes ); |
217 | |
218 | return $importOperations; |
219 | } |
220 | |
221 | private function prepareImportOperations( ImportOperations $importOperations ): void { |
222 | $status = $importOperations->prepare(); |
223 | if ( !$status->isOK() ) { |
224 | $this->logger->error( __METHOD__ . ' Failed to prepare operations.', [ |
225 | 'status' => $status->__toString(), |
226 | ] ); |
227 | throw new LocalizedImportException( Status::wrap( $status )->getMessage() ); |
228 | } |
229 | } |
230 | |
231 | private function validateImportOperations( StatusValue $status, ImportPlan $importPlan ): void { |
232 | if ( !$status->isGood() ) { |
233 | /** @var IApiMessage[] $newAbuseFilterWarnings */ |
234 | $newAbuseFilterWarnings = []; |
235 | |
236 | foreach ( $status->getErrors() as [ 'message' => $message, 'params' => $params ] ) { |
237 | if ( is_string( $message ) ) { |
238 | // Convert to an array compatible with MessageSpecifier |
239 | $message = [ $message, ...$params ]; |
240 | } |
241 | |
242 | if ( !( $message instanceof IApiMessage ) ) { |
243 | // Unexpected errors bubble up and surface in SpecialImportFile::doImport |
244 | throw new LocalizedImportException( $message ); |
245 | } |
246 | |
247 | $data = $message->getApiData()['abusefilter'] ?? null; |
248 | if ( !$data || |
249 | !in_array( 'warn', $data['actions'] ) || |
250 | in_array( 'disallow', $data['actions'] ) |
251 | ) { |
252 | throw new LocalizedImportException( $message ); |
253 | } |
254 | |
255 | // Skip AbuseFilter warnings we have seen before |
256 | if ( !in_array( $data['id'], $importPlan->getValidationWarnings() ) ) { |
257 | // @phan-suppress-next-line PhanTypeMismatchArgument |
258 | $importPlan->addValidationWarning( $data['id'] ); |
259 | $newAbuseFilterWarnings[] = $message; |
260 | } |
261 | } |
262 | |
263 | if ( $newAbuseFilterWarnings ) { |
264 | throw new AbuseFilterWarningsException( $newAbuseFilterWarnings ); |
265 | } |
266 | } |
267 | } |
268 | |
269 | private function commitImportOperations( ImportOperations $importOperations ): void { |
270 | $status = $importOperations->commit(); |
271 | if ( !$status->isOK() ) { |
272 | $this->logger->error( __METHOD__ . ' Failed to commit operations.', [ |
273 | 'status' => $status->__toString(), |
274 | ] ); |
275 | throw new LocalizedImportException( Status::wrap( $status )->getMessage() ); |
276 | } |
277 | } |
278 | |
279 | /** |
280 | * @return StatusValue isOK on success |
281 | */ |
282 | private function validateFileInfoText( |
283 | User $user, |
284 | ImportPlan $importPlan |
285 | ): StatusValue { |
286 | $status = $this->textRevisionValidator->validate( |
287 | $importPlan->getTitle(), |
288 | $user, |
289 | new \WikitextContent( $importPlan->getFileInfoText() ), |
290 | $importPlan->getRequest()->getIntendedSummary() ?? '', |
291 | false |
292 | ); |
293 | return $status; |
294 | } |
295 | |
296 | /** |
297 | * @return WikiPage |
298 | * @throws ImportException |
299 | */ |
300 | private function getPageFromImportPlan( ImportPlan $importPlan ) { |
301 | // T164729: READ_LATEST needed to select for a write |
302 | $articleIdForUpdate = $importPlan->getTitle()->getArticleID( IDBAccessObject::READ_LATEST ); |
303 | // T181391: Read from primary database, as the page has only just been created, and in multi-DB setups |
304 | // replicas will have lag. |
305 | $page = $this->wikiPageFactory->newFromId( $articleIdForUpdate, IDBAccessObject::READ_LATEST ); |
306 | |
307 | if ( !$page ) { |
308 | throw new ImportException( |
309 | 'Failed to create import edit with page id: ' . $articleIdForUpdate, |
310 | self::ERROR_NO_NEW_PAGE ); |
311 | } |
312 | |
313 | return $page; |
314 | } |
315 | |
316 | private function createPostImportNullRevision( |
317 | ImportPlan $importPlan, |
318 | UserIdentity $user |
319 | ): void { |
320 | $config = MediaWikiServices::getInstance()->getMainConfig(); |
321 | $summary = wfMsgReplaceArgs( |
322 | $config->get( 'FileImporterCommentForPostImportRevision' ), |
323 | [ $importPlan->getRequest()->getUrl() ] |
324 | ); |
325 | |
326 | try { |
327 | $this->nullRevisionCreator->createForLinkTarget( |
328 | $importPlan->getTitle(), |
329 | $importPlan->getDetails()->getFileRevisions()->getLatest(), |
330 | $user, |
331 | $summary |
332 | ); |
333 | } catch ( RuntimeException $ex ) { |
334 | $this->logger->error( __METHOD__ . ' Failed to create import revision.' ); |
335 | throw $ex; |
336 | } |
337 | } |
338 | |
339 | /** |
340 | * @param ImportPlan $importPlan |
341 | * @param WikiPage $page |
342 | * @param Authority $user |
343 | */ |
344 | private function createPostImportEdit( |
345 | ImportPlan $importPlan, |
346 | WikiPage $page, |
347 | Authority $user |
348 | ) { |
349 | // TODO: Replace with $page->newPageUpdater( … )->saveRevision( … ) |
350 | $editResult = $page->doUserEditContent( |
351 | new \WikitextContent( $importPlan->getFileInfoText() ), |
352 | $user, |
353 | $importPlan->getRequest()->getIntendedSummary(), |
354 | EDIT_UPDATE, |
355 | false, |
356 | [ 'fileimporter' ] |
357 | ); |
358 | |
359 | if ( !$editResult->isOK() ) { |
360 | $this->logger->error( __METHOD__ . ' Failed to create user edit.', [ |
361 | 'status' => $editResult->__toString(), |
362 | ] ); |
363 | throw new LocalizedImportException( Status::wrap( $editResult )->getMessage() ); |
364 | } |
365 | } |
366 | |
367 | } |