MediaWiki master
UploadFromChunks.php
Go to the documentation of this file.
1<?php
2
8use Psr\Log\LoggerInterface;
10
41 private $repo;
43 public $stash;
45 public $user;
46
47 protected $mOffset;
48 protected $mChunkIndex;
49 protected $mFileKey;
51
52 private LoggerInterface $logger;
53
63 public function __construct( User $user, $stash = false, $repo = false ) {
64 $this->user = $user;
65
66 if ( $repo ) {
67 $this->repo = $repo;
68 } else {
69 $this->repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
70 }
71
72 if ( $stash ) {
73 $this->stash = $stash;
74 } else {
75 wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() );
76 $this->stash = new UploadStash( $this->repo, $this->user );
77 }
78
79 $this->logger = LoggerFactory::getInstance( 'upload' );
80 }
81
85 public function tryStashFile( User $user, $isPartial = false ) {
86 try {
87 $this->verifyChunk();
89 return Status::newFatal( $e->msg );
90 }
91
92 return parent::tryStashFile( $user, $isPartial );
93 }
94
101 protected function doStashFile( User $user = null ) {
102 // Stash file is the called on creating a new chunk session:
103 $this->mChunkIndex = 0;
104 $this->mOffset = 0;
105
106 // Create a local stash target
107 $this->mStashFile = parent::doStashFile( $user );
108 // Update the initial file offset (based on file size)
109 $this->mOffset = $this->mStashFile->getSize();
110 $this->mFileKey = $this->mStashFile->getFileKey();
111
112 // Output a copy of this first to chunk 0 location:
113 $this->outputChunk( $this->mStashFile->getPath() );
114
115 // Update db table to reflect initial "chunk" state
116 $this->updateChunkStatus();
117
118 return $this->mStashFile;
119 }
120
128 public function continueChunks( $name, $key, $webRequestUpload ) {
129 $this->mFileKey = $key;
130 $this->mUpload = $webRequestUpload;
131 // Get the chunk status form the db:
132 $this->getChunkStatus();
133
134 $metadata = $this->stash->getMetadata( $key );
135 $this->initializePathInfo( $name,
136 $this->getRealPath( $metadata['us_path'] ),
137 $metadata['us_size'],
138 false
139 );
140 }
141
146 public function concatenateChunks() {
147 $oldFileKey = $this->mFileKey;
148 $chunkIndex = $this->getChunkIndex();
149 $this->logger->debug(
150 __METHOD__ . ' concatenate {totalChunks} chunks: {offset} inx: {curIndex}',
151 [
152 'offset' => $this->getOffset(),
153 'totalChunks' => $this->mChunkIndex,
154 'curIndex' => $chunkIndex,
155 'filekey' => $oldFileKey
156 ]
157 );
158
159 // Concatenate all the chunks to mVirtualTempPath
160 $fileList = [];
161 // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
162 for ( $i = 0; $i <= $chunkIndex; $i++ ) {
163 $fileList[] = $this->getVirtualChunkLocation( $i );
164 }
165
166 // Get the file extension from the last chunk
167 $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath );
168 // Get a 0-byte temp file to perform the concatenation at
169 $tmpFile = MediaWikiServices::getInstance()->getTempFSFileFactory()
170 ->newTempFSFile( 'chunkedupload_', $ext );
171 $tmpPath = false; // fail in concatenate()
172 if ( $tmpFile ) {
173 // keep alive with $this
174 $tmpPath = $tmpFile->bind( $this )->getPath();
175 } else {
176 $this->logger->warning( "Error getting tmp file", [ 'filekey' => $oldFileKey ] );
177 }
178
179 // Concatenate the chunks at the temp file
180 $tStart = microtime( true );
181 $status = $this->repo->concatenate( $fileList, $tmpPath );
182 $tAmount = microtime( true ) - $tStart;
183 if ( !$status->isOK() ) {
184 // This is a backend error and not user-related, so log is safe
185 // Upload verification further on is not safe to log server side
186 $this->logFileBackendStatus(
187 $status,
188 '[{type}] Error on concatenate {chunks} stashed files ({details})',
189 [ 'chunks' => $chunkIndex, 'filekey' => $oldFileKey ]
190 );
191 return $status;
192 } else {
193 // Delete old chunks in deferred job. Put in deferred job because deleting
194 // lots of chunks can take a long time, sometimes to the point of causing
195 // a timeout, and we do not want that to tank the operation. Note that chunks
196 // are also automatically deleted after a set time by cleanupUploadStash.php
197 // Additionally, using AutoCommitUpdate ensures that we do not delete files
198 // if the main transaction is rolled back for some reason.
199 DeferredUpdates::addUpdate( new AutoCommitUpdate(
200 $this->repo->getPrimaryDB(),
201 __METHOD__,
202 function () use( $fileList, $oldFileKey ) {
203 $status = $this->repo->quickPurgeBatch( $fileList );
204 if ( !$status->isOK() ) {
205 $this->logger->warning(
206 "Could not delete chunks of {filekey} - {status}",
207 [
208 'status' => (string)$status,
209 'filekey' => $oldFileKey,
210 ]
211 );
212 }
213 }
214 ) );
215 }
216
217 wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." );
218
219 // File system path of the actual full temp file
220 $this->setTempFile( $tmpPath );
221
222 $ret = $this->verifyUpload();
223 if ( $ret['status'] !== UploadBase::OK ) {
224 $this->logger->info(
225 "Verification failed for chunked upload {filekey}",
226 [
227 'user' => $this->user->getName(),
228 'filekey' => $oldFileKey
229 ]
230 );
231 $status->fatal( $this->getVerificationErrorCode( $ret['status'] ) );
232
233 return $status;
234 }
235
236 // Update the mTempPath and mStashFile
237 // (for FileUpload or normal Stash to take over)
238 $tStart = microtime( true );
239 // This is a re-implementation of UploadBase::tryStashFile(), we can't call it because we
240 // override doStashFile() with completely different functionality in this class...
241 $error = $this->runUploadStashFileHook( $this->user );
242 if ( $error ) {
243 $status->fatal( ...$error );
244 $this->logger->info( "Aborting stash upload due to hook - {status}",
245 [
246 'status' => (string)$status,
247 'user' => $this->user->getName(),
248 'filekey' => $this->mFileKey
249 ]
250 );
251 return $status;
252 }
253 try {
254 $this->mStashFile = parent::doStashFile( $this->user );
255 } catch ( UploadStashException $e ) {
256 $this->logger->warning( "Could not stash file for {user} because {error} {msg}",
257 [
258 'user' => $this->user->getName(),
259 'error' => get_class( $e ),
260 'msg' => $e->getMessage(),
261 'filekey' => $this->mFileKey
262 ]
263 );
264 $status->fatal( 'uploadstash-exception', get_class( $e ), $e->getMessage() );
265 return $status;
266 }
267
268 $tAmount = microtime( true ) - $tStart;
269 // @phan-suppress-next-line PhanTypeMismatchArgumentNullable tmpFile is set when tmpPath is set here
270 $this->mStashFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
271 $this->logger->info( "Stashed combined ({chunks} chunks) of {oldkey} under new name {filekey}",
272 [
273 'chunks' => $i,
274 'stashTime' => $tAmount,
275 'oldpath' => $this->mVirtualTempPath,
276 'filekey' => $this->mStashFile->getFileKey(),
277 'oldkey' => $oldFileKey,
278 'newpath' => $this->mStashFile->getPath(),
279 'user' => $this->user->getName()
280 ]
281 );
282 wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." );
283
284 return $status;
285 }
286
292 private function getVirtualChunkLocation( $index ) {
293 return $this->repo->getVirtualUrl( 'temp' ) .
294 '/' .
295 $this->repo->getHashPath(
296 $this->getChunkFileKey( $index )
297 ) .
298 $this->getChunkFileKey( $index );
299 }
300
309 public function addChunk( $chunkPath, $chunkSize, $offset ) {
310 // Get the offset before we add the chunk to the file system
311 $preAppendOffset = $this->getOffset();
312
313 if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize() ) {
314 $status = Status::newFatal( 'file-too-large' );
315 } else {
316 // Make sure the client is uploading the correct chunk with a matching offset.
317 if ( $preAppendOffset == $offset ) {
318 // Update local chunk index for the current chunk
319 $this->mChunkIndex++;
320 try {
321 # For some reason mTempPath is set to first part
322 $oldTemp = $this->mTempPath;
323 $this->mTempPath = $chunkPath;
324 $this->verifyChunk();
325 $this->mTempPath = $oldTemp;
326 } catch ( UploadChunkVerificationException $e ) {
327 $this->logger->info( "Error verifying upload chunk {msg}",
328 [
329 'user' => $this->user->getName(),
330 'msg' => $e->getMessage(),
331 'chunkIndex' => $this->mChunkIndex,
332 'filekey' => $this->mFileKey
333 ]
334 );
335
336 return Status::newFatal( $e->msg );
337 }
338 $status = $this->outputChunk( $chunkPath );
339 if ( $status->isGood() ) {
340 // Update local offset:
341 $this->mOffset = $preAppendOffset + $chunkSize;
342 // Update chunk table status db
343 $this->updateChunkStatus();
344 }
345 } else {
346 $status = Status::newFatal( 'invalid-chunk-offset' );
347 }
348 }
349
350 return $status;
351 }
352
356 private function updateChunkStatus() {
357 $this->logger->info( "update chunk status for {filekey} offset: {offset} inx: {inx}",
358 [
359 'offset' => $this->getOffset(),
360 'inx' => $this->getChunkIndex(),
361 'filekey' => $this->mFileKey,
362 'user' => $this->user->getName()
363 ]
364 );
365
366 $dbw = $this->repo->getPrimaryDB();
367 $dbw->newUpdateQueryBuilder()
368 ->update( 'uploadstash' )
369 ->set( [
370 'us_status' => 'chunks',
371 'us_chunk_inx' => $this->getChunkIndex(),
372 'us_size' => $this->getOffset()
373 ] )
374 ->where( [ 'us_key' => $this->mFileKey ] )
375 ->caller( __METHOD__ )->execute();
376 }
377
381 private function getChunkStatus() {
382 // get primary db to avoid race conditions.
383 // Otherwise, if chunk upload time < replag there will be spurious errors
384 $dbw = $this->repo->getPrimaryDB();
385 $row = $dbw->newSelectQueryBuilder()
386 ->select( [ 'us_chunk_inx', 'us_size', 'us_path' ] )
387 ->from( 'uploadstash' )
388 ->where( [ 'us_key' => $this->mFileKey ] )
389 ->caller( __METHOD__ )->fetchRow();
390 // Handle result:
391 if ( $row ) {
392 $this->mChunkIndex = $row->us_chunk_inx;
393 $this->mOffset = $row->us_size;
394 $this->mVirtualTempPath = $row->us_path;
395 }
396 }
397
402 private function getChunkIndex() {
403 if ( $this->mChunkIndex !== null ) {
404 return $this->mChunkIndex;
405 }
406
407 return 0;
408 }
409
414 public function getOffset() {
415 if ( $this->mOffset !== null ) {
416 return $this->mOffset;
417 }
418
419 return 0;
420 }
421
429 private function outputChunk( $chunkPath ) {
430 // Key is fileKey + chunk index
431 $fileKey = $this->getChunkFileKey();
432
433 // Store the chunk per its indexed fileKey:
434 $hashPath = $this->repo->getHashPath( $fileKey );
435 $storeStatus = $this->repo->quickImport( $chunkPath,
436 $this->repo->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
437
438 // Check for error in stashing the chunk:
439 if ( !$storeStatus->isOK() ) {
440 $error = $this->logFileBackendStatus(
441 $storeStatus,
442 '[{type}] Error storing chunk in "{chunkPath}" for {fileKey} ({details})',
443 [ 'chunkPath' => $chunkPath, 'fileKey' => $fileKey ]
444 );
445 throw new UploadChunkFileException( "Error storing file in '{chunkPath}': " .
446 implode( '; ', $error ), [ 'chunkPath' => $chunkPath ] );
447 }
448
449 return $storeStatus;
450 }
451
452 private function getChunkFileKey( $index = null ) {
453 return $this->mFileKey . '.' . ( $index ?? $this->getChunkIndex() );
454 }
455
461 private function verifyChunk() {
462 // Rest mDesiredDestName here so we verify the name as if it were mFileKey
463 $oldDesiredDestName = $this->mDesiredDestName;
464 $this->mDesiredDestName = $this->mFileKey;
465 $this->mTitle = false;
466 $res = $this->verifyPartialFile();
467 $this->mDesiredDestName = $oldDesiredDestName;
468 $this->mTitle = false;
469 if ( is_array( $res ) ) {
470 throw new UploadChunkVerificationException( $res );
471 }
472 }
473
483 private function logFileBackendStatus( Status $status, string $logMessage, array $context = [] ): array {
484 $logger = $this->logger;
485 $errorToThrow = null;
486 $warningToThrow = null;
487
488 foreach ( $status->getErrors() as $errorItem ) {
489 // The message key stands for distinct error situation from the file backend,
490 // each error situation should be shown up in aggregated stats as own point, replace in message
491 $logMessageType = str_replace( '{type}', $errorItem['message'], $logMessage );
492
493 // The message arguments often contains the name of the failing datacenter or file names
494 // and should not show up in aggregated stats, add to context
495 $context['details'] = implode( '; ', $errorItem['params'] );
496 $context['user'] = $this->user->getName();
497
498 if ( $errorItem['type'] === 'error' ) {
499 // Use the first error of the list for the exception text
500 $errorToThrow ??= [ $errorItem['message'], ...$errorItem['params'] ];
501 $logger->error( $logMessageType, $context );
502 } else {
503 // When no error is found, fall back to the first warning
504 $warningToThrow ??= [ $errorItem['message'], ...$errorItem['params'] ];
505 $logger->warning( $logMessageType, $context );
506 }
507 }
508 return $errorToThrow ?? $warningToThrow ?? [ 'unknown', 'no error recorded' ];
509 }
510}
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfDebugLog( $logGroup, $text, $dest='all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not.
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition LocalRepo.php:49
Create PSR-3 logger objects.
Service locator for MediaWiki core services.
Object to access the $_FILES array.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition Status.php:54
internal since 1.36
Definition User.php:93
getId( $wikiId=self::LOCAL)
Get the user's ID.
Definition User.php:1530
getErrors()
Get the list of errors.
isOK()
Returns whether the operation completed.
fatal( $message,... $parameters)
Add an error and set OK to false, indicating that the operation as a whole was fatal.
isGood()
Returns whether the operation completed and didn't have any error or warnings.
UploadStashFile null $mStashFile
getRealPath( $srcPath)
runUploadStashFileHook(User $user)
verifyPartialFile()
A verification routine suitable for partial files.
getVerificationErrorCode( $error)
string null $mDesiredDestName
setTempFile( $tempPath, $fileSize=null)
initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile=false)
static getMaxUploadSize( $forType=null)
Get MediaWiki's maximum uploaded file size for a given type of upload, based on $wgMaxUploadSize.
string null $mTempPath
Local file system path to the file to upload (or a local copy)
Implements uploading from chunks.
addChunk( $chunkPath, $chunkSize, $offset)
Add a chunk to the temporary directory.
doStashFile(User $user=null)
Calls the parent doStashFile and updates the uploadsession table to handle "chunks".
continueChunks( $name, $key, $webRequestUpload)
Continue chunk uploading.
tryStashFile(User $user, $isPartial=false)
Like stashFile(), but respects extensions' wishes to prevent the stashing.verifyUpload() must be call...
__construct(User $user, $stash=false, $repo=false)
@noinspection PhpMissingParentConstructorInspection
getOffset()
Get the offset at which the next uploaded chunk will be appended to.
concatenateChunks()
Append the final chunk and ready file for parent::performUpload()
Implements regular file uploads.
UploadStash is intended to accomplish a few things:
Base class for all file backend classes (including multi-write backends).