MediaWiki master
UploadFromChunks.php
Go to the documentation of this file.
1<?php
2
8
39 private $repo;
41 public $stash;
43 public $user;
44
45 protected $mOffset;
46 protected $mChunkIndex;
47 protected $mFileKey;
49
59 public function __construct( User $user, $stash = false, $repo = false ) {
60 $this->user = $user;
61
62 if ( $repo ) {
63 $this->repo = $repo;
64 } else {
65 $this->repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
66 }
67
68 if ( $stash ) {
69 $this->stash = $stash;
70 } else {
71 wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() );
72 $this->stash = new UploadStash( $this->repo, $this->user );
73 }
74 }
75
79 public function tryStashFile( User $user, $isPartial = false ) {
80 try {
81 $this->verifyChunk();
83 return Status::newFatal( $e->msg );
84 }
85
86 return parent::tryStashFile( $user, $isPartial );
87 }
88
95 protected function doStashFile( User $user = null ) {
96 // Stash file is the called on creating a new chunk session:
97 $this->mChunkIndex = 0;
98 $this->mOffset = 0;
99
100 // Create a local stash target
101 $this->mStashFile = parent::doStashFile( $user );
102 // Update the initial file offset (based on file size)
103 $this->mOffset = $this->mStashFile->getSize();
104 $this->mFileKey = $this->mStashFile->getFileKey();
105
106 // Output a copy of this first to chunk 0 location:
107 $this->outputChunk( $this->mStashFile->getPath() );
108
109 // Update db table to reflect initial "chunk" state
110 $this->updateChunkStatus();
111
112 return $this->mStashFile;
113 }
114
122 public function continueChunks( $name, $key, $webRequestUpload ) {
123 $this->mFileKey = $key;
124 $this->mUpload = $webRequestUpload;
125 // Get the chunk status form the db:
126 $this->getChunkStatus();
127
128 $metadata = $this->stash->getMetadata( $key );
129 $this->initializePathInfo( $name,
130 $this->getRealPath( $metadata['us_path'] ),
131 $metadata['us_size'],
132 false
133 );
134 }
135
140 public function concatenateChunks() {
141 $chunkIndex = $this->getChunkIndex();
142 wfDebug( __METHOD__ . " concatenate {$this->mChunkIndex} chunks:" .
143 $this->getOffset() . ' inx:' . $chunkIndex );
144
145 // Concatenate all the chunks to mVirtualTempPath
146 $fileList = [];
147 // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
148 for ( $i = 0; $i <= $chunkIndex; $i++ ) {
149 $fileList[] = $this->getVirtualChunkLocation( $i );
150 }
151
152 // Get the file extension from the last chunk
153 $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath );
154 // Get a 0-byte temp file to perform the concatenation at
155 $tmpFile = MediaWikiServices::getInstance()->getTempFSFileFactory()
156 ->newTempFSFile( 'chunkedupload_', $ext );
157 $tmpPath = false; // fail in concatenate()
158 if ( $tmpFile ) {
159 // keep alive with $this
160 $tmpPath = $tmpFile->bind( $this )->getPath();
161 }
162
163 // Concatenate the chunks at the temp file
164 $tStart = microtime( true );
165 $status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE );
166 $tAmount = microtime( true ) - $tStart;
167 if ( !$status->isOK() ) {
168 // This is a backend error and not user-related, so log is safe
169 // Upload verification further on is not safe to log server side
170 $this->logFileBackendStatus(
171 $status,
172 '[{type}] Error on concatenate {chunks} stashed files ({details})',
173 [ 'chunks' => $chunkIndex ]
174 );
175 return $status;
176 }
177
178 wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." );
179
180 // File system path of the actual full temp file
181 $this->setTempFile( $tmpPath );
182
183 $ret = $this->verifyUpload();
184 if ( $ret['status'] !== UploadBase::OK ) {
185 wfDebugLog( 'fileconcatenate', "Verification failed for chunked upload" );
186 $status->fatal( $this->getVerificationErrorCode( $ret['status'] ) );
187
188 return $status;
189 }
190
191 // Update the mTempPath and mStashFile
192 // (for FileUpload or normal Stash to take over)
193 $tStart = microtime( true );
194 // This is a re-implementation of UploadBase::tryStashFile(), we can't call it because we
195 // override doStashFile() with completely different functionality in this class...
196 $error = $this->runUploadStashFileHook( $this->user );
197 if ( $error ) {
198 $status->fatal( ...$error );
199 return $status;
200 }
201 try {
202 $this->mStashFile = parent::doStashFile( $this->user );
203 } catch ( UploadStashException $e ) {
204 $status->fatal( 'uploadstash-exception', get_class( $e ), $e->getMessage() );
205 return $status;
206 }
207
208 $tAmount = microtime( true ) - $tStart;
209 // @phan-suppress-next-line PhanTypeMismatchArgumentNullable tmpFile is set when tmpPath is set here
210 $this->mStashFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
211 wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." );
212
213 return $status;
214 }
215
221 private function getVirtualChunkLocation( $index ) {
222 return $this->repo->getVirtualUrl( 'temp' ) .
223 '/' .
224 $this->repo->getHashPath(
225 $this->getChunkFileKey( $index )
226 ) .
227 $this->getChunkFileKey( $index );
228 }
229
238 public function addChunk( $chunkPath, $chunkSize, $offset ) {
239 // Get the offset before we add the chunk to the file system
240 $preAppendOffset = $this->getOffset();
241
242 if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize() ) {
243 $status = Status::newFatal( 'file-too-large' );
244 } else {
245 // Make sure the client is uploading the correct chunk with a matching offset.
246 if ( $preAppendOffset == $offset ) {
247 // Update local chunk index for the current chunk
248 $this->mChunkIndex++;
249 try {
250 # For some reason mTempPath is set to first part
251 $oldTemp = $this->mTempPath;
252 $this->mTempPath = $chunkPath;
253 $this->verifyChunk();
254 $this->mTempPath = $oldTemp;
255 } catch ( UploadChunkVerificationException $e ) {
256 return Status::newFatal( $e->msg );
257 }
258 $status = $this->outputChunk( $chunkPath );
259 if ( $status->isGood() ) {
260 // Update local offset:
261 $this->mOffset = $preAppendOffset + $chunkSize;
262 // Update chunk table status db
263 $this->updateChunkStatus();
264 }
265 } else {
266 $status = Status::newFatal( 'invalid-chunk-offset' );
267 }
268 }
269
270 return $status;
271 }
272
276 private function updateChunkStatus() {
277 wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" .
278 $this->getOffset() . ' inx:' . $this->getChunkIndex() );
279
280 $dbw = $this->repo->getPrimaryDB();
281 $dbw->newUpdateQueryBuilder()
282 ->update( 'uploadstash' )
283 ->set( [
284 'us_status' => 'chunks',
285 'us_chunk_inx' => $this->getChunkIndex(),
286 'us_size' => $this->getOffset()
287 ] )
288 ->where( [ 'us_key' => $this->mFileKey ] )
289 ->caller( __METHOD__ )->execute();
290 }
291
295 private function getChunkStatus() {
296 // get primary db to avoid race conditions.
297 // Otherwise, if chunk upload time < replag there will be spurious errors
298 $dbw = $this->repo->getPrimaryDB();
299 $row = $dbw->newSelectQueryBuilder()
300 ->select( [ 'us_chunk_inx', 'us_size', 'us_path' ] )
301 ->from( 'uploadstash' )
302 ->where( [ 'us_key' => $this->mFileKey ] )
303 ->caller( __METHOD__ )->fetchRow();
304 // Handle result:
305 if ( $row ) {
306 $this->mChunkIndex = $row->us_chunk_inx;
307 $this->mOffset = $row->us_size;
308 $this->mVirtualTempPath = $row->us_path;
309 }
310 }
311
316 private function getChunkIndex() {
317 if ( $this->mChunkIndex !== null ) {
318 return $this->mChunkIndex;
319 }
320
321 return 0;
322 }
323
328 public function getOffset() {
329 if ( $this->mOffset !== null ) {
330 return $this->mOffset;
331 }
332
333 return 0;
334 }
335
343 private function outputChunk( $chunkPath ) {
344 // Key is fileKey + chunk index
345 $fileKey = $this->getChunkFileKey();
346
347 // Store the chunk per its indexed fileKey:
348 $hashPath = $this->repo->getHashPath( $fileKey );
349 $storeStatus = $this->repo->quickImport( $chunkPath,
350 $this->repo->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
351
352 // Check for error in stashing the chunk:
353 if ( !$storeStatus->isOK() ) {
354 $error = $this->logFileBackendStatus(
355 $storeStatus,
356 '[{type}] Error storing chunk in "{chunkKey}" for {fileKey} ({details})',
357 [ 'chunkPath' => $chunkPath, 'fileKey' => $fileKey ]
358 );
359 throw new UploadChunkFileException( "Error storing file in '{chunkPath}': " .
360 implode( '; ', $error ), [ 'chunkPath' => $chunkPath ] );
361 }
362
363 return $storeStatus;
364 }
365
366 private function getChunkFileKey( $index = null ) {
367 return $this->mFileKey . '.' . ( $index ?? $this->getChunkIndex() );
368 }
369
375 private function verifyChunk() {
376 // Rest mDesiredDestName here so we verify the name as if it were mFileKey
377 $oldDesiredDestName = $this->mDesiredDestName;
378 $this->mDesiredDestName = $this->mFileKey;
379 $this->mTitle = false;
380 $res = $this->verifyPartialFile();
381 $this->mDesiredDestName = $oldDesiredDestName;
382 $this->mTitle = false;
383 if ( is_array( $res ) ) {
384 throw new UploadChunkVerificationException( $res );
385 }
386 }
387
397 private function logFileBackendStatus( Status $status, string $logMessage, array $context = [] ): array {
398 $logger = LoggerFactory::getInstance( 'upload' );
399 $errorToThrow = null;
400 $warningToThrow = null;
401
402 foreach ( $status->getErrors() as $errorItem ) {
403 // The message key stands for distinct error situation from the file backend,
404 // each error situation should be shown up in aggregated stats as own point, replace in message
405 $logMessageType = str_replace( '{type}', $errorItem['message'], $logMessage );
406
407 // The message arguments often contains the name of the failing datacenter or file names
408 // and should not show up in aggregated stats, add to context
409 $context['details'] = implode( '; ', $errorItem['params'] );
410
411 if ( $errorItem['type'] === 'error' ) {
412 // Use the first error of the list for the exception text
413 $errorToThrow ??= array_merge( [ $errorItem['message'] ], $errorItem['params'] );
414 $logger->error( $logMessage, $context );
415 } else {
416 // When no error is found, fall back to the first warning
417 $warningToThrow ??= array_merge( [ $errorItem['message'] ], $errorItem['params'] );
418 $logger->warning( $logMessage, $context );
419 }
420 }
421 return $errorToThrow ?? $warningToThrow ?? [ 'unknown', 'no error recorded' ];
422 }
423}
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfDebugLog( $logGroup, $text, $dest='all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not.
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
const DELETE_SOURCE
Definition FileRepo.php:51
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition LocalRepo.php:48
msg( $key, $fallback,... $params)
Get a message from i18n.
Create PSR-3 logger objects.
Service locator for MediaWiki core services.
Object to access the $_FILES array.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition Status.php:54
internal since 1.36
Definition User.php:96
getId( $wikiId=self::LOCAL)
Get the user's ID.
Definition User.php:1569
getErrors()
Get the list of errors.
isOK()
Returns whether the operation completed.
fatal( $message,... $parameters)
Add an error and set OK to false, indicating that the operation as a whole was fatal.
isGood()
Returns whether the operation completed and didn't have any error or warnings.
UploadStashFile null $mStashFile
getRealPath( $srcPath)
runUploadStashFileHook(User $user)
verifyPartialFile()
A verification routine suitable for partial files.
getVerificationErrorCode( $error)
string null $mDesiredDestName
setTempFile( $tempPath, $fileSize=null)
initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile=false)
static getMaxUploadSize( $forType=null)
Get MediaWiki's maximum uploaded file size for a given type of upload, based on $wgMaxUploadSize.
string null $mTempPath
Local file system path to the file to upload (or a local copy)
Implements uploading from chunks.
addChunk( $chunkPath, $chunkSize, $offset)
Add a chunk to the temporary directory.
doStashFile(User $user=null)
Calls the parent doStashFile and updates the uploadsession table to handle "chunks".
continueChunks( $name, $key, $webRequestUpload)
Continue chunk uploading.
tryStashFile(User $user, $isPartial=false)
Like stashFile(), but respects extensions' wishes to prevent the stashing.verifyUpload() must be call...
__construct(User $user, $stash=false, $repo=false)
@noinspection PhpMissingParentConstructorInspection
getOffset()
Get the offset at which the next uploaded chunk will be appended to.
concatenateChunks()
Append the final chunk and ready file for parent::performUpload()
Implements regular file uploads.
UploadStash is intended to accomplish a few things:
if(!is_readable( $file)) $ext
Definition router.php:48