Code Coverage
 
Lines
Functions and Methods
Classes and Traits
Total
73.16% covered (warning)
73.16%
169 / 231
53.33% covered (warning)
53.33%
8 / 15
CRAP
0.00% covered (danger)
0.00%
0 / 1
UploadFromChunks
73.48% covered (warning)
73.48%
169 / 230
53.33% covered (warning)
53.33%
8 / 15
60.18
0.00% covered (danger)
0.00%
0 / 1
 __construct
90.00% covered (success)
90.00%
9 / 10
0.00% covered (danger)
0.00%
0 / 1
3.01
 tryStashFile
50.00% covered (danger)
50.00%
2 / 4
0.00% covered (danger)
0.00%
0 / 1
2.50
 doStashFile
100.00% covered (success)
100.00%
9 / 9
100.00% covered (success)
100.00%
1 / 1
1
 continueChunks
91.67% covered (success)
91.67%
11 / 12
0.00% covered (danger)
0.00%
0 / 1
2.00
 concatenateChunks
56.70% covered (warning)
56.70%
55 / 97
0.00% covered (danger)
0.00%
0 / 1
13.20
 getVirtualChunkLocation
100.00% covered (success)
100.00%
6 / 6
100.00% covered (success)
100.00%
1 / 1
1
 addChunk
55.56% covered (warning)
55.56%
15 / 27
0.00% covered (danger)
0.00%
0 / 1
9.16
 updateChunkStatus
100.00% covered (success)
100.00%
18 / 18
100.00% covered (success)
100.00%
1 / 1
1
 getChunkStatus
100.00% covered (success)
100.00%
10 / 10
100.00% covered (success)
100.00%
1 / 1
2
 getChunkIndex
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 getOffset
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 outputChunk
100.00% covered (success)
100.00%
13 / 13
100.00% covered (success)
100.00%
1 / 1
2
 getChunkFileKey
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 verifyChunk
87.50% covered (warning)
87.50%
7 / 8
0.00% covered (danger)
0.00%
0 / 1
2.01
 logFileBackendStatus
84.62% covered (warning)
84.62%
11 / 13
0.00% covered (danger)
0.00%
0 / 1
3.03
1<?php
2
3namespace MediaWiki\Upload;
4
5use MediaWiki\Deferred\AutoCommitUpdate;
6use MediaWiki\Deferred\DeferredUpdates;
7use MediaWiki\FileRepo\LocalRepo;
8use MediaWiki\Logger\LoggerFactory;
9use MediaWiki\MediaWikiServices;
10use MediaWiki\Request\WebRequestUpload;
11use MediaWiki\Status\Status;
12use MediaWiki\Upload\Exception\UploadChunkFileException;
13use MediaWiki\Upload\Exception\UploadChunkVerificationException;
14use MediaWiki\Upload\Exception\UploadStashBadPathException;
15use MediaWiki\Upload\Exception\UploadStashException;
16use MediaWiki\User\User;
17use Psr\Log\LoggerInterface;
18use Wikimedia\FileBackend\FileBackend;
19
20/**
21 * Backend for uploading files from chunks.
22 *
23 * @license GPL-2.0-or-later
24 * @file
25 * @ingroup Upload
26 */
27
28/**
29 * Implements uploading from chunks
30 *
31 * @ingroup Upload
32 * @author Michael Dale
33 */
34class UploadFromChunks extends UploadFromFile {
35    /** @var LocalRepo */
36    private $repo;
37    /** @var UploadStash */
38    public $stash;
39    /** @var User */
40    public $user;
41
42    /** @var int|null */
43    protected $mOffset;
44    /** @var int|null */
45    protected $mChunkIndex;
46    /** @var string */
47    protected $mFileKey;
48    /** @var string|null */
49    protected $mVirtualTempPath;
50
51    private LoggerInterface $logger;
52
53    /** @noinspection PhpMissingParentConstructorInspection */
54
55    /**
56     * Setup local pointers to stash, repo and user (similar to UploadFromStash)
57     *
58     * @param User $user
59     * @param UploadStash|false $stash Default: false
60     * @param LocalRepo|false $repo Default: false
61     */
62    public function __construct( User $user, $stash = false, $repo = false ) {
63        $this->user = $user;
64
65        if ( $repo ) {
66            $this->repo = $repo;
67        } else {
68            $this->repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
69        }
70
71        if ( $stash ) {
72            $this->stash = $stash;
73        } else {
74            wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() );
75            $this->stash = new UploadStash( $this->repo, $this->user );
76        }
77
78        $this->logger = LoggerFactory::getInstance( 'upload' );
79        parent::__construct();
80    }
81
82    /**
83     * @inheritDoc
84     */
85    public function tryStashFile( User $user, $isPartial = false ) {
86        try {
87            $this->verifyChunk();
88        } catch ( UploadChunkVerificationException $e ) {
89            return Status::newFatal( $e->msg );
90        }
91
92        return parent::tryStashFile( $user, $isPartial );
93    }
94
95    /**
96     * Calls the parent doStashFile and updates the uploadsession table to handle "chunks"
97     *
98     * @param User|null $user
99     * @return UploadStashFile Stashed file
100     */
101    protected function doStashFile( ?User $user = null ) {
102        // Stash file is the called on creating a new chunk session:
103        $this->mChunkIndex = 0;
104        $this->mOffset = 0;
105
106        // Create a local stash target
107        $this->mStashFile = parent::doStashFile( $user );
108        // Update the initial file offset (based on file size)
109        $this->mOffset = $this->mStashFile->getSize();
110        $this->mFileKey = $this->mStashFile->getFileKey();
111        $this->mVirtualTempPath = $this->mStashFile->getPath();
112
113        // Output a copy of this first to chunk 0 location:
114        $this->outputChunk( $this->mStashFile->getPath() );
115
116        // Update db table to reflect initial "chunk" state
117        $this->updateChunkStatus();
118
119        return $this->mStashFile;
120    }
121
122    /**
123     * Continue chunk uploading
124     *
125     * @param string $name
126     * @param string $key
127     * @param WebRequestUpload $webRequestUpload
128     */
129    public function continueChunks( $name, $key, $webRequestUpload ) {
130        $this->mFileKey = $key;
131        $this->mUpload = $webRequestUpload;
132        // Get the chunk status form the db:
133        $this->getChunkStatus();
134
135        $metadata = $this->stash->getMetadata( $key );
136        $tempPath = $this->getRealPath( $metadata['us_path'] );
137        if ( $tempPath === false ) {
138            throw new UploadStashBadPathException( wfMessage( 'uploadstash-bad-path' ) );
139        }
140        $this->initializePathInfo( $name,
141            $tempPath,
142            $metadata['us_size'],
143            false
144        );
145    }
146
147    /**
148     * Append the final chunk and ready file for parent::performUpload()
149     * @return Status
150     */
151    public function concatenateChunks() {
152        $oldFileKey = $this->mFileKey;
153        $chunkIndex = $this->getChunkIndex();
154        $this->logger->debug(
155            __METHOD__ . ' concatenate {totalChunks} chunks: {offset} inx: {curIndex}',
156            [
157                'offset' => $this->getOffset(),
158                'totalChunks' => $this->mChunkIndex,
159                'curIndex' => $chunkIndex,
160                'filekey' => $oldFileKey
161            ]
162        );
163
164        // Concatenate all the chunks to mVirtualTempPath
165        $fileList = [];
166        // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
167        for ( $i = 0; $i <= $chunkIndex; $i++ ) {
168            $fileList[] = $this->getVirtualChunkLocation( $i );
169        }
170
171        // Get the file extension from the last chunk
172        $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath );
173        // Get a 0-byte temp file to perform the concatenation at
174        $tmpFile = MediaWikiServices::getInstance()->getTempFSFileFactory()
175            ->newTempFSFile( 'chunkedupload_', $ext );
176        $tmpPath = false; // fail in concatenate()
177        if ( $tmpFile ) {
178            // keep alive with $this
179            $tmpPath = $tmpFile->bind( $this )->getPath();
180        } else {
181            $this->logger->warning( "Error getting tmp file", [ 'filekey' => $oldFileKey ] );
182        }
183
184        // Concatenate the chunks at the temp file
185        $tStart = microtime( true );
186        $status = $this->repo->concatenate( $fileList, $tmpPath );
187        $tAmount = microtime( true ) - $tStart;
188        if ( !$status->isOK() ) {
189            // This is a backend error and not user-related, so log is safe
190            // Upload verification further on is not safe to log server side
191            $this->logFileBackendStatus(
192                $status,
193                '[{type}] Error on concatenate {chunks} stashed files ({details})',
194                [ 'chunks' => $chunkIndex, 'filekey' => $oldFileKey ]
195            );
196            return $status;
197        } else {
198            // Delete old chunks in deferred job. Put in deferred job because deleting
199            // lots of chunks can take a long time, sometimes to the point of causing
200            // a timeout, and we do not want that to tank the operation. Note that chunks
201            // are also automatically deleted after a set time by cleanupUploadStash.php
202            // Additionally, using AutoCommitUpdate ensures that we do not delete files
203            // if the main transaction is rolled back for some reason.
204            DeferredUpdates::addUpdate( new AutoCommitUpdate(
205                $this->repo->getPrimaryDB(),
206                __METHOD__,
207                function () use ( $fileList, $oldFileKey ) {
208                    $status = $this->repo->quickPurgeBatch( $fileList );
209                    if ( !$status->isOK() ) {
210                        $this->logger->warning(
211                            "Could not delete chunks of {filekey} - {status}",
212                            [
213                                'status' => (string)$status,
214                                'filekey' => $oldFileKey,
215                            ]
216                        );
217                    }
218                }
219            ) );
220        }
221
222        wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." );
223
224        // File system path of the actual full temp file
225        $this->setTempFile( $tmpPath );
226
227        $ret = $this->verifyUpload();
228        if ( $ret['status'] !== UploadBase::OK ) {
229            $this->logger->info(
230                "Verification failed for chunked upload {filekey}",
231                [
232                    'user' => $this->user->getName(),
233                    'filekey' => $oldFileKey
234                ]
235            );
236            // @phan-suppress-next-line PhanTypeMismatchReturnProbablyReal
237            return $this->convertVerifyErrorToStatus( $ret );
238        }
239
240        // Update the mTempPath and mStashFile
241        // (for FileUpload or normal Stash to take over)
242        $tStart = microtime( true );
243        // This is a re-implementation of UploadBase::tryStashFile(), we can't call it because we
244        // override doStashFile() with completely different functionality in this class...
245        $error = $this->runUploadStashFileHook( $this->user );
246        if ( $error ) {
247            $status->fatal( ...$error );
248            $this->logger->info( "Aborting stash upload due to hook - {status}",
249                [
250                    'status' => (string)$status,
251                    'user' => $this->user->getName(),
252                    'filekey' => $this->mFileKey
253                ]
254            );
255            return $status;
256        }
257        try {
258            $this->mStashFile = parent::doStashFile( $this->user );
259        } catch ( UploadStashException $e ) {
260            $this->logger->warning( "Could not stash file for {user} because {error} {msg}",
261                [
262                    'user' => $this->user->getName(),
263                    'error' => get_class( $e ),
264                    'msg' => $e->getMessage(),
265                    'filekey' => $this->mFileKey
266                ]
267            );
268            $status->fatal( 'uploadstash-exception', get_class( $e ), $e->getMessage() );
269            return $status;
270        }
271
272        $tAmount = microtime( true ) - $tStart;
273        // @phan-suppress-next-line PhanTypeMismatchArgumentNullable tmpFile is set when tmpPath is set here
274        $this->mStashFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
275        $this->logger->info( "Stashed combined ({chunks} chunks) of {oldkey} under new name {filekey}",
276            [
277                'chunks' => $i,
278                'stashTime' => $tAmount,
279                'oldpath' => $this->mVirtualTempPath,
280                'filekey' => $this->mStashFile->getFileKey(),
281                'oldkey' => $oldFileKey,
282                'newpath' => $this->mStashFile->getPath(),
283                'user' => $this->user->getName()
284            ]
285        );
286        wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." );
287
288        return $status;
289    }
290
291    /**
292     * Returns the virtual chunk location:
293     * @param int $index
294     * @return string
295     */
296    private function getVirtualChunkLocation( $index ) {
297        return $this->repo->getVirtualUrl( 'temp' ) .
298            '/' .
299            $this->repo->getHashPath(
300                $this->getChunkFileKey( $index )
301            ) .
302            $this->getChunkFileKey( $index );
303    }
304
305    /**
306     * Add a chunk to the temporary directory
307     *
308     * @param string $chunkPath Path to temporary chunk file
309     * @param int $chunkSize Size of the current chunk
310     * @param int $offset Offset of current chunk (must match database chunk offset)
311     * @return Status
312     */
313    public function addChunk( $chunkPath, $chunkSize, $offset ) {
314        // Get the offset before we add the chunk to the file system
315        $preAppendOffset = $this->getOffset();
316
317        if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize() ) {
318            $status = Status::newFatal( 'file-too-large' );
319        } else {
320            // Make sure the client is uploading the correct chunk with a matching offset.
321            if ( $preAppendOffset == $offset ) {
322                // Update local chunk index for the current chunk
323                $this->mChunkIndex++;
324                try {
325                    # For some reason mTempPath is set to first part
326                    $oldTemp = $this->mTempPath;
327                    $this->mTempPath = $chunkPath;
328                    $this->verifyChunk();
329                    $this->mTempPath = $oldTemp;
330                } catch ( UploadChunkVerificationException $e ) {
331                    $this->logger->info( "Error verifying upload chunk {msg}",
332                        [
333                            'user' => $this->user->getName(),
334                            'msg' => $e->getMessage(),
335                            'chunkIndex' => $this->mChunkIndex,
336                            'filekey' => $this->mFileKey
337                        ]
338                    );
339
340                    return Status::newFatal( $e->msg );
341                }
342                try {
343                    $status = $this->outputChunk( $chunkPath );
344                } catch ( UploadChunkFileException $uploadChunkFileException ) {
345                    $status = Status::newFatal( $uploadChunkFileException->getMessage() );
346                }
347                if ( $status->isGood() ) {
348                    // Update local offset:
349                    $this->mOffset = $preAppendOffset + $chunkSize;
350                    // Update chunk table status db
351                    $this->updateChunkStatus();
352                }
353            } else {
354                $status = Status::newFatal( 'invalid-chunk-offset' );
355            }
356        }
357
358        return $status;
359    }
360
361    /**
362     * Update the chunk db table with the current status:
363     */
364    private function updateChunkStatus() {
365        $this->logger->info( "update chunk status for {filekey} offset: {offset} inx: {inx}",
366            [
367                'offset' => $this->getOffset(),
368                'inx' => $this->getChunkIndex(),
369                'filekey' => $this->mFileKey,
370                'user' => $this->user->getName()
371            ]
372        );
373
374        $dbw = $this->repo->getPrimaryDB();
375        $dbw->newUpdateQueryBuilder()
376            ->update( 'uploadstash' )
377            ->set( [
378                'us_status' => 'chunks',
379                'us_chunk_inx' => $this->getChunkIndex(),
380                'us_size' => $this->getOffset()
381            ] )
382            ->where( [ 'us_key' => $this->mFileKey ] )
383            ->caller( __METHOD__ )->execute();
384    }
385
386    /**
387     * Get the chunk db state and populate update relevant local values
388     */
389    private function getChunkStatus() {
390        // get primary db to avoid race conditions.
391        // Otherwise, if chunk upload time < replag there will be spurious errors
392        $dbw = $this->repo->getPrimaryDB();
393        $row = $dbw->newSelectQueryBuilder()
394            ->select( [ 'us_chunk_inx', 'us_size', 'us_path' ] )
395            ->from( 'uploadstash' )
396            ->where( [ 'us_key' => $this->mFileKey ] )
397            ->caller( __METHOD__ )->fetchRow();
398        // Handle result:
399        if ( $row ) {
400            $this->mChunkIndex = $row->us_chunk_inx;
401            $this->mOffset = $row->us_size;
402            $this->mVirtualTempPath = $row->us_path;
403        }
404    }
405
406    /**
407     * Get the current Chunk index
408     * @return int Index of the current chunk
409     */
410    private function getChunkIndex() {
411        return $this->mChunkIndex ?? 0;
412    }
413
414    /**
415     * Get the offset at which the next uploaded chunk will be appended to
416     * @return int Current byte offset of the chunk file set
417     */
418    public function getOffset() {
419        return $this->mOffset ?? 0;
420    }
421
422    /**
423     * Output the chunk to disk
424     *
425     * @param string $chunkPath
426     * @throws UploadChunkFileException
427     * @return Status
428     */
429    private function outputChunk( $chunkPath ) {
430        // Key is fileKey + chunk index
431        $fileKey = $this->getChunkFileKey();
432
433        // Store the chunk per its indexed fileKey:
434        $hashPath = $this->repo->getHashPath( $fileKey );
435        $storeStatus = $this->repo->quickImport( $chunkPath,
436            $this->repo->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
437
438        // Check for error in stashing the chunk:
439        if ( !$storeStatus->isOK() ) {
440            $error = $this->logFileBackendStatus(
441                $storeStatus,
442                '[{type}] Error storing chunk in "{chunkPath}" for {fileKey} ({details})',
443                [ 'chunkPath' => $chunkPath, 'fileKey' => $fileKey ]
444            );
445            throw new UploadChunkFileException( "Error storing file in '{chunkPath}': " .
446                implode( '; ', $error ), [ 'chunkPath' => $chunkPath ] );
447        }
448
449        return $storeStatus;
450    }
451
452    private function getChunkFileKey( ?int $index = null ): string {
453        return $this->mFileKey . '.' . ( $index ?? $this->getChunkIndex() );
454    }
455
456    /**
457     * Verify that the chunk isn't really an evil html file
458     *
459     * @throws UploadChunkVerificationException
460     */
461    private function verifyChunk() {
462        // Rest mDesiredDestName here so we verify the name as if it were mFileKey
463        $oldDesiredDestName = $this->mDesiredDestName;
464        $this->mDesiredDestName = $this->mFileKey;
465        $this->mTitle = false;
466        $res = $this->verifyPartialFile();
467        $this->mDesiredDestName = $oldDesiredDestName;
468        $this->mTitle = false;
469        if ( is_array( $res ) ) {
470            throw new UploadChunkVerificationException( $res );
471        }
472    }
473
474    /**
475     * Log a status object from FileBackend functions (via FileRepo functions) to the upload log channel.
476     * Return a array with the first error to build up a exception message
477     *
478     * @param Status $status
479     * @param string $logMessage
480     * @param array $context
481     * @return array
482     */
483    private function logFileBackendStatus( Status $status, string $logMessage, array $context = [] ): array {
484        $logger = $this->logger;
485        $errorToThrow = null;
486        $warningToThrow = null;
487
488        foreach ( $status->getErrors() as $errorItem ) {
489            // The message key stands for distinct error situation from the file backend,
490            // each error situation should be shown up in aggregated stats as own point, replace in message
491            $logMessageType = str_replace( '{type}', $errorItem['message'], $logMessage );
492
493            // The message arguments often contains the name of the failing datacenter or file names
494            // and should not show up in aggregated stats, add to context
495            $context['details'] = implode( '; ', $errorItem['params'] );
496            $context['user'] = $this->user->getName();
497
498            if ( $errorItem['type'] === 'error' ) {
499                // Use the first error of the list for the exception text
500                $errorToThrow ??= [ $errorItem['message'], ...$errorItem['params'] ];
501                $logger->error( $logMessageType, $context );
502            } else {
503                // When no error is found, fall back to the first warning
504                $warningToThrow ??= [ $errorItem['message'], ...$errorItem['params'] ];
505                $logger->warning( $logMessageType, $context );
506            }
507        }
508        return $errorToThrow ?? $warningToThrow ?? [ 'unknown', 'no error recorded' ];
509    }
510}
511
512/** @deprecated class alias since 1.46 */
513class_alias( UploadFromChunks::class, 'UploadFromChunks' );