MediaWiki  master
UploadFromChunks.php
Go to the documentation of this file.
1 <?php
2 
4 
35  private $repo;
37  public $stash;
39  public $user;
40 
41  protected $mOffset;
42  protected $mChunkIndex;
43  protected $mFileKey;
44  protected $mVirtualTempPath;
45 
55  public function __construct( User $user, $stash = false, $repo = false ) {
56  $this->user = $user;
57 
58  if ( $repo ) {
59  $this->repo = $repo;
60  } else {
61  $this->repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
62  }
63 
64  if ( $stash ) {
65  $this->stash = $stash;
66  } else {
67  wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() );
68  $this->stash = new UploadStash( $this->repo, $this->user );
69  }
70  }
71 
75  public function tryStashFile( User $user, $isPartial = false ) {
76  try {
77  $this->verifyChunk();
78  } catch ( UploadChunkVerificationException $e ) {
79  return Status::newFatal( $e->msg );
80  }
81 
82  return parent::tryStashFile( $user, $isPartial );
83  }
84 
91  protected function doStashFile( User $user = null ) {
92  // Stash file is the called on creating a new chunk session:
93  $this->mChunkIndex = 0;
94  $this->mOffset = 0;
95 
96  // Create a local stash target
97  $this->mStashFile = parent::doStashFile( $user );
98  // Update the initial file offset (based on file size)
99  $this->mOffset = $this->mStashFile->getSize();
100  $this->mFileKey = $this->mStashFile->getFileKey();
101 
102  // Output a copy of this first to chunk 0 location:
103  $this->outputChunk( $this->mStashFile->getPath() );
104 
105  // Update db table to reflect initial "chunk" state
106  $this->updateChunkStatus();
107 
108  return $this->mStashFile;
109  }
110 
118  public function continueChunks( $name, $key, $webRequestUpload ) {
119  $this->mFileKey = $key;
120  $this->mUpload = $webRequestUpload;
121  // Get the chunk status form the db:
122  $this->getChunkStatus();
123 
124  $metadata = $this->stash->getMetadata( $key );
125  // @phan-suppress-next-line SecurityCheckMulti,SecurityCheck-PathTraversal
126  $this->initializePathInfo( $name,
127  $this->getRealPath( $metadata['us_path'] ),
128  $metadata['us_size'],
129  false
130  );
131  }
132 
137  public function concatenateChunks() {
138  $chunkIndex = $this->getChunkIndex();
139  wfDebug( __METHOD__ . " concatenate {$this->mChunkIndex} chunks:" .
140  $this->getOffset() . ' inx:' . $chunkIndex );
141 
142  // Concatenate all the chunks to mVirtualTempPath
143  $fileList = [];
144  // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
145  for ( $i = 0; $i <= $chunkIndex; $i++ ) {
146  $fileList[] = $this->getVirtualChunkLocation( $i );
147  }
148 
149  // Get the file extension from the last chunk
150  $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath );
151  // Get a 0-byte temp file to perform the concatenation at
152  $tmpFile = MediaWikiServices::getInstance()->getTempFSFileFactory()
153  ->newTempFSFile( 'chunkedupload_', $ext );
154  $tmpPath = false; // fail in concatenate()
155  if ( $tmpFile ) {
156  // keep alive with $this
157  $tmpPath = $tmpFile->bind( $this )->getPath();
158  }
159 
160  // Concatenate the chunks at the temp file
161  $tStart = microtime( true );
162  $status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE );
163  $tAmount = microtime( true ) - $tStart;
164  if ( !$status->isOK() ) {
165  return $status;
166  }
167 
168  wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." );
169 
170  // File system path of the actual full temp file
171  $this->setTempFile( $tmpPath );
172 
173  $ret = $this->verifyUpload();
174  if ( $ret['status'] !== UploadBase::OK ) {
175  wfDebugLog( 'fileconcatenate', "Verification failed for chunked upload" );
176  $status->fatal( $this->getVerificationErrorCode( $ret['status'] ) );
177 
178  return $status;
179  }
180 
181  // Update the mTempPath and mStashFile
182  // (for FileUpload or normal Stash to take over)
183  $tStart = microtime( true );
184  // This is a re-implementation of UploadBase::tryStashFile(), we can't call it because we
185  // override doStashFile() with completely different functionality in this class...
186  $error = $this->runUploadStashFileHook( $this->user );
187  if ( $error ) {
188  $status->fatal( ...$error );
189  return $status;
190  }
191  try {
192  $this->mStashFile = parent::doStashFile( $this->user );
193  } catch ( UploadStashException $e ) {
194  $status->fatal( 'uploadstash-exception', get_class( $e ), $e->getMessage() );
195  return $status;
196  }
197 
198  $tAmount = microtime( true ) - $tStart;
199  $this->mStashFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
200  wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." );
201 
202  return $status;
203  }
204 
210  private function getVirtualChunkLocation( $index ) {
211  return $this->repo->getVirtualUrl( 'temp' ) .
212  '/' .
213  $this->repo->getHashPath(
214  $this->getChunkFileKey( $index )
215  ) .
216  $this->getChunkFileKey( $index );
217  }
218 
227  public function addChunk( $chunkPath, $chunkSize, $offset ) {
228  // Get the offset before we add the chunk to the file system
229  $preAppendOffset = $this->getOffset();
230 
231  if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize() ) {
232  $status = Status::newFatal( 'file-too-large' );
233  } else {
234  // Make sure the client is uploading the correct chunk with a matching offset.
235  if ( $preAppendOffset == $offset ) {
236  // Update local chunk index for the current chunk
237  $this->mChunkIndex++;
238  try {
239  # For some reason mTempPath is set to first part
240  $oldTemp = $this->mTempPath;
241  $this->mTempPath = $chunkPath;
242  $this->verifyChunk();
243  $this->mTempPath = $oldTemp;
244  } catch ( UploadChunkVerificationException $e ) {
245  return Status::newFatal( $e->msg );
246  }
247  $status = $this->outputChunk( $chunkPath );
248  if ( $status->isGood() ) {
249  // Update local offset:
250  $this->mOffset = $preAppendOffset + $chunkSize;
251  // Update chunk table status db
252  $this->updateChunkStatus();
253  }
254  } else {
255  $status = Status::newFatal( 'invalid-chunk-offset' );
256  }
257  }
258 
259  return $status;
260  }
261 
265  private function updateChunkStatus() {
266  wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" .
267  $this->getOffset() . ' inx:' . $this->getChunkIndex() );
268 
269  $dbw = $this->repo->getPrimaryDB();
270  $dbw->update(
271  'uploadstash',
272  [
273  'us_status' => 'chunks',
274  'us_chunk_inx' => $this->getChunkIndex(),
275  'us_size' => $this->getOffset()
276  ],
277  [ 'us_key' => $this->mFileKey ],
278  __METHOD__
279  );
280  }
281 
285  private function getChunkStatus() {
286  // get primary db to avoid race conditions.
287  // Otherwise, if chunk upload time < replag there will be spurious errors
288  $dbw = $this->repo->getPrimaryDB();
289  $row = $dbw->selectRow(
290  'uploadstash',
291  [
292  'us_chunk_inx',
293  'us_size',
294  'us_path',
295  ],
296  [ 'us_key' => $this->mFileKey ],
297  __METHOD__
298  );
299  // Handle result:
300  if ( $row ) {
301  $this->mChunkIndex = $row->us_chunk_inx;
302  $this->mOffset = $row->us_size;
303  $this->mVirtualTempPath = $row->us_path;
304  }
305  }
306 
311  private function getChunkIndex() {
312  if ( $this->mChunkIndex !== null ) {
313  return $this->mChunkIndex;
314  }
315 
316  return 0;
317  }
318 
323  public function getOffset() {
324  if ( $this->mOffset !== null ) {
325  return $this->mOffset;
326  }
327 
328  return 0;
329  }
330 
338  private function outputChunk( $chunkPath ) {
339  // Key is fileKey + chunk index
340  $fileKey = $this->getChunkFileKey();
341 
342  // Store the chunk per its indexed fileKey:
343  $hashPath = $this->repo->getHashPath( $fileKey );
344  $storeStatus = $this->repo->quickImport( $chunkPath,
345  $this->repo->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
346 
347  // Check for error in stashing the chunk:
348  if ( !$storeStatus->isOK() ) {
349  $error = $storeStatus->getErrorsArray();
350  $error = reset( $error );
351  if ( !count( $error ) ) {
352  $error = $storeStatus->getWarningsArray();
353  $error = reset( $error );
354  if ( !count( $error ) ) {
355  $error = [ 'unknown', 'no error recorded' ];
356  }
357  }
358  throw new UploadChunkFileException( "Error storing file in '$chunkPath': " .
359  implode( '; ', $error ) );
360  }
361 
362  return $storeStatus;
363  }
364 
365  private function getChunkFileKey( $index = null ) {
366  if ( $index === null ) {
367  $index = $this->getChunkIndex();
368  }
369 
370  return $this->mFileKey . '.' . $index;
371  }
372 
378  private function verifyChunk() {
379  // Rest mDesiredDestName here so we verify the name as if it were mFileKey
380  $oldDesiredDestName = $this->mDesiredDestName;
381  $this->mDesiredDestName = $this->mFileKey;
382  $this->mTitle = false;
383  $res = $this->verifyPartialFile();
384  $this->mDesiredDestName = $oldDesiredDestName;
385  $this->mTitle = false;
386  if ( is_array( $res ) ) {
388  }
389  }
390 }
UploadBase\getRealPath
getRealPath( $srcPath)
Definition: UploadBase.php:332
UploadFromChunks\updateChunkStatus
updateChunkStatus()
Update the chunk db table with the current status:
Definition: UploadFromChunks.php:265
StatusValue\newFatal
static newFatal( $message,... $parameters)
Factory function for fatal errors.
Definition: StatusValue.php:70
UploadFromChunks\getOffset
getOffset()
Get the offset at which the next uploaded chunk will be appended to.
Definition: UploadFromChunks.php:323
UploadFromChunks\getChunkIndex
getChunkIndex()
Get the current Chunk index.
Definition: UploadFromChunks.php:311
UploadFromChunks\outputChunk
outputChunk( $chunkPath)
Output the chunk to disk.
Definition: UploadFromChunks.php:338
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:200
UploadFromChunks\$mVirtualTempPath
$mVirtualTempPath
Definition: UploadFromChunks.php:44
UploadStash
UploadStash is intended to accomplish a few things:
Definition: UploadStash.php:55
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1583
UploadBase\OK
const OK
Definition: UploadBase.php:103
$res
$res
Definition: testCompression.php:57
UploadFromChunks\__construct
__construct(User $user, $stash=false, $repo=false)
@noinspection PhpMissingParentConstructorInspection
Definition: UploadFromChunks.php:55
wfDebugLog
wfDebugLog( $logGroup, $text, $dest='all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not.
Definition: GlobalFunctions.php:958
UploadBase\runUploadStashFileHook
runUploadStashFileHook(User $user)
Definition: UploadBase.php:1176
UploadBase\$mTempPath
string null $mTempPath
Local file system path to the file to upload (or a local copy)
Definition: UploadBase.php:51
UploadBase\getVerificationErrorCode
getVerificationErrorCode( $error)
Definition: UploadBase.php:120
UploadChunkFileException
@newable
Definition: UploadChunkFileException.php:27
UploadFromChunks\addChunk
addChunk( $chunkPath, $chunkSize, $offset)
Add a chunk to the temporary directory.
Definition: UploadFromChunks.php:227
UploadBase\setTempFile
setTempFile( $tempPath, $fileSize=null)
Definition: UploadBase.php:281
UploadFromChunks\$mChunkIndex
$mChunkIndex
Definition: UploadFromChunks.php:42
User\getId
getId( $wikiId=self::LOCAL)
Get the user's ID.
Definition: User.php:1879
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:894
UploadFromChunks\$mOffset
$mOffset
Definition: UploadFromChunks.php:41
UploadFromChunks\concatenateChunks
concatenateChunks()
Append the final chunk and ready file for parent::performUpload()
Definition: UploadFromChunks.php:137
UploadFromChunks\getChunkStatus
getChunkStatus()
Get the chunk db state and populate update relevant local values.
Definition: UploadFromChunks.php:285
MWException\msg
msg( $key, $fallback,... $params)
Get a message from i18n.
Definition: MWException.php:81
UploadFromChunks\tryStashFile
tryStashFile(User $user, $isPartial=false)
Like stashFile(), but respects extensions' wishes to prevent the stashing.verifyUpload() must be call...
Definition: UploadFromChunks.php:75
UploadFromChunks\$repo
LocalRepo $repo
Definition: UploadFromChunks.php:35
UploadFromChunks\getChunkFileKey
getChunkFileKey( $index=null)
Definition: UploadFromChunks.php:365
UploadBase\verifyPartialFile
verifyPartialFile()
A verification routine suitable for partial files.
Definition: UploadBase.php:545
UploadBase\getMaxUploadSize
static getMaxUploadSize( $forType=null)
Get MediaWiki's maximum uploaded file size for given type of upload, based on $wgMaxUploadSize.
Definition: UploadBase.php:2206
UploadFromChunks\$mFileKey
$mFileKey
Definition: UploadFromChunks.php:43
UploadBase\initializePathInfo
initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile=false)
Definition: UploadBase.php:260
UploadFromChunks\getVirtualChunkLocation
getVirtualChunkLocation( $index)
Returns the virtual chunk location:
Definition: UploadFromChunks.php:210
UploadFromFile\verifyUpload
verifyUpload()
Definition: UploadFromFile.php:80
UploadFromChunks\doStashFile
doStashFile(User $user=null)
Calls the parent doStashFile and updates the uploadsession table to handle "chunks".
Definition: UploadFromChunks.php:91
UploadFromChunks\continueChunks
continueChunks( $name, $key, $webRequestUpload)
Continue chunk uploading.
Definition: UploadFromChunks.php:118
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
FileRepo\DELETE_SOURCE
const DELETE_SOURCE
Definition: FileRepo.php:46
UploadBase\$mStashFile
UploadStashFile null $mStashFile
Definition: UploadBase.php:73
UploadFromChunks\$user
User $user
Definition: UploadFromChunks.php:39
UploadFromChunks
Implements uploading from chunks.
Definition: UploadFromChunks.php:33
UploadFromChunks\$stash
UploadStash $stash
Definition: UploadFromChunks.php:37
User
The User object encapsulates all of the user-specific settings (user_id, name, rights,...
Definition: User.php:67
UploadFromFile
Implements regular file uploads.
Definition: UploadFromFile.php:30
LocalRepo
A repository that stores files in the local filesystem and registers them in the wiki's own database.
Definition: LocalRepo.php:41
UploadBase\$mDesiredDestName
string null $mDesiredDestName
Definition: UploadBase.php:55
UploadStashException
@newable
Definition: UploadStashException.php:27
UploadChunkVerificationException
@newable
Definition: UploadChunkVerificationException.php:27
UploadFromChunks\verifyChunk
verifyChunk()
Verify that the chunk isn't really an evil html file.
Definition: UploadFromChunks.php:378