Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 226 |
|
0.00% |
0 / 14 |
CRAP | |
0.00% |
0 / 1 |
| DjVuImage | |
0.00% |
0 / 226 |
|
0.00% |
0 / 14 |
4290 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| isValid | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| getImageSize | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
6 | |||
| dump | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
2 | |||
| dumpForm | |
0.00% |
0 / 16 |
|
0.00% |
0 / 1 |
30 | |||
| getInfo | |
0.00% |
0 / 21 |
|
0.00% |
0 / 1 |
42 | |||
| readChunk | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
6 | |||
| skipChunk | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
12 | |||
| getMultiPageInfo | |
0.00% |
0 / 15 |
|
0.00% |
0 / 1 |
42 | |||
| getPageInfo | |
0.00% |
0 / 24 |
|
0.00% |
0 / 1 |
20 | |||
| retrieveMetaData | |
0.00% |
0 / 69 |
|
0.00% |
0 / 1 |
342 | |||
| pageTextCallback | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| convertDumpToJSON | |
0.00% |
0 / 34 |
|
0.00% |
0 / 1 |
132 | |||
| parseFormDjvu | |
0.00% |
0 / 19 |
|
0.00% |
0 / 1 |
20 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * DjVu image handler. |
| 4 | * |
| 5 | * Copyright © 2006 Brooke Vibber <bvibber@wikimedia.org> |
| 6 | * https://www.mediawiki.org/ |
| 7 | * |
| 8 | * @license GPL-2.0-or-later |
| 9 | * @file |
| 10 | * @ingroup Media |
| 11 | */ |
| 12 | |
| 13 | use MediaWiki\MainConfigNames; |
| 14 | use MediaWiki\MediaWikiServices; |
| 15 | use MediaWiki\Shell\Shell; |
| 16 | use Wikimedia\AtEase\AtEase; |
| 17 | |
| 18 | /** |
| 19 | * Support for detecting/validating DjVu image files and getting |
| 20 | * some basic file metadata (resolution etc) |
| 21 | * |
| 22 | * File format docs are available in source package for DjVuLibre: |
| 23 | * http://djvulibre.djvuzone.org/ |
| 24 | * |
| 25 | * @ingroup Media |
| 26 | */ |
| 27 | class DjVuImage { |
| 28 | |
| 29 | /** |
| 30 | * Memory limit for the DjVu description software |
| 31 | */ |
| 32 | private const DJVUTXT_MEMORY_LIMIT = 300_000_000; |
| 33 | |
| 34 | /** @var string */ |
| 35 | private $mFilename; |
| 36 | |
| 37 | /** |
| 38 | * @param string $filename The DjVu file name. |
| 39 | */ |
| 40 | public function __construct( $filename ) { |
| 41 | $this->mFilename = $filename; |
| 42 | } |
| 43 | |
| 44 | /** |
| 45 | * Check if the given file is indeed a valid DjVu image file |
| 46 | * @return bool |
| 47 | */ |
| 48 | public function isValid() { |
| 49 | $info = $this->getInfo(); |
| 50 | |
| 51 | return $info !== false; |
| 52 | } |
| 53 | |
| 54 | /** |
| 55 | * Return width and height |
| 56 | * @return array An array with "width" and "height" keys, or an empty array on failure. |
| 57 | */ |
| 58 | public function getImageSize() { |
| 59 | $data = $this->getInfo(); |
| 60 | |
| 61 | if ( $data !== false ) { |
| 62 | return [ |
| 63 | 'width' => $data['width'], |
| 64 | 'height' => $data['height'] |
| 65 | ]; |
| 66 | } |
| 67 | return []; |
| 68 | } |
| 69 | |
| 70 | // --------- |
| 71 | |
| 72 | /** |
| 73 | * For debugging; dump the IFF chunk structure |
| 74 | */ |
| 75 | public function dump() { |
| 76 | $file = fopen( $this->mFilename, 'rb' ); |
| 77 | $header = fread( $file, 12 ); |
| 78 | $arr = unpack( 'a4magic/a4chunk/NchunkLength', $header ); |
| 79 | $chunk = $arr['chunk']; |
| 80 | $chunkLength = $arr['chunkLength']; |
| 81 | echo "$chunk $chunkLength\n"; |
| 82 | $this->dumpForm( $file, $chunkLength, 1 ); |
| 83 | fclose( $file ); |
| 84 | } |
| 85 | |
| 86 | /** |
| 87 | * @param resource $file |
| 88 | * @param int $length |
| 89 | * @param int $indent |
| 90 | */ |
| 91 | private function dumpForm( $file, int $length, int $indent ) { |
| 92 | $start = ftell( $file ); |
| 93 | $secondary = fread( $file, 4 ); |
| 94 | echo str_repeat( ' ', $indent * 4 ) . "($secondary)\n"; |
| 95 | while ( ftell( $file ) - $start < $length ) { |
| 96 | $chunkHeader = fread( $file, 8 ); |
| 97 | if ( $chunkHeader == '' ) { |
| 98 | break; |
| 99 | } |
| 100 | $arr = unpack( 'a4chunk/NchunkLength', $chunkHeader ); |
| 101 | $chunk = $arr['chunk']; |
| 102 | $chunkLength = $arr['chunkLength']; |
| 103 | echo str_repeat( ' ', $indent * 4 ) . "$chunk $chunkLength\n"; |
| 104 | |
| 105 | if ( $chunk === 'FORM' ) { |
| 106 | $this->dumpForm( $file, $chunkLength, $indent + 1 ); |
| 107 | } else { |
| 108 | fseek( $file, $chunkLength, SEEK_CUR ); |
| 109 | if ( $chunkLength & 1 ) { |
| 110 | // Padding byte between chunks |
| 111 | fseek( $file, 1, SEEK_CUR ); |
| 112 | } |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | /** @return array|false */ |
| 118 | private function getInfo() { |
| 119 | AtEase::suppressWarnings(); |
| 120 | $file = fopen( $this->mFilename, 'rb' ); |
| 121 | AtEase::restoreWarnings(); |
| 122 | if ( $file === false ) { |
| 123 | wfDebug( __METHOD__ . ": missing or failed file read" ); |
| 124 | |
| 125 | return false; |
| 126 | } |
| 127 | |
| 128 | $header = fread( $file, 16 ); |
| 129 | $info = false; |
| 130 | |
| 131 | if ( strlen( $header ) < 16 ) { |
| 132 | wfDebug( __METHOD__ . ": too short file header" ); |
| 133 | } else { |
| 134 | $arr = unpack( 'a4magic/a4form/NformLength/a4subtype', $header ); |
| 135 | |
| 136 | $subtype = $arr['subtype']; |
| 137 | if ( $arr['magic'] !== 'AT&T' ) { |
| 138 | wfDebug( __METHOD__ . ": not a DjVu file" ); |
| 139 | } elseif ( $subtype === 'DJVU' ) { |
| 140 | // Single-page document |
| 141 | $info = $this->getPageInfo( $file ); |
| 142 | } elseif ( $subtype === 'DJVM' ) { |
| 143 | // Multi-page document |
| 144 | $info = $this->getMultiPageInfo( $file, $arr['formLength'] ); |
| 145 | } else { |
| 146 | wfDebug( __METHOD__ . ": unrecognized DJVU file type '{$arr['subtype']}'" ); |
| 147 | } |
| 148 | } |
| 149 | fclose( $file ); |
| 150 | |
| 151 | return $info; |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * @param resource $file |
| 156 | */ |
| 157 | private function readChunk( $file ): array { |
| 158 | $header = fread( $file, 8 ); |
| 159 | if ( strlen( $header ) < 8 ) { |
| 160 | return [ false, 0 ]; |
| 161 | } |
| 162 | $arr = unpack( 'a4chunk/Nlength', $header ); |
| 163 | |
| 164 | return [ $arr['chunk'], $arr['length'] ]; |
| 165 | } |
| 166 | |
| 167 | /** |
| 168 | * @param resource $file |
| 169 | * @param int $chunkLength |
| 170 | */ |
| 171 | private function skipChunk( $file, int $chunkLength ) { |
| 172 | fseek( $file, $chunkLength, SEEK_CUR ); |
| 173 | |
| 174 | if ( ( $chunkLength & 1 ) && !feof( $file ) ) { |
| 175 | // padding byte |
| 176 | fseek( $file, 1, SEEK_CUR ); |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | /** |
| 181 | * @param resource $file |
| 182 | * @param int $formLength |
| 183 | * @return array|false |
| 184 | */ |
| 185 | private function getMultiPageInfo( $file, int $formLength ) { |
| 186 | // For now, we'll just look for the first page in the file |
| 187 | // and report its information, hoping others are the same size. |
| 188 | $start = ftell( $file ); |
| 189 | do { |
| 190 | [ $chunk, $length ] = $this->readChunk( $file ); |
| 191 | if ( !$chunk ) { |
| 192 | break; |
| 193 | } |
| 194 | |
| 195 | if ( $chunk === 'FORM' ) { |
| 196 | $subtype = fread( $file, 4 ); |
| 197 | if ( $subtype === 'DJVU' ) { |
| 198 | wfDebug( __METHOD__ . ": found first subpage" ); |
| 199 | |
| 200 | return $this->getPageInfo( $file ); |
| 201 | } |
| 202 | $this->skipChunk( $file, $length - 4 ); |
| 203 | } else { |
| 204 | wfDebug( __METHOD__ . ": skipping '$chunk' chunk" ); |
| 205 | $this->skipChunk( $file, $length ); |
| 206 | } |
| 207 | } while ( $length != 0 && !feof( $file ) && ftell( $file ) - $start < $formLength ); |
| 208 | |
| 209 | wfDebug( __METHOD__ . ": multi-page DJVU file contained no pages" ); |
| 210 | |
| 211 | return false; |
| 212 | } |
| 213 | |
| 214 | /** |
| 215 | * @param resource $file |
| 216 | * @return array|false |
| 217 | */ |
| 218 | private function getPageInfo( $file ) { |
| 219 | [ $chunk, $length ] = $this->readChunk( $file ); |
| 220 | if ( $chunk !== 'INFO' ) { |
| 221 | wfDebug( __METHOD__ . ": expected INFO chunk, got '$chunk'" ); |
| 222 | |
| 223 | return false; |
| 224 | } |
| 225 | |
| 226 | if ( $length < 9 ) { |
| 227 | wfDebug( __METHOD__ . ": INFO should be 9 or 10 bytes, found $length" ); |
| 228 | |
| 229 | return false; |
| 230 | } |
| 231 | $data = fread( $file, $length ); |
| 232 | if ( strlen( $data ) < $length ) { |
| 233 | wfDebug( __METHOD__ . ": INFO chunk cut off" ); |
| 234 | |
| 235 | return false; |
| 236 | } |
| 237 | |
| 238 | $arr = unpack( |
| 239 | 'nwidth/' . |
| 240 | 'nheight/' . |
| 241 | 'Cminor/' . |
| 242 | 'Cmajor/' . |
| 243 | 'vresolution/' . |
| 244 | 'Cgamma', $data ); |
| 245 | |
| 246 | # Newer files have rotation info in byte 10, but we don't use it yet. |
| 247 | |
| 248 | return [ |
| 249 | 'width' => $arr['width'], |
| 250 | 'height' => $arr['height'], |
| 251 | 'version' => "{$arr['major']}.{$arr['minor']}", |
| 252 | 'resolution' => $arr['resolution'], |
| 253 | 'gamma' => $arr['gamma'] / 10.0 ]; |
| 254 | } |
| 255 | |
| 256 | /** |
| 257 | * Return an array describing the DjVu image |
| 258 | * @return array|null|false |
| 259 | */ |
| 260 | public function retrieveMetaData() { |
| 261 | $config = MediaWikiServices::getInstance()->getMainConfig(); |
| 262 | $djvuDump = $config->get( MainConfigNames::DjvuDump ); |
| 263 | $djvuTxt = $config->get( MainConfigNames::DjvuTxt ); |
| 264 | $djvuUseBoxedCommand = $config->get( MainConfigNames::DjvuUseBoxedCommand ); |
| 265 | $shell = $config->get( MainConfigNames::ShellboxShell ); |
| 266 | if ( !$this->isValid() ) { |
| 267 | return false; |
| 268 | } |
| 269 | |
| 270 | if ( $djvuTxt === null && $djvuDump === null ) { |
| 271 | return []; |
| 272 | } |
| 273 | |
| 274 | $txt = null; |
| 275 | $dump = null; |
| 276 | |
| 277 | if ( $djvuUseBoxedCommand ) { |
| 278 | $command = MediaWikiServices::getInstance()->getShellCommandFactory() |
| 279 | ->createBoxed( 'djvu' ) |
| 280 | ->disableNetwork() |
| 281 | ->firejailDefaultSeccomp() |
| 282 | ->routeName( 'djvu-metadata' ) |
| 283 | ->params( $shell, 'scripts/retrieveDjvuMetaData.sh' ) |
| 284 | ->inputFileFromFile( |
| 285 | 'scripts/retrieveDjvuMetaData.sh', |
| 286 | __DIR__ . '/scripts/retrieveDjvuMetaData.sh' ) |
| 287 | ->inputFileFromFile( 'file.djvu', $this->mFilename ) |
| 288 | ->memoryLimit( self::DJVUTXT_MEMORY_LIMIT ); |
| 289 | $env = []; |
| 290 | if ( $djvuDump !== null ) { |
| 291 | $env['DJVU_DUMP'] = $djvuDump; |
| 292 | $command->outputFileToString( 'dump' ); |
| 293 | } |
| 294 | if ( $djvuTxt !== null ) { |
| 295 | $env['DJVU_TXT'] = $djvuTxt; |
| 296 | $command->outputFileToString( 'txt' ); |
| 297 | } |
| 298 | |
| 299 | $result = $command |
| 300 | ->environment( $env ) |
| 301 | ->execute(); |
| 302 | if ( $result->getExitCode() !== 0 ) { |
| 303 | wfDebug( 'retrieveDjvuMetaData failed with exit code ' . $result->getExitCode() ); |
| 304 | return false; |
| 305 | } |
| 306 | if ( $djvuDump !== null ) { |
| 307 | if ( $result->wasReceived( 'dump' ) ) { |
| 308 | $dump = $result->getFileContents( 'dump' ); |
| 309 | } else { |
| 310 | wfDebug( __METHOD__ . ": did not receive dump file" ); |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | if ( $djvuTxt !== null ) { |
| 315 | if ( $result->wasReceived( 'txt' ) ) { |
| 316 | $txt = $result->getFileContents( 'txt' ); |
| 317 | } else { |
| 318 | wfDebug( __METHOD__ . ": did not receive text file" ); |
| 319 | } |
| 320 | } |
| 321 | } else { // No boxedcommand |
| 322 | if ( $djvuDump !== null ) { |
| 323 | # djvudump is faster than djvutoxml (now abandoned) as of version 3.5 |
| 324 | # https://sourceforge.net/p/djvu/bugs/71/ |
| 325 | $cmd = Shell::escape( $djvuDump ) . ' ' . Shell::escape( $this->mFilename ); |
| 326 | $dump = wfShellExec( $cmd ); |
| 327 | } |
| 328 | if ( $djvuTxt !== null ) { |
| 329 | $cmd = Shell::escape( $djvuTxt ) . ' --detail=page ' . Shell::escape( $this->mFilename ); |
| 330 | wfDebug( __METHOD__ . ": $cmd" ); |
| 331 | $retval = 0; |
| 332 | $txt = wfShellExec( $cmd, $retval, [], [ 'memory' => self::DJVUTXT_MEMORY_LIMIT ] ); |
| 333 | if ( $retval !== 0 ) { |
| 334 | $txt = null; |
| 335 | } |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | # Convert dump to array |
| 340 | $json = []; |
| 341 | if ( $dump !== null ) { |
| 342 | $data = $this->convertDumpToJSON( $dump ); |
| 343 | if ( $data !== false ) { |
| 344 | $json = [ 'data' => $data ]; |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | # Text layer |
| 349 | if ( $txt !== null ) { |
| 350 | # Strip some control characters |
| 351 | # Ignore carriage returns |
| 352 | $txt = preg_replace( "/\\\\013/", "", $txt ); |
| 353 | # Replace runs of OCR region separators with a single extra line break |
| 354 | $txt = preg_replace( "/(?:\\\\(035|037))+/", "\n", $txt ); |
| 355 | |
| 356 | $reg = <<<EOR |
| 357 | /\(page\s[\d-]*\s[\d-]*\s[\d-]*\s[\d-]*\s*" |
| 358 | ((?> # Text to match is composed of atoms of either: |
| 359 | \\\\. # - any escaped character |
| 360 | | # - any character different from " and \ |
| 361 | [^"\\\\]+ |
| 362 | )*?) |
| 363 | "\s*\) |
| 364 | | # Or page can be empty ; in this case, djvutxt dumps () |
| 365 | \(\s*()\)/sx |
| 366 | EOR; |
| 367 | $matches = []; |
| 368 | preg_match_all( $reg, $txt, $matches ); |
| 369 | $json['text'] = array_map( $this->pageTextCallback( ... ), $matches[1] ); |
| 370 | } else { |
| 371 | $json['text'] = []; |
| 372 | } |
| 373 | |
| 374 | return $json; |
| 375 | } |
| 376 | |
| 377 | private function pageTextCallback( string $match ): string { |
| 378 | # Get rid of invalid UTF-8 |
| 379 | $val = UtfNormal\Validator::cleanUp( stripcslashes( $match ) ); |
| 380 | return str_replace( '�', '', $val ); |
| 381 | } |
| 382 | |
| 383 | /** |
| 384 | * @param string $dump |
| 385 | * @return array|false |
| 386 | */ |
| 387 | private function convertDumpToJSON( $dump ) { |
| 388 | if ( strval( $dump ) == '' ) { |
| 389 | return false; |
| 390 | } |
| 391 | |
| 392 | $dump = str_replace( "\r", '', $dump ); |
| 393 | $line = strtok( $dump, "\n" ); |
| 394 | $m = false; |
| 395 | $good = false; |
| 396 | $result = []; |
| 397 | if ( preg_match( '/^( *)FORM:DJVU/', $line, $m ) ) { |
| 398 | # Single-page |
| 399 | $parsed = $this->parseFormDjvu( $line ); |
| 400 | if ( $parsed ) { |
| 401 | $good = true; |
| 402 | } else { |
| 403 | return false; |
| 404 | } |
| 405 | $result['pages'] = [ $parsed ]; |
| 406 | } elseif ( preg_match( '/^( *)FORM:DJVM/', $line, $m ) ) { |
| 407 | # Multi-page |
| 408 | $parentLevel = strlen( $m[1] ); |
| 409 | # Find DIRM |
| 410 | $line = strtok( "\n" ); |
| 411 | $result['pages'] = []; |
| 412 | while ( $line !== false ) { |
| 413 | $childLevel = strspn( $line, ' ' ); |
| 414 | if ( $childLevel <= $parentLevel ) { |
| 415 | # End of chunk |
| 416 | break; |
| 417 | } |
| 418 | |
| 419 | if ( preg_match( '/^ *DIRM.*indirect/', $line ) ) { |
| 420 | wfDebug( "Indirect multi-page DjVu document, bad for server!" ); |
| 421 | |
| 422 | return false; |
| 423 | } |
| 424 | |
| 425 | if ( preg_match( '/^ *FORM:DJVU/', $line ) ) { |
| 426 | # Found page |
| 427 | $parsed = $this->parseFormDjvu( $line ); |
| 428 | if ( $parsed ) { |
| 429 | $good = true; |
| 430 | } else { |
| 431 | return false; |
| 432 | } |
| 433 | $result['pages'][] = $parsed; |
| 434 | } |
| 435 | $line = strtok( "\n" ); |
| 436 | } |
| 437 | } |
| 438 | if ( !$good ) { |
| 439 | return false; |
| 440 | } |
| 441 | |
| 442 | return $result; |
| 443 | } |
| 444 | |
| 445 | /** @return array|false */ |
| 446 | private function parseFormDjvu( string $line ) { |
| 447 | $parentLevel = strspn( $line, ' ' ); |
| 448 | $line = strtok( "\n" ); |
| 449 | # Find INFO |
| 450 | while ( $line !== false ) { |
| 451 | $childLevel = strspn( $line, ' ' ); |
| 452 | if ( $childLevel <= $parentLevel ) { |
| 453 | # End of chunk |
| 454 | break; |
| 455 | } |
| 456 | |
| 457 | if ( preg_match( |
| 458 | '/^ *INFO *\[\d*] *DjVu *(\d+)x(\d+), *\w*, *(\d+) *dpi, *gamma=([0-9.-]+)/', |
| 459 | $line, |
| 460 | $m |
| 461 | ) ) { |
| 462 | return [ |
| 463 | 'height' => (int)$m[2], |
| 464 | 'width' => (int)$m[1], |
| 465 | 'dpi' => (float)$m[3], |
| 466 | 'gamma' => (float)$m[4], |
| 467 | ]; |
| 468 | } |
| 469 | $line = strtok( "\n" ); |
| 470 | } |
| 471 | |
| 472 | # Not found |
| 473 | return false; |
| 474 | } |
| 475 | } |