Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
81.46% |
167 / 205 |
|
33.33% |
4 / 12 |
CRAP | |
0.00% |
0 / 1 |
| DjVuImage | |
81.86% |
167 / 204 |
|
33.33% |
4 / 12 |
90.21 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| isValid | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getImageSize | |
85.71% |
6 / 7 |
|
0.00% |
0 / 1 |
2.01 | |||
| dump | n/a |
0 / 0 |
n/a |
0 / 0 |
1 | |||||
| dumpForm | n/a |
0 / 0 |
n/a |
0 / 0 |
5 | |||||
| getInfo | |
68.42% |
13 / 19 |
|
0.00% |
0 / 1 |
7.13 | |||
| readChunk | |
80.00% |
4 / 5 |
|
0.00% |
0 / 1 |
2.03 | |||
| skipChunk | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
3 | |||
| getMultiPageInfo | |
80.00% |
12 / 15 |
|
0.00% |
0 / 1 |
6.29 | |||
| getPageInfo | |
75.00% |
18 / 24 |
|
0.00% |
0 / 1 |
4.25 | |||
| retrieveMetaData | |
91.89% |
68 / 74 |
|
0.00% |
0 / 1 |
18.17 | |||
| pageTextCallback | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| convertDumpToJSON | |
67.65% |
23 / 34 |
|
0.00% |
0 / 1 |
15.10 | |||
| parseFormDjvu | |
84.21% |
16 / 19 |
|
0.00% |
0 / 1 |
4.06 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * DjVu image handler. |
| 4 | * |
| 5 | * Copyright © 2006 Brooke Vibber <bvibber@wikimedia.org> |
| 6 | * https://www.mediawiki.org/ |
| 7 | * |
| 8 | * @license GPL-2.0-or-later |
| 9 | * @file |
| 10 | * @ingroup Media |
| 11 | */ |
| 12 | |
| 13 | namespace MediaWiki\Media; |
| 14 | |
| 15 | use MediaWiki\MainConfigNames; |
| 16 | use MediaWiki\MediaWikiServices; |
| 17 | use MediaWiki\Shell\Shell; |
| 18 | |
| 19 | /** |
| 20 | * Support for detecting/validating DjVu image files and getting |
| 21 | * some basic file metadata (resolution etc) |
| 22 | * |
| 23 | * File format docs are available in source package for DjVuLibre: |
| 24 | * http://djvulibre.djvuzone.org/ |
| 25 | * |
| 26 | * @ingroup Media |
| 27 | */ |
| 28 | class DjVuImage { |
| 29 | |
| 30 | /** |
| 31 | * Memory limit for the DjVu description software |
| 32 | */ |
| 33 | private const DJVUTXT_MEMORY_LIMIT = 300_000_000; |
| 34 | |
| 35 | /** |
| 36 | * @param string $filename The DjVu file name. |
| 37 | */ |
| 38 | public function __construct( private readonly string $filename ) { |
| 39 | } |
| 40 | |
| 41 | /** |
| 42 | * Check if the given file is indeed a valid DjVu image file |
| 43 | */ |
| 44 | public function isValid(): bool { |
| 45 | return $this->getInfo() !== false; |
| 46 | } |
| 47 | |
| 48 | /** |
| 49 | * Return width and height |
| 50 | * @return array An array with "width" and "height" keys, or an empty array on failure. |
| 51 | */ |
| 52 | public function getImageSize(): array { |
| 53 | $data = $this->getInfo(); |
| 54 | |
| 55 | if ( $data !== false ) { |
| 56 | return [ |
| 57 | 'width' => $data['width'], |
| 58 | 'height' => $data['height'] |
| 59 | ]; |
| 60 | } |
| 61 | return []; |
| 62 | } |
| 63 | |
| 64 | /** |
| 65 | * For debugging; dump the IFF chunk structure |
| 66 | * |
| 67 | * @codeCoverageIgnore |
| 68 | */ |
| 69 | public function dump(): void { |
| 70 | $file = fopen( $this->filename, 'rb' ); |
| 71 | $header = fread( $file, 12 ); |
| 72 | $arr = unpack( 'a4magic/a4chunk/NchunkLength', $header ); |
| 73 | $chunk = $arr['chunk']; |
| 74 | $chunkLength = $arr['chunkLength']; |
| 75 | echo "$chunk $chunkLength\n"; |
| 76 | $this->dumpForm( $file, $chunkLength, 1 ); |
| 77 | fclose( $file ); |
| 78 | } |
| 79 | |
| 80 | /** |
| 81 | * @codeCoverageIgnore |
| 82 | * |
| 83 | * @param resource $file |
| 84 | * @param int $length |
| 85 | * @param int $indent |
| 86 | */ |
| 87 | private function dumpForm( $file, int $length, int $indent ): void { |
| 88 | $start = ftell( $file ); |
| 89 | $secondary = fread( $file, 4 ); |
| 90 | echo str_repeat( ' ', $indent * 4 ) . "($secondary)\n"; |
| 91 | while ( ftell( $file ) - $start < $length ) { |
| 92 | $chunkHeader = fread( $file, 8 ); |
| 93 | if ( $chunkHeader == '' ) { |
| 94 | break; |
| 95 | } |
| 96 | $arr = unpack( 'a4chunk/NchunkLength', $chunkHeader ); |
| 97 | $chunk = $arr['chunk']; |
| 98 | $chunkLength = $arr['chunkLength']; |
| 99 | echo str_repeat( ' ', $indent * 4 ) . "$chunk $chunkLength\n"; |
| 100 | |
| 101 | if ( $chunk === 'FORM' ) { |
| 102 | $this->dumpForm( $file, $chunkLength, $indent + 1 ); |
| 103 | } else { |
| 104 | fseek( $file, $chunkLength, SEEK_CUR ); |
| 105 | if ( $chunkLength & 1 ) { |
| 106 | // Padding byte between chunks |
| 107 | fseek( $file, 1, SEEK_CUR ); |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | private function getInfo(): array|false { |
| 114 | // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged |
| 115 | $file = @fopen( $this->filename, 'rb' ); |
| 116 | if ( $file === false ) { |
| 117 | wfDebug( __METHOD__ . ": missing or failed file read" ); |
| 118 | |
| 119 | return false; |
| 120 | } |
| 121 | |
| 122 | $header = fread( $file, 16 ); |
| 123 | $info = false; |
| 124 | |
| 125 | if ( strlen( $header ) < 16 ) { |
| 126 | wfDebug( __METHOD__ . ": too short file header" ); |
| 127 | } else { |
| 128 | $arr = unpack( 'a4magic/a4form/NformLength/a4subtype', $header ); |
| 129 | |
| 130 | $subtype = $arr['subtype']; |
| 131 | if ( $arr['magic'] !== 'AT&T' ) { |
| 132 | wfDebug( __METHOD__ . ": not a DjVu file" ); |
| 133 | } elseif ( $subtype === 'DJVU' ) { |
| 134 | // Single-page document |
| 135 | $info = $this->getPageInfo( $file ); |
| 136 | } elseif ( $subtype === 'DJVM' ) { |
| 137 | // Multi-page document |
| 138 | $info = $this->getMultiPageInfo( $file, $arr['formLength'] ); |
| 139 | } else { |
| 140 | wfDebug( __METHOD__ . ": unrecognized DJVU file type '{$arr['subtype']}'" ); |
| 141 | } |
| 142 | } |
| 143 | fclose( $file ); |
| 144 | |
| 145 | return $info; |
| 146 | } |
| 147 | |
| 148 | /** |
| 149 | * @param resource $file |
| 150 | */ |
| 151 | private function readChunk( $file ): array { |
| 152 | $header = fread( $file, 8 ); |
| 153 | if ( strlen( $header ) < 8 ) { |
| 154 | return [ false, 0 ]; |
| 155 | } |
| 156 | $arr = unpack( 'a4chunk/Nlength', $header ); |
| 157 | |
| 158 | return [ $arr['chunk'], $arr['length'] ]; |
| 159 | } |
| 160 | |
| 161 | /** |
| 162 | * @param resource $file |
| 163 | * @param int $chunkLength |
| 164 | */ |
| 165 | private function skipChunk( $file, int $chunkLength ): void { |
| 166 | fseek( $file, $chunkLength, SEEK_CUR ); |
| 167 | |
| 168 | if ( ( $chunkLength & 1 ) && !feof( $file ) ) { |
| 169 | // padding byte |
| 170 | fseek( $file, 1, SEEK_CUR ); |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | /** |
| 175 | * @param resource $file |
| 176 | * @param int $formLength |
| 177 | * @return array|false |
| 178 | */ |
| 179 | private function getMultiPageInfo( $file, int $formLength ): array|false { |
| 180 | // For now, we'll just look for the first page in the file |
| 181 | // and report its information, hoping others are the same size. |
| 182 | $start = ftell( $file ); |
| 183 | do { |
| 184 | [ $chunk, $length ] = $this->readChunk( $file ); |
| 185 | if ( !$chunk ) { |
| 186 | break; |
| 187 | } |
| 188 | |
| 189 | if ( $chunk === 'FORM' ) { |
| 190 | $subtype = fread( $file, 4 ); |
| 191 | if ( $subtype === 'DJVU' ) { |
| 192 | wfDebug( __METHOD__ . ": found first subpage" ); |
| 193 | |
| 194 | return $this->getPageInfo( $file ); |
| 195 | } |
| 196 | $this->skipChunk( $file, $length - 4 ); |
| 197 | } else { |
| 198 | wfDebug( __METHOD__ . ": skipping '$chunk' chunk" ); |
| 199 | $this->skipChunk( $file, $length ); |
| 200 | } |
| 201 | } while ( $length != 0 && !feof( $file ) && ftell( $file ) - $start < $formLength ); |
| 202 | |
| 203 | wfDebug( __METHOD__ . ": multi-page DJVU file contained no pages" ); |
| 204 | |
| 205 | return false; |
| 206 | } |
| 207 | |
| 208 | /** |
| 209 | * @param resource $file |
| 210 | * @return array|false |
| 211 | */ |
| 212 | private function getPageInfo( $file ): array|false { |
| 213 | [ $chunk, $length ] = $this->readChunk( $file ); |
| 214 | if ( $chunk !== 'INFO' ) { |
| 215 | wfDebug( __METHOD__ . ": expected INFO chunk, got '$chunk'" ); |
| 216 | |
| 217 | return false; |
| 218 | } |
| 219 | |
| 220 | if ( $length < 9 ) { |
| 221 | wfDebug( __METHOD__ . ": INFO should be 9 or 10 bytes, found $length" ); |
| 222 | |
| 223 | return false; |
| 224 | } |
| 225 | $data = fread( $file, $length ); |
| 226 | if ( strlen( $data ) < $length ) { |
| 227 | wfDebug( __METHOD__ . ": INFO chunk cut off" ); |
| 228 | |
| 229 | return false; |
| 230 | } |
| 231 | |
| 232 | $arr = unpack( |
| 233 | 'nwidth/' . |
| 234 | 'nheight/' . |
| 235 | 'Cminor/' . |
| 236 | 'Cmajor/' . |
| 237 | 'vresolution/' . |
| 238 | 'Cgamma', $data ); |
| 239 | |
| 240 | // Newer files have rotation info in byte 10, but we don't use it yet. |
| 241 | return [ |
| 242 | 'width' => $arr['width'], |
| 243 | 'height' => $arr['height'], |
| 244 | 'version' => "{$arr['major']}.{$arr['minor']}", |
| 245 | 'resolution' => $arr['resolution'], |
| 246 | 'gamma' => $arr['gamma'] / 10.0 ]; |
| 247 | } |
| 248 | |
| 249 | /** |
| 250 | * Return an array describing the DjVu image |
| 251 | */ |
| 252 | public function retrieveMetaData(): array|null|false { |
| 253 | $config = MediaWikiServices::getInstance()->getMainConfig(); |
| 254 | $djvuDump = $config->get( MainConfigNames::DjvuDump ); |
| 255 | $djvuTxt = $config->get( MainConfigNames::DjvuTxt ); |
| 256 | |
| 257 | if ( $djvuTxt === null && $djvuDump === null ) { |
| 258 | // @codeCoverageIgnoreStart |
| 259 | return []; |
| 260 | // @codeCoverageIgnoreEnd |
| 261 | } |
| 262 | |
| 263 | if ( !$this->isValid() ) { |
| 264 | return false; |
| 265 | } |
| 266 | |
| 267 | $djvuUseBoxedCommand = $config->get( MainConfigNames::DjvuUseBoxedCommand ); |
| 268 | $shell = $config->get( MainConfigNames::ShellboxShell ); |
| 269 | |
| 270 | $txt = null; |
| 271 | $dump = null; |
| 272 | |
| 273 | if ( $djvuUseBoxedCommand ) { |
| 274 | $command = MediaWikiServices::getInstance()->getShellCommandFactory() |
| 275 | ->createBoxed( 'djvu' ) |
| 276 | ->disableNetwork() |
| 277 | ->firejailDefaultSeccomp() |
| 278 | ->routeName( 'djvu-metadata' ) |
| 279 | ->params( $shell, 'scripts/retrieveDjvuMetaData.sh' ) |
| 280 | ->inputFileFromFile( |
| 281 | 'scripts/retrieveDjvuMetaData.sh', |
| 282 | __DIR__ . '/scripts/retrieveDjvuMetaData.sh' ) |
| 283 | ->inputFileFromFile( 'file.djvu', $this->filename ) |
| 284 | ->memoryLimit( self::DJVUTXT_MEMORY_LIMIT ); |
| 285 | $env = []; |
| 286 | if ( $djvuDump !== null ) { |
| 287 | $env['DJVU_DUMP'] = $djvuDump; |
| 288 | $command->outputFileToString( 'dump' ); |
| 289 | } |
| 290 | if ( $djvuTxt !== null ) { |
| 291 | $env['DJVU_TXT'] = $djvuTxt; |
| 292 | $command->outputFileToString( 'txt' ); |
| 293 | } |
| 294 | |
| 295 | $result = $command |
| 296 | ->environment( $env ) |
| 297 | ->execute(); |
| 298 | if ( $result->getExitCode() !== 0 ) { |
| 299 | wfDebug( 'retrieveDjvuMetaData failed with exit code ' . $result->getExitCode() ); |
| 300 | return false; |
| 301 | } |
| 302 | if ( $djvuDump !== null ) { |
| 303 | if ( $result->wasReceived( 'dump' ) ) { |
| 304 | $dump = $result->getFileContents( 'dump' ); |
| 305 | } else { |
| 306 | wfDebug( __METHOD__ . ": did not receive dump file" ); |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | if ( $djvuTxt !== null ) { |
| 311 | if ( $result->wasReceived( 'txt' ) ) { |
| 312 | $txt = $result->getFileContents( 'txt' ); |
| 313 | } else { |
| 314 | wfDebug( __METHOD__ . ": did not receive text file" ); |
| 315 | } |
| 316 | } |
| 317 | } else { |
| 318 | // No boxedcommand |
| 319 | if ( $djvuDump !== null ) { |
| 320 | // djvudump is faster than djvutoxml (now abandoned) as of version 3.5 |
| 321 | // https://sourceforge.net/p/djvu/bugs/71/ |
| 322 | $cmd = Shell::escape( $djvuDump ) . ' ' . Shell::escape( $this->filename ); |
| 323 | $dump = Shell::command()->unsafeCommand( $cmd )->execute()->getStdout(); |
| 324 | } |
| 325 | if ( $djvuTxt !== null ) { |
| 326 | $cmd = Shell::escape( $djvuTxt ) . ' --detail=page ' . Shell::escape( $this->filename ); |
| 327 | wfDebug( __METHOD__ . ": $cmd" ); |
| 328 | $txt = Shell::command()->unsafeCommand( $cmd )->environment( |
| 329 | [ 'memory' => (string)self::DJVUTXT_MEMORY_LIMIT ] |
| 330 | )->execute(); |
| 331 | if ( $txt->getExitCode() !== 0 ) { |
| 332 | $txt = null; |
| 333 | } else { |
| 334 | $txt = $txt->getStdout(); |
| 335 | } |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | // Convert dump to array |
| 340 | $json = []; |
| 341 | if ( $dump !== null ) { |
| 342 | $data = $this->convertDumpToJSON( $dump ); |
| 343 | if ( $data !== false ) { |
| 344 | $json = [ 'data' => $data ]; |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | // Text layer |
| 349 | $json['text'] = []; |
| 350 | if ( $txt !== null ) { |
| 351 | // Strip some control characters |
| 352 | // Ignore carriage returns |
| 353 | $txt = preg_replace( "/\\\\013/", "", $txt ); |
| 354 | // Replace runs of OCR region separators with a single extra line break |
| 355 | $txt = preg_replace( "/(?:\\\\(035|037))+/", "\n", $txt ); |
| 356 | |
| 357 | $reg = <<<EOR |
| 358 | /\(page\s[\d-]*\s[\d-]*\s[\d-]*\s[\d-]*\s*" |
| 359 | ((?> # Text to match is composed of atoms of either: |
| 360 | \\\\. # - any escaped character |
| 361 | | # - any character different from " and \ |
| 362 | [^"\\\\]+ |
| 363 | )*?) |
| 364 | "\s*\) |
| 365 | | # Or page can be empty ; in this case, djvutxt dumps () |
| 366 | \(\s*()\)/sx |
| 367 | EOR; |
| 368 | $matches = []; |
| 369 | preg_match_all( $reg, $txt, $matches ); |
| 370 | $textEntries = array_filter( |
| 371 | array_map( $this->pageTextCallback( ... ), $matches[1] ), |
| 372 | static fn ( string $t ) => $t !== '' |
| 373 | ); |
| 374 | $json['text'] = $textEntries; |
| 375 | } |
| 376 | |
| 377 | return $json; |
| 378 | } |
| 379 | |
| 380 | private function pageTextCallback( string $match ): string { |
| 381 | // Get rid of invalid UTF-8 |
| 382 | $val = \UtfNormal\Validator::cleanUp( stripcslashes( $match ) ); |
| 383 | return str_replace( '�', '', $val ); |
| 384 | } |
| 385 | |
| 386 | /** |
| 387 | * @param string $dump |
| 388 | * @return array|false |
| 389 | */ |
| 390 | private function convertDumpToJSON( string $dump ): array|false { |
| 391 | if ( $dump === '' ) { |
| 392 | return false; |
| 393 | } |
| 394 | |
| 395 | $dump = str_replace( "\r", '', $dump ); |
| 396 | $line = strtok( $dump, "\n" ); |
| 397 | $m = false; |
| 398 | $good = false; |
| 399 | $result = []; |
| 400 | if ( preg_match( '/^( *)FORM:DJVU/', $line, $m ) ) { |
| 401 | // Single-page |
| 402 | $parsed = $this->parseFormDjvu( $line ); |
| 403 | if ( $parsed ) { |
| 404 | $good = true; |
| 405 | } else { |
| 406 | return false; |
| 407 | } |
| 408 | $result['pages'] = [ $parsed ]; |
| 409 | } elseif ( preg_match( '/^( *)FORM:DJVM/', $line, $m ) ) { |
| 410 | // Multi-page |
| 411 | $parentLevel = strlen( $m[1] ); |
| 412 | // Find DIRM |
| 413 | $line = strtok( "\n" ); |
| 414 | $result['pages'] = []; |
| 415 | while ( $line !== false ) { |
| 416 | $childLevel = strspn( $line, ' ' ); |
| 417 | if ( $childLevel <= $parentLevel ) { |
| 418 | # End of chunk |
| 419 | break; |
| 420 | } |
| 421 | |
| 422 | if ( preg_match( '/^ *DIRM.*indirect/', $line ) ) { |
| 423 | wfDebug( "Indirect multi-page DjVu document, bad for server!" ); |
| 424 | |
| 425 | return false; |
| 426 | } |
| 427 | |
| 428 | if ( preg_match( '/^ *FORM:DJVU/', $line ) ) { |
| 429 | // Found page |
| 430 | $parsed = $this->parseFormDjvu( $line ); |
| 431 | if ( $parsed ) { |
| 432 | $good = true; |
| 433 | } else { |
| 434 | return false; |
| 435 | } |
| 436 | $result['pages'][] = $parsed; |
| 437 | } |
| 438 | $line = strtok( "\n" ); |
| 439 | } |
| 440 | } |
| 441 | if ( !$good ) { |
| 442 | return false; |
| 443 | } |
| 444 | |
| 445 | return $result; |
| 446 | } |
| 447 | |
| 448 | /** @return array|false */ |
| 449 | private function parseFormDjvu( string $line ): array|false { |
| 450 | $parentLevel = strspn( $line, ' ' ); |
| 451 | $line = strtok( "\n" ); |
| 452 | // Find INFO |
| 453 | while ( $line !== false ) { |
| 454 | $childLevel = strspn( $line, ' ' ); |
| 455 | if ( $childLevel <= $parentLevel ) { |
| 456 | // End of chunk |
| 457 | break; |
| 458 | } |
| 459 | |
| 460 | if ( preg_match( |
| 461 | '/^ *INFO *\[\d*] *DjVu *(\d+)x(\d+), *\w*, *(\d+) *dpi, *gamma=([0-9.-]+)/', |
| 462 | $line, |
| 463 | $m |
| 464 | ) ) { |
| 465 | return [ |
| 466 | 'height' => (int)$m[2], |
| 467 | 'width' => (int)$m[1], |
| 468 | 'dpi' => (float)$m[3], |
| 469 | 'gamma' => (float)$m[4], |
| 470 | ]; |
| 471 | } |
| 472 | $line = strtok( "\n" ); |
| 473 | } |
| 474 | |
| 475 | # Not found |
| 476 | return false; |
| 477 | } |
| 478 | } |
| 479 | |
| 480 | /** @deprecated class alias since 1.46 */ |
| 481 | class_alias( DjVuImage::class, 'DjVuImage' ); |