MediaWiki master
DjVuImage.php
Go to the documentation of this file.
1<?php
30use Wikimedia\AtEase\AtEase;
31
41class DjVuImage {
42
46 private const DJVUTXT_MEMORY_LIMIT = 300_000_000;
47
49 private $mFilename;
50
54 public function __construct( $filename ) {
55 $this->mFilename = $filename;
56 }
57
62 public function isValid() {
63 $info = $this->getInfo();
64
65 return $info !== false;
66 }
67
72 public function getImageSize() {
73 $data = $this->getInfo();
74
75 if ( $data !== false ) {
76 return [
77 'width' => $data['width'],
78 'height' => $data['height']
79 ];
80 }
81 return [];
82 }
83
84 // ---------
85
89 public function dump() {
90 $file = fopen( $this->mFilename, 'rb' );
91 $header = fread( $file, 12 );
92 $arr = unpack( 'a4magic/a4chunk/NchunkLength', $header );
93 $chunk = $arr['chunk'];
94 $chunkLength = $arr['chunkLength'];
95 echo "$chunk $chunkLength\n";
96 $this->dumpForm( $file, $chunkLength, 1 );
97 fclose( $file );
98 }
99
105 private function dumpForm( $file, int $length, int $indent ) {
106 $start = ftell( $file );
107 $secondary = fread( $file, 4 );
108 echo str_repeat( ' ', $indent * 4 ) . "($secondary)\n";
109 while ( ftell( $file ) - $start < $length ) {
110 $chunkHeader = fread( $file, 8 );
111 if ( $chunkHeader == '' ) {
112 break;
113 }
114 $arr = unpack( 'a4chunk/NchunkLength', $chunkHeader );
115 $chunk = $arr['chunk'];
116 $chunkLength = $arr['chunkLength'];
117 echo str_repeat( ' ', $indent * 4 ) . "$chunk $chunkLength\n";
118
119 if ( $chunk === 'FORM' ) {
120 $this->dumpForm( $file, $chunkLength, $indent + 1 );
121 } else {
122 fseek( $file, $chunkLength, SEEK_CUR );
123 if ( $chunkLength & 1 ) {
124 // Padding byte between chunks
125 fseek( $file, 1, SEEK_CUR );
126 }
127 }
128 }
129 }
130
132 private function getInfo() {
133 AtEase::suppressWarnings();
134 $file = fopen( $this->mFilename, 'rb' );
135 AtEase::restoreWarnings();
136 if ( $file === false ) {
137 wfDebug( __METHOD__ . ": missing or failed file read" );
138
139 return false;
140 }
141
142 $header = fread( $file, 16 );
143 $info = false;
144
145 if ( strlen( $header ) < 16 ) {
146 wfDebug( __METHOD__ . ": too short file header" );
147 } else {
148 $arr = unpack( 'a4magic/a4form/NformLength/a4subtype', $header );
149
150 $subtype = $arr['subtype'];
151 if ( $arr['magic'] !== 'AT&T' ) {
152 wfDebug( __METHOD__ . ": not a DjVu file" );
153 } elseif ( $subtype === 'DJVU' ) {
154 // Single-page document
155 $info = $this->getPageInfo( $file );
156 } elseif ( $subtype === 'DJVM' ) {
157 // Multi-page document
158 $info = $this->getMultiPageInfo( $file, $arr['formLength'] );
159 } else {
160 wfDebug( __METHOD__ . ": unrecognized DJVU file type '{$arr['subtype']}'" );
161 }
162 }
163 fclose( $file );
164
165 return $info;
166 }
167
171 private function readChunk( $file ): array {
172 $header = fread( $file, 8 );
173 if ( strlen( $header ) < 8 ) {
174 return [ false, 0 ];
175 }
176 $arr = unpack( 'a4chunk/Nlength', $header );
177
178 return [ $arr['chunk'], $arr['length'] ];
179 }
180
185 private function skipChunk( $file, int $chunkLength ) {
186 fseek( $file, $chunkLength, SEEK_CUR );
187
188 if ( ( $chunkLength & 1 ) && !feof( $file ) ) {
189 // padding byte
190 fseek( $file, 1, SEEK_CUR );
191 }
192 }
193
199 private function getMultiPageInfo( $file, int $formLength ) {
200 // For now, we'll just look for the first page in the file
201 // and report its information, hoping others are the same size.
202 $start = ftell( $file );
203 do {
204 [ $chunk, $length ] = $this->readChunk( $file );
205 if ( !$chunk ) {
206 break;
207 }
208
209 if ( $chunk === 'FORM' ) {
210 $subtype = fread( $file, 4 );
211 if ( $subtype === 'DJVU' ) {
212 wfDebug( __METHOD__ . ": found first subpage" );
213
214 return $this->getPageInfo( $file );
215 }
216 $this->skipChunk( $file, $length - 4 );
217 } else {
218 wfDebug( __METHOD__ . ": skipping '$chunk' chunk" );
219 $this->skipChunk( $file, $length );
220 }
221 } while ( $length != 0 && !feof( $file ) && ftell( $file ) - $start < $formLength );
222
223 wfDebug( __METHOD__ . ": multi-page DJVU file contained no pages" );
224
225 return false;
226 }
227
232 private function getPageInfo( $file ) {
233 [ $chunk, $length ] = $this->readChunk( $file );
234 if ( $chunk !== 'INFO' ) {
235 wfDebug( __METHOD__ . ": expected INFO chunk, got '$chunk'" );
236
237 return false;
238 }
239
240 if ( $length < 9 ) {
241 wfDebug( __METHOD__ . ": INFO should be 9 or 10 bytes, found $length" );
242
243 return false;
244 }
245 $data = fread( $file, $length );
246 if ( strlen( $data ) < $length ) {
247 wfDebug( __METHOD__ . ": INFO chunk cut off" );
248
249 return false;
250 }
251
252 $arr = unpack(
253 'nwidth/' .
254 'nheight/' .
255 'Cminor/' .
256 'Cmajor/' .
257 'vresolution/' .
258 'Cgamma', $data );
259
260 # Newer files have rotation info in byte 10, but we don't use it yet.
261
262 return [
263 'width' => $arr['width'],
264 'height' => $arr['height'],
265 'version' => "{$arr['major']}.{$arr['minor']}",
266 'resolution' => $arr['resolution'],
267 'gamma' => $arr['gamma'] / 10.0 ];
268 }
269
274 public function retrieveMetaData() {
275 $config = MediaWikiServices::getInstance()->getMainConfig();
276 $djvuDump = $config->get( MainConfigNames::DjvuDump );
277 $djvuTxt = $config->get( MainConfigNames::DjvuTxt );
278 $djvuUseBoxedCommand = $config->get( MainConfigNames::DjvuUseBoxedCommand );
279 $shell = $config->get( MainConfigNames::ShellboxShell );
280 if ( !$this->isValid() ) {
281 return false;
282 }
283
284 if ( $djvuTxt === null && $djvuDump === null ) {
285 return [];
286 }
287
288 $txt = null;
289 $dump = null;
290
291 if ( $djvuUseBoxedCommand ) {
292 $command = MediaWikiServices::getInstance()->getShellCommandFactory()
293 ->createBoxed( 'djvu' )
294 ->disableNetwork()
295 ->firejailDefaultSeccomp()
296 ->routeName( 'djvu-metadata' )
297 ->params( $shell, 'scripts/retrieveDjvuMetaData.sh' )
298 ->inputFileFromFile(
299 'scripts/retrieveDjvuMetaData.sh',
300 __DIR__ . '/scripts/retrieveDjvuMetaData.sh' )
301 ->inputFileFromFile( 'file.djvu', $this->mFilename )
302 ->memoryLimit( self::DJVUTXT_MEMORY_LIMIT );
303 $env = [];
304 if ( $djvuDump !== null ) {
305 $env['DJVU_DUMP'] = $djvuDump;
306 $command->outputFileToString( 'dump' );
307 }
308 if ( $djvuTxt !== null ) {
309 $env['DJVU_TXT'] = $djvuTxt;
310 $command->outputFileToString( 'txt' );
311 }
312
313 $result = $command
314 ->environment( $env )
315 ->execute();
316 if ( $result->getExitCode() !== 0 ) {
317 wfDebug( 'retrieveDjvuMetaData failed with exit code ' . $result->getExitCode() );
318 return false;
319 }
320 if ( $djvuDump !== null ) {
321 if ( $result->wasReceived( 'dump' ) ) {
322 $dump = $result->getFileContents( 'dump' );
323 } else {
324 wfDebug( __METHOD__ . ": did not receive dump file" );
325 }
326 }
327
328 if ( $djvuTxt !== null ) {
329 if ( $result->wasReceived( 'txt' ) ) {
330 $txt = $result->getFileContents( 'txt' );
331 } else {
332 wfDebug( __METHOD__ . ": did not receive text file" );
333 }
334 }
335 } else { // No boxedcommand
336 if ( $djvuDump !== null ) {
337 # djvudump is faster than djvutoxml (now abandoned) as of version 3.5
338 # https://sourceforge.net/p/djvu/bugs/71/
339 $cmd = Shell::escape( $djvuDump ) . ' ' . Shell::escape( $this->mFilename );
340 $dump = wfShellExec( $cmd );
341 }
342 if ( $djvuTxt !== null ) {
343 $cmd = Shell::escape( $djvuTxt ) . ' --detail=page ' . Shell::escape( $this->mFilename );
344 wfDebug( __METHOD__ . ": $cmd" );
345 $retval = 0;
346 $txt = wfShellExec( $cmd, $retval, [], [ 'memory' => self::DJVUTXT_MEMORY_LIMIT ] );
347 if ( $retval !== 0 ) {
348 $txt = null;
349 }
350 }
351 }
352
353 # Convert dump to array
354 $json = [];
355 if ( $dump !== null ) {
356 $data = $this->convertDumpToJSON( $dump );
357 if ( $data !== false ) {
358 $json = [ 'data' => $data ];
359 }
360 }
361
362 # Text layer
363 if ( $txt !== null ) {
364 # Strip some control characters
365 # Ignore carriage returns
366 $txt = preg_replace( "/\\\\013/", "", $txt );
367 # Replace runs of OCR region separators with a single extra line break
368 $txt = preg_replace( "/(?:\\\\(035|037))+/", "\n", $txt );
369
370 $reg = <<<EOR
371 /\‍(page\s[\d-]*\s[\d-]*\s[\d-]*\s[\d-]*\s*"
372 ((?> # Text to match is composed of atoms of either:
373 \\\\. # - any escaped character
374 | # - any character different from " and \
375 [^"\\\\]+
376 )*?)
377 "\s*\‍)
378 | # Or page can be empty ; in this case, djvutxt dumps ()
379 \‍(\s*()\)/sx
380EOR;
381 $matches = [];
382 preg_match_all( $reg, $txt, $matches );
383 $json['text'] = array_map( [ $this, 'pageTextCallback' ], $matches[1] );
384 } else {
385 $json['text'] = [];
386 }
387
388 return $json;
389 }
390
391 private function pageTextCallback( string $match ): string {
392 # Get rid of invalid UTF-8
393 $val = UtfNormal\Validator::cleanUp( stripcslashes( $match ) );
394 return str_replace( '�', '', $val );
395 }
396
401 private function convertDumpToJSON( $dump ) {
402 if ( strval( $dump ) == '' ) {
403 return false;
404 }
405
406 $dump = str_replace( "\r", '', $dump );
407 $line = strtok( $dump, "\n" );
408 $m = false;
409 $good = false;
410 $result = [];
411 if ( preg_match( '/^( *)FORM:DJVU/', $line, $m ) ) {
412 # Single-page
413 $parsed = $this->parseFormDjvu( $line );
414 if ( $parsed ) {
415 $good = true;
416 } else {
417 return false;
418 }
419 $result['pages'] = [ $parsed ];
420 } elseif ( preg_match( '/^( *)FORM:DJVM/', $line, $m ) ) {
421 # Multi-page
422 $parentLevel = strlen( $m[1] );
423 # Find DIRM
424 $line = strtok( "\n" );
425 $result['pages'] = [];
426 while ( $line !== false ) {
427 $childLevel = strspn( $line, ' ' );
428 if ( $childLevel <= $parentLevel ) {
429 # End of chunk
430 break;
431 }
432
433 if ( preg_match( '/^ *DIRM.*indirect/', $line ) ) {
434 wfDebug( "Indirect multi-page DjVu document, bad for server!" );
435
436 return false;
437 }
438
439 if ( preg_match( '/^ *FORM:DJVU/', $line ) ) {
440 # Found page
441 $parsed = $this->parseFormDjvu( $line );
442 if ( $parsed ) {
443 $good = true;
444 } else {
445 return false;
446 }
447 $result['pages'][] = $parsed;
448 }
449 $line = strtok( "\n" );
450 }
451 }
452 if ( !$good ) {
453 return false;
454 }
455
456 return $result;
457 }
458
460 private function parseFormDjvu( string $line ) {
461 $parentLevel = strspn( $line, ' ' );
462 $line = strtok( "\n" );
463 # Find INFO
464 while ( $line !== false ) {
465 $childLevel = strspn( $line, ' ' );
466 if ( $childLevel <= $parentLevel ) {
467 # End of chunk
468 break;
469 }
470
471 if ( preg_match(
472 '/^ *INFO *\[\d*] *DjVu *(\d+)x(\d+), *\w*, *(\d+) *dpi, *gamma=([0-9.-]+)/',
473 $line,
474 $m
475 ) ) {
476 return [
477 'height' => (int)$m[2],
478 'width' => (int)$m[1],
479 'dpi' => (float)$m[3],
480 'gamma' => (float)$m[4],
481 ];
482 }
483 $line = strtok( "\n" );
484 }
485
486 # Not found
487 return false;
488 }
489}
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfShellExec( $cmd, &$retval=null, $environ=[], $limits=[], $options=[])
Execute a shell command, with time and memory limits mirrored from the PHP configuration if supported...
Support for detecting/validating DjVu image files and getting some basic file metadata (resolution et...
Definition DjVuImage.php:41
retrieveMetaData()
Return an array describing the DjVu image.
dump()
For debugging; dump the IFF chunk structure.
Definition DjVuImage.php:89
isValid()
Check if the given file is indeed a valid DjVu image file.
Definition DjVuImage.php:62
getImageSize()
Return width and height.
Definition DjVuImage.php:72
__construct( $filename)
Definition DjVuImage.php:54
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
Executes shell commands.
Definition Shell.php:46
$header