46 private const DJVUTXT_MEMORY_LIMIT = 300_000_000;
55 $this->mFilename = $filename;
63 $info = $this->getInfo();
65 return $info !==
false;
73 $data = $this->getInfo();
75 if ( $data !==
false ) {
77 'width' => $data[
'width'],
78 'height' => $data[
'height']
90 $file = fopen( $this->mFilename,
'rb' );
92 $arr = unpack(
'a4magic/a4chunk/NchunkLength',
$header );
93 $chunk = $arr[
'chunk'];
94 $chunkLength = $arr[
'chunkLength'];
95 echo
"$chunk $chunkLength\n";
96 $this->dumpForm( $file, $chunkLength, 1 );
105 private function dumpForm( $file,
int $length,
int $indent ) {
106 $start = ftell( $file );
107 $secondary = fread( $file, 4 );
108 echo str_repeat(
' ', $indent * 4 ) .
"($secondary)\n";
109 while ( ftell( $file ) - $start < $length ) {
110 $chunkHeader = fread( $file, 8 );
111 if ( $chunkHeader ==
'' ) {
114 $arr = unpack(
'a4chunk/NchunkLength', $chunkHeader );
115 $chunk = $arr[
'chunk'];
116 $chunkLength = $arr[
'chunkLength'];
117 echo str_repeat(
' ', $indent * 4 ) .
"$chunk $chunkLength\n";
119 if ( $chunk ===
'FORM' ) {
120 $this->dumpForm( $file, $chunkLength, $indent + 1 );
122 fseek( $file, $chunkLength, SEEK_CUR );
123 if ( $chunkLength & 1 ) {
125 fseek( $file, 1, SEEK_CUR );
132 private function getInfo() {
133 AtEase::suppressWarnings();
134 $file = fopen( $this->mFilename,
'rb' );
135 AtEase::restoreWarnings();
136 if ( $file ===
false ) {
137 wfDebug( __METHOD__ .
": missing or failed file read" );
145 if ( strlen(
$header ) < 16 ) {
146 wfDebug( __METHOD__ .
": too short file header" );
148 $arr = unpack(
'a4magic/a4form/NformLength/a4subtype',
$header );
150 $subtype = $arr[
'subtype'];
151 if ( $arr[
'magic'] !==
'AT&T' ) {
152 wfDebug( __METHOD__ .
": not a DjVu file" );
153 } elseif ( $subtype ===
'DJVU' ) {
155 $info = $this->getPageInfo( $file );
156 } elseif ( $subtype ===
'DJVM' ) {
158 $info = $this->getMultiPageInfo( $file, $arr[
'formLength'] );
160 wfDebug( __METHOD__ .
": unrecognized DJVU file type '{$arr['subtype']}'" );
171 private function readChunk( $file ): array {
176 $arr = unpack(
'a4chunk/Nlength',
$header );
178 return [ $arr[
'chunk'], $arr[
'length'] ];
185 private function skipChunk( $file,
int $chunkLength ) {
186 fseek( $file, $chunkLength, SEEK_CUR );
188 if ( ( $chunkLength & 1 ) && !feof( $file ) ) {
190 fseek( $file, 1, SEEK_CUR );
199 private function getMultiPageInfo( $file,
int $formLength ) {
202 $start = ftell( $file );
204 [ $chunk, $length ] = $this->readChunk( $file );
209 if ( $chunk ===
'FORM' ) {
210 $subtype = fread( $file, 4 );
211 if ( $subtype ===
'DJVU' ) {
212 wfDebug( __METHOD__ .
": found first subpage" );
214 return $this->getPageInfo( $file );
216 $this->skipChunk( $file, $length - 4 );
218 wfDebug( __METHOD__ .
": skipping '$chunk' chunk" );
219 $this->skipChunk( $file, $length );
221 }
while ( $length != 0 && !feof( $file ) && ftell( $file ) - $start < $formLength );
223 wfDebug( __METHOD__ .
": multi-page DJVU file contained no pages" );
232 private function getPageInfo( $file ) {
233 [ $chunk, $length ] = $this->readChunk( $file );
234 if ( $chunk !==
'INFO' ) {
235 wfDebug( __METHOD__ .
": expected INFO chunk, got '$chunk'" );
241 wfDebug( __METHOD__ .
": INFO should be 9 or 10 bytes, found $length" );
245 $data = fread( $file, $length );
246 if ( strlen( $data ) < $length ) {
247 wfDebug( __METHOD__ .
": INFO chunk cut off" );
260 # Newer files have rotation info in byte 10, but we don't use it yet.
263 'width' => $arr[
'width'],
264 'height' => $arr[
'height'],
265 'version' =>
"{$arr['major']}.{$arr['minor']}",
266 'resolution' => $arr[
'resolution'],
267 'gamma' => $arr[
'gamma'] / 10.0 ];
275 $config = MediaWikiServices::getInstance()->getMainConfig();
276 $djvuDump = $config->get( MainConfigNames::DjvuDump );
277 $djvuTxt = $config->get( MainConfigNames::DjvuTxt );
278 $djvuUseBoxedCommand = $config->get( MainConfigNames::DjvuUseBoxedCommand );
279 $shell = $config->get( MainConfigNames::ShellboxShell );
280 if ( !$this->isValid() ) {
284 if ( $djvuTxt ===
null && $djvuDump ===
null ) {
291 if ( $djvuUseBoxedCommand ) {
292 $command = MediaWikiServices::getInstance()->getShellCommandFactory()
293 ->createBoxed(
'djvu' )
295 ->firejailDefaultSeccomp()
296 ->routeName(
'djvu-metadata' )
297 ->params( $shell,
'scripts/retrieveDjvuMetaData.sh' )
299 'scripts/retrieveDjvuMetaData.sh',
300 __DIR__ .
'/scripts/retrieveDjvuMetaData.sh' )
301 ->inputFileFromFile(
'file.djvu', $this->mFilename )
302 ->memoryLimit( self::DJVUTXT_MEMORY_LIMIT );
304 if ( $djvuDump !==
null ) {
305 $env[
'DJVU_DUMP'] = $djvuDump;
306 $command->outputFileToString(
'dump' );
308 if ( $djvuTxt !==
null ) {
309 $env[
'DJVU_TXT'] = $djvuTxt;
310 $command->outputFileToString(
'txt' );
314 ->environment( $env )
316 if ( $result->getExitCode() !== 0 ) {
317 wfDebug(
'retrieveDjvuMetaData failed with exit code ' . $result->getExitCode() );
320 if ( $djvuDump !==
null ) {
321 if ( $result->wasReceived(
'dump' ) ) {
322 $dump = $result->getFileContents(
'dump' );
324 wfDebug( __METHOD__ .
": did not receive dump file" );
328 if ( $djvuTxt !==
null ) {
329 if ( $result->wasReceived(
'txt' ) ) {
330 $txt = $result->getFileContents(
'txt' );
332 wfDebug( __METHOD__ .
": did not receive text file" );
336 if ( $djvuDump !==
null ) {
337 # djvudump is faster than djvutoxml (now abandoned) as of version 3.5
339 $cmd = Shell::escape( $djvuDump ) .
' ' . Shell::escape( $this->mFilename );
342 if ( $djvuTxt !==
null ) {
343 $cmd = Shell::escape( $djvuTxt ) .
' --detail=page ' . Shell::escape( $this->mFilename );
344 wfDebug( __METHOD__ .
": $cmd" );
346 $txt =
wfShellExec( $cmd, $retval, [], [
'memory' => self::DJVUTXT_MEMORY_LIMIT ] );
347 if ( $retval !== 0 ) {
353 # Convert dump to array
355 if ( $dump !==
null ) {
356 $data = $this->convertDumpToJSON( $dump );
357 if ( $data !==
false ) {
358 $json = [
'data' => $data ];
363 if ( $txt !==
null ) {
364 # Strip some control characters
365 # Ignore carriage returns
366 $txt = preg_replace(
"/\\\\013/",
"", $txt );
367 # Replace runs of OCR region separators with a single extra line break
368 $txt = preg_replace(
"/(?:\\\\(035|037))+/",
"\n", $txt );
371 /\(page\s[\d-]*\s[\d-]*\s[\d-]*\s[\d-]*\s*
"
372 ((?> # Text to match is composed of atoms of either:
373 \\\\. # - any escaped character
374 | # - any character different from " and \
378 | # Or page can be empty ; in
this case, djvutxt dumps ()
382 preg_match_all( $reg, $txt,
$matches );
383 $json[
'text'] = array_map( [ $this,
'pageTextCallback' ],
$matches[1] );
391 private function pageTextCallback(
string $match ): string {
392 # Get rid of invalid UTF-8
393 $val = UtfNormal\Validator::cleanUp( stripcslashes( $match ) );
394 return str_replace(
'�',
'', $val );
401 private function convertDumpToJSON( $dump ) {
402 if ( strval( $dump ) ==
'' ) {
406 $dump = str_replace(
"\r",
'', $dump );
407 $line = strtok( $dump,
"\n" );
411 if ( preg_match(
'/^( *)FORM:DJVU/', $line, $m ) ) {
413 $parsed = $this->parseFormDjvu( $line );
419 $result[
'pages'] = [ $parsed ];
420 } elseif ( preg_match(
'/^( *)FORM:DJVM/', $line, $m ) ) {
422 $parentLevel = strlen( $m[1] );
424 $line = strtok(
"\n" );
425 $result[
'pages'] = [];
426 while ( $line !==
false ) {
427 $childLevel = strspn( $line,
' ' );
428 if ( $childLevel <= $parentLevel ) {
433 if ( preg_match(
'/^ *DIRM.*indirect/', $line ) ) {
434 wfDebug(
"Indirect multi-page DjVu document, bad for server!" );
439 if ( preg_match(
'/^ *FORM:DJVU/', $line ) ) {
441 $parsed = $this->parseFormDjvu( $line );
447 $result[
'pages'][] = $parsed;
449 $line = strtok(
"\n" );
460 private function parseFormDjvu(
string $line ) {
461 $parentLevel = strspn( $line,
' ' );
462 $line = strtok(
"\n" );
464 while ( $line !==
false ) {
465 $childLevel = strspn( $line,
' ' );
466 if ( $childLevel <= $parentLevel ) {
472 '/^ *INFO *\[\d*] *DjVu *(\d+)x(\d+), *\w*, *(\d+) *dpi, *gamma=([0-9.-]+)/',
477 'height' => (int)$m[2],
478 'width' => (
int)$m[1],
479 'dpi' => (float)$m[3],
480 'gamma' => (
float)$m[4],
483 $line = strtok(
"\n" );
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfShellExec( $cmd, &$retval=null, $environ=[], $limits=[], $options=[])
Execute a shell command, with time and memory limits mirrored from the PHP configuration if supported...