Code Coverage
 
Lines
Functions and Methods
Classes and Traits
Total
0.00% covered (danger)
0.00%
0 / 280
0.00% covered (danger)
0.00%
0 / 6
CRAP
0.00% covered (danger)
0.00%
0 / 1
Benchmark
0.00% covered (danger)
0.00%
0 / 274
0.00% covered (danger)
0.00%
0 / 6
992
0.00% covered (danger)
0.00%
0 / 1
 __construct
0.00% covered (danger)
0.00%
0 / 36
0.00% covered (danger)
0.00%
0 / 1
2
 executeSetUp
0.00% covered (danger)
0.00%
0 / 20
0.00% covered (danger)
0.00%
0 / 1
20
 executeValidateInput
0.00% covered (danger)
0.00%
0 / 30
0.00% covered (danger)
0.00%
0 / 1
56
 executeSegmenting
0.00% covered (danger)
0.00%
0 / 22
0.00% covered (danger)
0.00%
0 / 1
6
 executeSynthesizing
0.00% covered (danger)
0.00%
0 / 81
0.00% covered (danger)
0.00%
0 / 1
182
 execute
0.00% covered (danger)
0.00%
0 / 85
0.00% covered (danger)
0.00%
0 / 1
20
1<?php
2
3namespace MediaWiki\Wikispeech;
4
5/**
6 * @file
7 * @ingroup Extensions
8 * @license GPL-2.0-or-later
9 */
10
11use EmptyBagOStuff;
12use Maintenance;
13use MediaWiki\Context\RequestContext;
14use MediaWiki\MediaWikiServices;
15use Mediawiki\Title\Title;
16use MediaWiki\Wikispeech\Segment\SegmentList;
17use MediaWiki\Wikispeech\Segment\SegmentPageFactory;
18use RuntimeException;
19use Wikimedia\ObjectCache\WANObjectCache;
20
21/** @var string MediaWiki installation path */
22$IP = getenv( 'MW_INSTALL_PATH' );
23if ( $IP === false ) {
24    $IP = __DIR__ . '/../../..';
25}
26require_once "$IP/maintenance/Maintenance.php";
27
28/**
29 * Maintenance script to evaluate interesting resource use metrics
30 * related to executing Wikispeech and Speechoid on a page.
31 *
32 * php extensions/Wikispeech/maintenance/benchmark.php -p Barack_Obama
33 *
34 * @since 0.1.8
35 */
36class Benchmark extends Maintenance {
37
38    /** @var VoiceHandler */
39    private $voiceHandler;
40
41    /** @var SegmentPageFactory */
42    private $segmentPageFactory;
43
44    /** @var SpeechoidConnector */
45    private $speechoidConnector;
46
47    /** @var bool Whether or not ctrl-c has been pressed. */
48    private $caughtSigInt;
49
50    /** @var SegmentList */
51    private $segments;
52
53    /** @var int */
54    private $synthesizeResponseTimeoutSeconds;
55
56    /** @var float|int */
57    private $millisecondsSpentSegmenting;
58
59    /** @var int */
60    private $numberOfSuccessfullySynthesizedSegments;
61
62    /** @var int|float */
63    private $totalMillisecondsSpentSynthesizing;
64
65    /** @var int */
66    private $totalMillisecondsSynthesizedVoice;
67
68    /** @var int */
69    private $totalNumberOfTokensSynthesizedVoice;
70
71    /** @var int */
72    private $totalBytesSynthesizedVoice;
73
74    /** @var int */
75    private $totalNumberOfTokenCharactersSynthesizedVoice;
76
77    /** @var string */
78    private $language;
79
80    /** @var string */
81    private $voice;
82
83    /** @var Title */
84    private $title;
85
86    /**
87     * Benchmark constructor.
88     *
89     * @since 0.1.8
90     */
91    public function __construct() {
92        parent::__construct();
93        $this->requireExtension( 'Wikispeech' );
94        $this->addDescription( 'Benchmark use of resources.' );
95        $this->addOption(
96            'language',
97            'Synthesized language. If not set, page language is selected.',
98            false,
99            true,
100            'l'
101        );
102        $this->addOption(
103            'voice',
104            'Synthesized voice. If not set, default voice for language is selected.',
105            false,
106            true,
107            'v'
108        );
109        $this->addOption(
110            'page',
111            'Title of page to be segmented and synthesized.',
112            true,
113            true,
114            'p'
115        );
116        $this->addOption(
117            'timeout',
118            'Maximum number of seconds to await Speechoid synthesize HTTP response. Defaults to 240.',
119            false,
120            true,
121            't'
122        );
123
124        $this->caughtSigInt = false;
125        declare( ticks = 1 );
126        pcntl_async_signals( true );
127        pcntl_signal( SIGINT, function () {
128            // Clean ctrl-c
129            $this->caughtSigInt = true;
130        } );
131    }
132
133    private function executeSetUp(): void {
134        // Non PHP core classes aren't available prior to this point,
135        // i.e. we can't initialize the fields in the constructor,
136        // and we have to be lenient for mocked instances set by tests.
137
138        $services = MediaWikiServices::getInstance();
139        $config = $services->getConfigFactory()->makeConfig( 'wikispeech' );
140        $requestFactory = $services->getHttpRequestFactory();
141
142        $emptyWanCache = new WANObjectCache( [ 'cache' => new EmptyBagOStuff() ] );
143
144        if ( !$this->speechoidConnector ) {
145            $this->speechoidConnector = new SpeechoidConnector( $config, $requestFactory );
146        }
147        if ( !$this->voiceHandler ) {
148            $this->voiceHandler = WikispeechServices::getVoiceHandler();
149        }
150        if ( !$this->segmentPageFactory ) {
151            $this->segmentPageFactory = new SegmentPageFactory(
152                $emptyWanCache,
153                $config,
154                $services->getRevisionStore(),
155                $services->getHttpRequestFactory()
156            );
157            $this->segmentPageFactory
158                ->setUseSegmentsCache( false )
159                ->setUseRevisionPropertiesCache( false )
160                ->setContextSource( new RequestContext() )
161                ->setRevisionStore( $services->getRevisionStore() );
162        }
163    }
164
165    private function executeValidateInput(): bool {
166        $this->language = '';
167        $this->voice = '';
168        $this->title = Title::newFromText( $this->getOption( 'page' ) );
169        if ( !$this->title->isKnown() ) {
170            $this->output( "Error: Page is not known.\n" );
171            return false;
172        }
173        if ( $this->title->isSpecialPage() ) {
174            $this->output( "Error: Page is a SpecialPage.\n" );
175            return false;
176        }
177
178        if ( !$this->getOption( 'language', false ) ) {
179            $language = $this->title->getPageLanguage();
180            if ( !$language ) {
181                $this->output( "Error: Unable to read language for page. Use parameter language.\n" );
182                return false;
183            }
184            $this->language = $language->getCode();
185            $this->output( "Language $this->language set from page default.\n" );
186        } else {
187            $this->language = $this->getOption( 'language' );
188            $this->output( "Language $this->language set from option.\n" );
189            // todo validate language
190        }
191
192        if ( !$this->getOption( 'voice', false ) ) {
193            $this->voice = $this->voiceHandler->getDefaultVoice( $this->language );
194            if ( !$this->voice ) {
195                // This will never occur unless underlying default voice logic change.
196                // I.e. if the default voice cannot be found
197                // then your language must not be defined (in Speechoid or locally)
198                $this->output( "Error: No default voice for language $this->language. Use parameter voice.\n" );
199                return false;
200            }
201            $this->output( "Voice $this->voice set from default for language $this->language.\n" );
202        } else {
203            $this->voice = $this->getOption( 'voice' );
204            $this->output( "Voice $this->voice set from option.\n" );
205            // todo validate voice of language
206        }
207
208        $this->synthesizeResponseTimeoutSeconds = intval(
209            $this->getOption( 'timeout', 240 )
210        );
211
212        return true;
213    }
214
215    private function executeSegmenting(): void {
216        // @todo consider adding revision as script parameter.
217        // Setting null will requests the most recent for the title.
218        $revisionId = null;
219
220        $this->output( 'Benchmarking page ' .
221            "{$this->title->getText()} using language " .
222            "$this->language and voice " .
223            "$this->voice.\n"
224        );
225
226        // We don't want to count time spent rendering to segmenting time,
227        // so we call the segmenter twice. Segmenting cache is turned off.
228        $this->output( "Allowing for MediaWiki to render page...\n" );
229        $this->segmentPageFactory->segmentPage(
230            $this->title,
231            $revisionId
232        );
233
234        $this->output( "Segmenting...\n" );
235        $startSegmenting = microtime( true ) * 1000;
236        $segments = $this->segmentPageFactory->segmentPage(
237            $this->title,
238            $revisionId
239        )->getSegments();
240        if ( $segments === null ) {
241            throw new RuntimeException( 'Segments is null!' );
242        }
243        $this->segments = $segments;
244        $endSegmenting = microtime( true ) * 1000;
245        $this->millisecondsSpentSegmenting = $endSegmenting - $startSegmenting;
246    }
247
248    private function executeSynthesizing(): void {
249        $this->numberOfSuccessfullySynthesizedSegments = 0;
250
251        $this->totalBytesSynthesizedVoice = 0;
252        $this->totalNumberOfTokenCharactersSynthesizedVoice = 0;
253        $this->totalNumberOfTokensSynthesizedVoice = 0;
254        $this->totalMillisecondsSynthesizedVoice = 0;
255        $this->output( 'Synthesizing ' . count( $this->segments->getSegments() ) . " segments... \n" );
256        $this->output( "Press ^C to abort and calculate on evaluated state.\n" );
257        $this->totalMillisecondsSpentSynthesizing = 0;
258
259        $failures = '';
260
261        $progressCounterLength = 40;
262        $segmentCounter = 0;
263        $progressCounter = 0;
264        foreach ( $this->segments->getSegments() as $segment ) {
265            if ( $this->caughtSigInt ) {
266                break;
267            }
268            $segmentCounter++;
269
270            $segmentText = '';
271            foreach ( $segment->getContent() as $content ) {
272                $segmentText .= $content->getString();
273            }
274
275            $attempt = 0;
276            $maximumAttempts = 3;
277            $retriesLeft = $maximumAttempts;
278            while ( true ) {
279                $attempt++;
280                $startSynthesizing = microtime( true ) * 1000;
281                try {
282                    $speechoidResponse = $this->speechoidConnector->synthesizeText(
283                        $this->language, $this->voice, $segmentText, $this->synthesizeResponseTimeoutSeconds
284                    );
285                    $endSynthesizing = microtime( true ) * 1000;
286                    $millisecondsSpentSynthesizingSegment = $endSynthesizing - $startSynthesizing;
287                    $this->totalMillisecondsSpentSynthesizing += $millisecondsSpentSynthesizingSegment;
288
289                    $bytesSynthesizedVoiceInSegment = mb_strlen( $speechoidResponse['audio_data'] );
290                    $this->totalBytesSynthesizedVoice += $bytesSynthesizedVoiceInSegment;
291
292                    $numberOfTokensInSegment = count( $speechoidResponse[ 'tokens' ] );
293                    $this->totalNumberOfTokensSynthesizedVoice += $numberOfTokensInSegment;
294
295                    $millisecondsSynthesizedVoiceInSegment =
296                        $speechoidResponse['tokens'][ $numberOfTokensInSegment - 1 ]['endtime'];
297                    $this->totalMillisecondsSynthesizedVoice += $millisecondsSynthesizedVoiceInSegment;
298
299                    $charactersInSegmentTokens = 0;
300                    foreach ( $speechoidResponse['tokens'] as $token ) {
301                        $charactersInSegmentTokens += mb_strlen( $token['orth'] );
302                    }
303                    $this->totalNumberOfTokenCharactersSynthesizedVoice += $charactersInSegmentTokens;
304
305                    if ( $attempt > 1 ) {
306                        $this->output( strval( $attempt ) );
307                    } else {
308                        $this->output( '.' );
309                    }
310                    $this->numberOfSuccessfullySynthesizedSegments++;
311                } catch ( SpeechoidConnectorException $speechoidConnectorException ) {
312                    $millisecondsSpentBeforeException = ( microtime( true ) * 1000 ) - $startSynthesizing;
313                    $failures .= "\nException $millisecondsSpentBeforeException milliseconds after request.\n";
314                    $failures .= $speechoidConnectorException->getMessage() . "\n";
315                    $retriesLeft--;
316                    if ( $retriesLeft == 0 ) {
317                        $failures .= "Giving up after attempt #$attempt. Segment ignored.\n";
318                        $failures .= $segmentText;
319                        $failures .= "\n";
320                        $this->output( 'E' );
321                    } else {
322                        continue;
323                    }
324                }
325                $progressCounter++;
326                if ( $progressCounter === $progressCounterLength ) {
327                    $progressCounter = 0;
328
329                    $eta = ', ETA ~';
330                    $meanMillisecondsSpentSynthesizingPerSegment =
331                        $this->totalMillisecondsSpentSynthesizing / $this->numberOfSuccessfullySynthesizedSegments;
332                    $millisecondsEta = intval( count( $this->segments->getSegments() ) - $segmentCounter )
333                        * $meanMillisecondsSpentSynthesizingPerSegment;
334                    if ( $millisecondsEta < 1000 ) {
335                        $eta .= $millisecondsEta . ' ms';
336                    } elseif ( $millisecondsEta < 1000 * 60 ) {
337                        $eta .= intdiv( $millisecondsEta, 1000 ) . ' seconds';
338                    } else {
339                        $eta .= intdiv( $millisecondsEta, 1000 * 60 ) . ' minutes';
340                    }
341                    $eta .= ' (~' .    intdiv( $meanMillisecondsSpentSynthesizingPerSegment, 1000 ) . 's/seg)';
342                    $this->output(
343                        ' ' .
344                        $segmentCounter . ' / ' . count( $this->segments->getSegments() ) .
345                        $eta . "\n"
346                    );
347                }
348                break;
349            }
350        }
351
352        if ( $failures ) {
353            $this->output( "\n" );
354            $this->output( $failures );
355            $this->output( "\n" );
356        }
357    }
358
359    /**
360     * @since 0.1.8
361     * @return bool success
362     */
363    public function execute() {
364        $this->executeSetUp();
365        if ( !$this->executeValidateInput() ) {
366            return false;
367        }
368        $this->executeSegmenting();
369        $this->executeSynthesizing();
370
371        $this->output( "\n\n" );
372        $this->output( "Benchmark results\n" );
373        $this->output( "-----------------\n" );
374        $this->output( "\n" );
375
376        $this->output( 'Number of segments: ' .
377            count( $this->segments->getSegments() ) . "\n" );
378        $this->output( "Milliseconds spent segmenting: $this->millisecondsSpentSegmenting\n" );
379
380        $meanMillisecondsSpentSegmentingPerSegment =
381            $this->millisecondsSpentSegmenting / count( $this->segments->getSegments() );
382
383        $this->output( 'Mean milliseconds spent segmenting per segment: ' .
384            "$meanMillisecondsSpentSegmentingPerSegment\n" );
385
386        if ( $this->numberOfSuccessfullySynthesizedSegments === 0 ) {
387            $this->output( "Nothing synthesized, no further metrics available.\n" );
388            exit( 1 );
389        }
390
391        $this->totalMillisecondsSpentSynthesizing = intval( $this->totalMillisecondsSpentSynthesizing );
392        $this->totalMillisecondsSynthesizedVoice = intval( $this->totalMillisecondsSynthesizedVoice );
393
394        $meanMillisecondsSynthesizingPerToken =
395            $this->totalMillisecondsSynthesizedVoice / $this->totalNumberOfTokensSynthesizedVoice;
396        $meanMillisecondsSynthesizingPerCharacter =
397            $this->totalMillisecondsSynthesizedVoice / $this->totalNumberOfTokenCharactersSynthesizedVoice;
398        $meanBytesSynthesizedVoicePerToken =
399            $this->totalBytesSynthesizedVoice / $this->totalNumberOfTokensSynthesizedVoice;
400        $meanBytesSynthesizedVoicePerCharacter =
401            $this->totalBytesSynthesizedVoice / $this->totalNumberOfTokenCharactersSynthesizedVoice;
402
403        $meanTokensPerSegment =
404            $this->totalNumberOfTokensSynthesizedVoice / $this->numberOfSuccessfullySynthesizedSegments;
405        $meanTokenCharactersPerSegment =
406            $this->totalNumberOfTokenCharactersSynthesizedVoice /
407            $this->numberOfSuccessfullySynthesizedSegments;
408
409        $meanMillisecondsSpentSegmentingPerToken =
410            ( $meanMillisecondsSpentSegmentingPerSegment * $this->numberOfSuccessfullySynthesizedSegments ) /
411            $this->totalNumberOfTokensSynthesizedVoice;
412        $meanMillisecondsSpentSegmentingPerTokenCharacter =
413            ( $meanMillisecondsSpentSegmentingPerSegment * $this->numberOfSuccessfullySynthesizedSegments ) /
414            $this->totalNumberOfTokenCharactersSynthesizedVoice;
415
416        $this->output( 'Mean milliseconds spent segmenting per token synthesized: ' .
417            "$meanMillisecondsSpentSegmentingPerToken\n" );
418        $this->output( 'Mean milliseconds spent segmenting per token character synthesized: ' .
419            "$meanMillisecondsSpentSegmentingPerTokenCharacter\n" );
420
421        if ( $this->numberOfSuccessfullySynthesizedSegments != count( $this->segments->getSegments() ) ) {
422            $this->output( 'Warning! Not all segments synthesized, ' .
423                "mean segmenting per token values might be slightly off.\n" );
424        }
425
426        $this->output( "\n" );
427
428        $this->output( 'Number of synthesized segments: ' .
429            "$this->numberOfSuccessfullySynthesizedSegments\n" );
430        $this->output( "Number of synthesized tokens: $this->totalNumberOfTokensSynthesizedVoice\n" );
431        $this->output( 'Number of synthesized token characters: ' .
432            "$this->totalNumberOfTokenCharactersSynthesizedVoice\n" );
433
434        $this->output( "\n" );
435
436        $this->output( "Mean number of tokens per synthesized segment: $meanTokensPerSegment\n" );
437        $this->output( 'Mean number of token characters per synthesized segment: ' .
438            "$meanTokenCharactersPerSegment\n" );
439
440        $this->output( "\n" );
441
442        $this->output( 'Mean milliseconds synthesizing per token: ' .
443            "$meanMillisecondsSynthesizingPerToken\n" );
444        $this->output( 'Mean milliseconds synthesizing per token character: ' .
445            "$meanMillisecondsSynthesizingPerCharacter\n" );
446
447        $this->output( 'Mean bytes synthesized voice per token: ' .
448            intval( $meanBytesSynthesizedVoicePerToken ) . "\n" );
449        $this->output( 'Mean bytes synthesized voice per token character: ' .
450            intval( $meanBytesSynthesizedVoicePerCharacter ) . "\n" );
451
452        $this->output( "\n" );
453
454        $this->output( "Milliseconds of synthesized voice: $this->totalMillisecondsSynthesizedVoice\n" );
455        $this->output( 'Seconds of synthesized voice: ' .
456            intdiv( $this->totalMillisecondsSynthesizedVoice, 1000 ) . "\n" );
457        $this->output( 'Minutes of synthesized voice: ' .
458            intdiv( $this->totalMillisecondsSynthesizedVoice, 1000 * 60 ) . "\n" );
459
460        $this->output( "\n" );
461
462        $this->output( "Milliseconds spent synthesizing: $this->totalMillisecondsSpentSynthesizing\n" );
463        $this->output( 'Seconds spent synthesizing: ' .
464            intdiv( $this->totalMillisecondsSpentSynthesizing, 1000 ) . "\n" );
465        $this->output( 'Minutes spent synthesizing: ' .
466            intdiv( $this->totalMillisecondsSpentSynthesizing, 1000 * 60 ) . "\n" );
467
468        $this->output( "\n" );
469
470        $this->output( "Synthesized voice bytes: $this->totalBytesSynthesizedVoice\n" );
471        $this->output( 'Synthesized voice kilobytes: ' .
472            intdiv( $this->totalBytesSynthesizedVoice, 1024 ) . "\n" );
473        $this->output( 'Synthesized voice megabytes: ' .
474            intdiv( $this->totalBytesSynthesizedVoice, 1024 * 1024 ) . "\n" );
475
476        return true;
477    }
478
479}
480
481/** @var string This class, required to start via Maintenance. */
482$maintClass = Benchmark::class;
483
484require_once RUN_MAINTENANCE_IF_MAIN;