Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 74 |
|
0.00% |
0 / 4 |
CRAP | |
0.00% |
0 / 1 |
BackfillPageTriageQueue | |
0.00% |
0 / 68 |
|
0.00% |
0 / 4 |
90 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
2 | |||
execute | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
2 | |||
backfillScores | |
0.00% |
0 / 50 |
|
0.00% |
0 / 1 |
12 | |||
retry | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
20 |
1 | <?php |
2 | |
3 | namespace ORES\Maintenance; |
4 | |
5 | use BatchRowIterator; |
6 | use Exception; |
7 | use Maintenance; |
8 | use ORES\Services\ORESServices; |
9 | use ORES\Services\ScoreFetcher; |
10 | |
11 | $IP = getenv( 'MW_INSTALL_PATH' ); |
12 | if ( $IP === false ) { |
13 | $IP = __DIR__ . '/../../..'; |
14 | } |
15 | |
16 | require_once "$IP/maintenance/Maintenance.php"; |
17 | |
18 | class BackfillPageTriageQueue extends Maintenance { |
19 | |
20 | private const ORES_RECOMMENDED_BATCH_SIZE = 50; |
21 | |
22 | public function __construct() { |
23 | parent::__construct(); |
24 | $this->addDescription( 'Backfills the missing scores for the articles in the PageTriage queue' ); |
25 | $this->addOption( 'dry-run', 'Do not fetch scores, only print revisions.' ); |
26 | $this->setBatchSize( self::ORES_RECOMMENDED_BATCH_SIZE ); |
27 | $this->requireExtension( 'ORES' ); |
28 | $this->requireExtension( 'PageTriage' ); |
29 | } |
30 | |
31 | public function execute() { |
32 | $this->backfillScores( 'draftquality' ); |
33 | $this->backfillScores( 'articlequality' ); |
34 | |
35 | $this->output( "\nAll done\n" ); |
36 | } |
37 | |
38 | private function backfillScores( $modelName ) { |
39 | $dbr = $this->getDB( DB_REPLICA ); |
40 | $modelId = ORESServices::getModelLookup()->getModelId( $modelName ); |
41 | $this->output( "\nStarting model $modelName (id: $modelId)\n" ); |
42 | |
43 | $iterator = new BatchRowIterator( |
44 | $dbr, |
45 | [ 'revision', 'page', 'pagetriage_page', 'ores_classification' ], |
46 | 'rev_id', |
47 | $this->getBatchSize() |
48 | ); |
49 | $iterator->setFetchColumns( [ 'rev_id', 'oresc_probability' ] ); |
50 | $iterator->addJoinConditions( [ |
51 | 'page' => [ 'INNER JOIN', 'page_latest = rev_id' ], |
52 | 'pagetriage_page' => [ 'INNER JOIN', 'page_id = ptrp_page_id' ], |
53 | 'ores_classification' => [ 'LEFT JOIN', [ |
54 | 'rev_id = oresc_rev', |
55 | 'oresc_model' => $modelId, |
56 | ] ], |
57 | ] ); |
58 | $iterator->addConditions( [ |
59 | 'page_is_redirect' => 0, |
60 | 'oresc_probability' => null, |
61 | ] ); |
62 | $iterator->setCaller( __METHOD__ ); |
63 | |
64 | foreach ( $iterator as $rows ) { |
65 | $revIds = array_map( static function ( $row ) { |
66 | return $row->rev_id; |
67 | }, $rows ); |
68 | |
69 | if ( $this->hasOption( 'dry-run' ) ) { |
70 | $this->output( "Revs: " . implode( ', ', $revIds ) . "\n" ); |
71 | continue; |
72 | } |
73 | |
74 | $scores = $this->retry( static function () use ( $revIds, $modelName ) { |
75 | return ScoreFetcher::instance()->getScores( |
76 | $revIds, |
77 | $modelName, |
78 | true |
79 | ); |
80 | }, 5, 3 ); |
81 | |
82 | $errors = 0; |
83 | ORESServices::getScoreStorage()->storeScores( |
84 | $scores, |
85 | function ( $msg, $revision ) use ( &$errors ) { |
86 | $this->output( "WARNING: ScoreFetcher errored for $revision: $msg\n" ); |
87 | $errors++; |
88 | } |
89 | ); |
90 | |
91 | $count = count( $revIds ); |
92 | $first = reset( $revIds ); |
93 | $last = end( $revIds ); |
94 | $this->output( "Processed $count revisions with $errors errors. From $first to $last.\n" ); |
95 | } |
96 | |
97 | $this->output( "Finished model $modelName\n" ); |
98 | } |
99 | |
100 | private function retry( $fn, $tries, $wait ) { |
101 | $tried = 0; |
102 | while ( true ) { |
103 | try { |
104 | return $fn(); |
105 | } catch ( Exception $ex ) { |
106 | $tried++; |
107 | if ( $tried > $tries ) { |
108 | throw $ex; |
109 | } else { |
110 | $this->error( $ex->getMessage() ); |
111 | sleep( $wait ); |
112 | } |
113 | } |
114 | } |
115 | } |
116 | |
117 | } |
118 | |
119 | $maintClass = BackfillPageTriageQueue::class; |
120 | |
121 | require_once RUN_MAINTENANCE_IF_MAIN; |