Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
91.72% |
144 / 157 |
|
75.00% |
12 / 16 |
CRAP | |
0.00% |
0 / 1 |
| DataCollector | |
91.72% |
144 / 157 |
|
75.00% |
12 / 16 |
72.78 | |
0.00% |
0 / 1 |
| setLanguage | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| setMultiLang | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| setTemplateParser | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| setLicenseParser | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| collect | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
1 | |||
| verifyAttributionMetadata | |
96.43% |
27 / 28 |
|
0.00% |
0 / 1 |
18 | |||
| getCategoryMetadata | |
100.00% |
12 / 12 |
|
100.00% |
1 / 1 |
1 | |||
| getTemplateMetadata | |
100.00% |
31 / 31 |
|
100.00% |
1 / 1 |
9 | |||
| getDescriptionText | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
3 | |||
| getCategories | |
63.16% |
12 / 19 |
|
0.00% |
0 / 1 |
11.20 | |||
| getLicensesAndRemoveFromCategories | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
3 | |||
| getAssessmentsAndRemoveFromCategories | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
4 | |||
| selectFirst | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
2 | |||
| selectInformationTemplate | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
5 | |||
| selectLicense | |
76.47% |
13 / 17 |
|
0.00% |
0 / 1 |
6.47 | |||
| normalizeMetadataTimestamps | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
6 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CommonsMetadata; |
| 4 | |
| 5 | use File; |
| 6 | use ForeignAPIFile; |
| 7 | use InvalidArgumentException; |
| 8 | use Language; |
| 9 | use LocalFile; |
| 10 | use MediaWiki\MediaWikiServices; |
| 11 | use ParserOutput; |
| 12 | use WikiFilePage; |
| 13 | |
| 14 | /** |
| 15 | * Class to handle metadata collection and formatting, and manage more specific data extraction |
| 16 | * classes. |
| 17 | */ |
| 18 | class DataCollector { |
| 19 | |
| 20 | /** |
| 21 | * Mapping of category names to assesment levels. Array keys are regexps which will be |
| 22 | * matched case-insensitively against category names; the first match is returned. |
| 23 | * @var array |
| 24 | */ |
| 25 | protected static $assessmentCategories = [ |
| 26 | 'poty' => '/^pictures of the year \(.*\)/', |
| 27 | 'potd' => '/^pictures of the day \(.*\)/', |
| 28 | 'featured' => '/^featured (pictures|sounds) on wikimedia commons/', |
| 29 | 'quality' => '/^quality images/', |
| 30 | 'valued' => '/^valued images/', |
| 31 | ]; |
| 32 | |
| 33 | /** |
| 34 | * Language in which data should be collected. Can be null, which means collect all languages. |
| 35 | * @var Language |
| 36 | */ |
| 37 | protected $language; |
| 38 | |
| 39 | /** |
| 40 | * If true, ignore $language and collect metadata in all languages. |
| 41 | * @var bool |
| 42 | */ |
| 43 | protected $multiLang; |
| 44 | |
| 45 | /** @var TemplateParser */ |
| 46 | protected $templateParser; |
| 47 | |
| 48 | /** @var LicenseParser */ |
| 49 | protected $licenseParser; |
| 50 | |
| 51 | /** |
| 52 | * @param Language $language |
| 53 | */ |
| 54 | public function setLanguage( $language ) { |
| 55 | $this->language = $language; |
| 56 | } |
| 57 | |
| 58 | /** |
| 59 | * @param bool $multiLang |
| 60 | */ |
| 61 | public function setMultiLang( $multiLang ) { |
| 62 | $this->multiLang = $multiLang; |
| 63 | } |
| 64 | |
| 65 | /** |
| 66 | * @param TemplateParser $templateParser |
| 67 | */ |
| 68 | public function setTemplateParser( TemplateParser $templateParser ) { |
| 69 | $this->templateParser = $templateParser; |
| 70 | } |
| 71 | |
| 72 | /** |
| 73 | * @param LicenseParser $licenseParser |
| 74 | */ |
| 75 | public function setLicenseParser( LicenseParser $licenseParser ) { |
| 76 | $this->licenseParser = $licenseParser; |
| 77 | } |
| 78 | |
| 79 | /** |
| 80 | * Collects metadata from a file, and adds it to a metadata array. |
| 81 | * The array has the following format: |
| 82 | * |
| 83 | * '<metadata field name>' => array( |
| 84 | * 'value' => '<value>', |
| 85 | * 'source' => '<where did the data come from>', |
| 86 | * ) |
| 87 | * |
| 88 | * For fields with multiple values and/or in multiple languages the format is more complex; |
| 89 | * see the documentation for the extmetadata API. |
| 90 | * |
| 91 | * @param array &$previousMetadata metadata collected so far; |
| 92 | * new metadata will be added to this array |
| 93 | * @param File $file |
| 94 | */ |
| 95 | public function collect( array &$previousMetadata, File $file ) { |
| 96 | $this->normalizeMetadataTimestamps( $previousMetadata ); |
| 97 | |
| 98 | $descriptionText = $this->getDescriptionText( $file, $this->language ); |
| 99 | |
| 100 | $categories = $this->getCategories( $file, $previousMetadata ); |
| 101 | $previousMetadata = array_merge( $previousMetadata, |
| 102 | $this->getCategoryMetadata( $categories ) ); |
| 103 | |
| 104 | $templateData = $this->templateParser->parsePage( $descriptionText ); |
| 105 | $previousMetadata = array_merge( $previousMetadata, |
| 106 | $this->getTemplateMetadata( $templateData ) ); |
| 107 | } |
| 108 | |
| 109 | /** |
| 110 | * Checks for the presence of metadata needed for attributing the file (author, source, license) |
| 111 | * and returns a list of keys corresponding to problems. |
| 112 | * @param ParserOutput $parserOutput |
| 113 | * @param File $file |
| 114 | * @return array one or more of the following keys: |
| 115 | * - no-license - failed to detect a license |
| 116 | * - no-description - failed to detect any image description |
| 117 | * - no-author - failed to detect author name or a custom attribution text |
| 118 | * - no-source - failed to detect the source of the image or a custom attribution text |
| 119 | */ |
| 120 | public function verifyAttributionMetadata( ParserOutput $parserOutput, File $file ) { |
| 121 | // HTML code of the file description |
| 122 | if ( !$parserOutput->hasText() ) { |
| 123 | $descriptionText = ''; |
| 124 | } else { |
| 125 | $descriptionText = $parserOutput->getText(); |
| 126 | } |
| 127 | |
| 128 | $templateData = $this->templateParser->parsePage( $descriptionText ); |
| 129 | $problems = $licenseData = $informationData = []; |
| 130 | |
| 131 | if ( isset( $templateData[TemplateParser::LICENSES_KEY] ) ) { |
| 132 | $licenseData = $this->selectLicense( $templateData[TemplateParser::LICENSES_KEY] ); |
| 133 | } |
| 134 | if ( isset( $templateData[TemplateParser::INFORMATION_FIELDS_KEY] ) ) { |
| 135 | $informationData = $this->selectInformationTemplate( |
| 136 | $templateData[TemplateParser::INFORMATION_FIELDS_KEY] ); |
| 137 | } |
| 138 | |
| 139 | if ( !isset( $licenseData['LicenseShortName'] ) |
| 140 | || $licenseData['LicenseShortName'] === '' |
| 141 | ) { |
| 142 | $problems[] = 'no-license'; |
| 143 | } |
| 144 | if ( !isset( $informationData['ImageDescription'] ) |
| 145 | || $informationData['ImageDescription'] === '' |
| 146 | ) { |
| 147 | $problems[] = 'no-description'; |
| 148 | } |
| 149 | if ( |
| 150 | ( !isset( $informationData['Artist'] ) || $informationData['Artist'] === '' ) && |
| 151 | ( !isset( $informationData['Attribution'] ) || $informationData['Attribution'] === '' ) |
| 152 | ) { |
| 153 | $problems[] = 'no-author'; |
| 154 | } |
| 155 | if ( |
| 156 | ( !isset( $informationData['Credit'] ) || $informationData['Credit'] === '' ) && |
| 157 | ( !isset( $informationData['Attribution'] ) || $informationData['Attribution'] === '' ) |
| 158 | ) { |
| 159 | $problems[] = 'no-source'; |
| 160 | } |
| 161 | |
| 162 | // Certain uploads (3D objects) need a patent license |
| 163 | $templates = $parserOutput->getTemplates(); |
| 164 | $templates = $templates[NS_TEMPLATE] ?? []; |
| 165 | if ( |
| 166 | !array_key_exists( '3dpatent', $templates ) && |
| 167 | $file->getMimeType() === 'application/sla' |
| 168 | ) { |
| 169 | $problems[] = 'no-patent'; |
| 170 | } |
| 171 | |
| 172 | return $problems; |
| 173 | } |
| 174 | |
| 175 | /** |
| 176 | * @param array $categories |
| 177 | * @return array |
| 178 | */ |
| 179 | protected function getCategoryMetadata( array $categories ) { |
| 180 | $assessments = $this->getAssessmentsAndRemoveFromCategories( $categories ); |
| 181 | $licenses = $this->getLicensesAndRemoveFromCategories( $categories ); |
| 182 | |
| 183 | return [ |
| 184 | 'Categories' => [ |
| 185 | 'value' => implode( '|', $categories ), |
| 186 | 'source' => 'commons-categories', |
| 187 | ], |
| 188 | 'Assessments' => [ |
| 189 | 'value' => implode( '|', $assessments ), |
| 190 | 'source' => 'commons-categories', |
| 191 | ], |
| 192 | ]; |
| 193 | } |
| 194 | |
| 195 | /** |
| 196 | * @param array $templateData |
| 197 | * @return array |
| 198 | */ |
| 199 | protected function getTemplateMetadata( $templateData ) { |
| 200 | // GetExtendedMetadata does not handle multivalued fields, |
| 201 | // we need to select one of everything |
| 202 | $templateFields = []; |
| 203 | |
| 204 | if ( isset( $templateData[TemplateParser::COORDINATES_KEY] ) ) { |
| 205 | $templateFields = array_merge( $templateFields, |
| 206 | $this->selectFirst( $templateData[TemplateParser::COORDINATES_KEY] ) ); |
| 207 | } |
| 208 | |
| 209 | if ( isset( $templateData[TemplateParser::INFORMATION_FIELDS_KEY] ) ) { |
| 210 | $templateFields = array_merge( $templateFields, $this->selectInformationTemplate( |
| 211 | $templateData[TemplateParser::INFORMATION_FIELDS_KEY] ) ); |
| 212 | } |
| 213 | |
| 214 | if ( isset( $templateData[TemplateParser::LICENSES_KEY] ) ) { |
| 215 | $templateFields = array_merge( $templateFields, |
| 216 | $this->selectLicense( $templateData[TemplateParser::LICENSES_KEY] ) ); |
| 217 | } |
| 218 | |
| 219 | if ( isset( $templateData[TemplateParser::DELETION_KEY] ) ) { |
| 220 | $templateFields = array_merge( $templateFields, |
| 221 | $this->selectFirst( $templateData[TemplateParser::DELETION_KEY] ) ); |
| 222 | } |
| 223 | |
| 224 | if ( isset( $templateData[TemplateParser::RESTRICTIONS_KEY] ) ) { |
| 225 | $templateFields = array_merge( $templateFields, |
| 226 | $this->selectFirst( $templateData[TemplateParser::RESTRICTIONS_KEY] ) ); |
| 227 | } |
| 228 | |
| 229 | $metadata = []; |
| 230 | foreach ( $templateFields as $name => $value ) { |
| 231 | $metadata[ $name ] = [ |
| 232 | 'value' => $value, |
| 233 | 'source' => 'commons-desc-page' |
| 234 | ]; |
| 235 | } |
| 236 | |
| 237 | // use short name to generate internal name used in i18n |
| 238 | if ( isset( $templateFields['LicenseShortName'] ) ) { |
| 239 | $licenseData = $this->licenseParser->parseLicenseString( |
| 240 | $templateFields['LicenseShortName'] ); |
| 241 | if ( isset( $licenseData['name'] ) ) { |
| 242 | $metadata['License'] = [ |
| 243 | 'value' => $licenseData['name'], |
| 244 | 'source' => 'commons-templates', |
| 245 | ]; |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | return $metadata; |
| 250 | } |
| 251 | |
| 252 | /** |
| 253 | * Gets the text of the file's description page. |
| 254 | * @param File $file |
| 255 | * @param Language $language |
| 256 | * @return string |
| 257 | */ |
| 258 | protected function getDescriptionText( File $file, Language $language ) { |
| 259 | # Note: If this is a local file, there is no caching here. |
| 260 | # However, the results of this module have longer caching for local |
| 261 | # files to help compensate. For foreign files, this method is cached |
| 262 | # via parser cache, and possibly a second cache depending on |
| 263 | # descriptionCacheExpiry (disabled on Wikimedia). |
| 264 | $text = $file->getDescriptionText( $language ); |
| 265 | |
| 266 | if ( get_class( $file ) == 'LocalFile' || get_class( $file ) == 'LocalFileMock' ) { |
| 267 | // LocalFile gets the text in a different way, and ends up with different output |
| 268 | // (specifically, relative instead of absolute URLs), so transform local URLs |
| 269 | // to absolute URLs after parse. |
| 270 | $text = ( new ParserOutput( $text ) )->getText( [ 'absoluteURLs' => true ] ); |
| 271 | } |
| 272 | |
| 273 | return $text; |
| 274 | } |
| 275 | |
| 276 | /** |
| 277 | * @param File $file |
| 278 | * @param array $data metadata passed to the onGetExtendedMetadata hook |
| 279 | * @return string[] list of category names in human-readable format |
| 280 | */ |
| 281 | protected function getCategories( File $file, array $data ) { |
| 282 | $categories = []; |
| 283 | |
| 284 | if ( is_a( $file, 'LocalFileMock' ) || is_a( $file, 'ForeignDBFileMock' ) ) { |
| 285 | // with all the hard-coded dependencies, mocking categoriy retrieval properly is |
| 286 | // pretty much impossible |
| 287 | return $file->mockedCategories; |
| 288 | } elseif ( $file instanceof LocalFile ) { |
| 289 | // for local or shared DB files (which are also LocalFile subclasses) |
| 290 | // categories can be queried directly from the database |
| 291 | |
| 292 | $page = MediaWikiServices::getInstance()->getWikiPageFactory()->newFromTitle( $file->getOriginalTitle() ); |
| 293 | if ( !$page instanceof WikiFilePage ) { |
| 294 | throw new InvalidArgumentException( |
| 295 | 'Cannot instance WikiFilePage to get categories for ' . $file->getName() |
| 296 | . ', got instance of ' . get_class( $page ) |
| 297 | ); |
| 298 | } |
| 299 | $page->setFile( $file ); |
| 300 | |
| 301 | $categoryTitles = $page->getForeignCategories(); |
| 302 | |
| 303 | foreach ( $categoryTitles as $title ) { |
| 304 | $categories[] = $title->getText(); |
| 305 | } |
| 306 | } elseif ( |
| 307 | $file instanceof ForeignAPIFile |
| 308 | && isset( $data['Categories'] ) |
| 309 | ) { |
| 310 | // getting categories for a ForeignAPIFile is not supported, but in case |
| 311 | // CommonsMetadata is installed on the remote repository as well, its output |
| 312 | // (including categories) is sent together with the extended file metadata, |
| 313 | // when the file is loaded. onGetExtendedMetadata hooks get that metadata |
| 314 | // when they are invoked. |
| 315 | $categories = explode( '|', $data['Categories']['value'] ); |
| 316 | } else { |
| 317 | // out of luck - file is probably from a ForeignAPIRepo |
| 318 | // with CommonsMetadata not installed there |
| 319 | wfDebug( 'CommonsMetadata: cannot read category data' ); |
| 320 | } |
| 321 | |
| 322 | return $categories; |
| 323 | } |
| 324 | |
| 325 | /** |
| 326 | * Matches category names to a category => license mapping, removes the matching categories |
| 327 | * and returns the corresponding licenses. |
| 328 | * @param array &$categories a list of human-readable category names. |
| 329 | * @return array |
| 330 | */ |
| 331 | protected function getLicensesAndRemoveFromCategories( &$categories ) { |
| 332 | $licenses = []; |
| 333 | foreach ( $categories as $i => $category ) { |
| 334 | $licenseData = $this->licenseParser->parseLicenseString( $category ); |
| 335 | if ( $licenseData ) { |
| 336 | $licenses[] = $licenseData['name']; |
| 337 | unset( $categories[$i] ); |
| 338 | } |
| 339 | } |
| 340 | $categories = array_merge( $categories ); // renumber to avoid holes in array |
| 341 | return $licenses; |
| 342 | } |
| 343 | |
| 344 | /** |
| 345 | * Matches category names to a category => assessment mapping, removes the matching categories |
| 346 | * and returns the corresponding assessments (valued image, picture of the day etc). |
| 347 | * @param array &$categories a list of human-readable category names. |
| 348 | * @return array |
| 349 | */ |
| 350 | protected function getAssessmentsAndRemoveFromCategories( &$categories ) { |
| 351 | $assessments = []; |
| 352 | foreach ( $categories as $i => $category ) { |
| 353 | foreach ( self::$assessmentCategories as $assessmentType => $regexp ) { |
| 354 | if ( preg_match( $regexp . 'i', $category ) ) { |
| 355 | $assessments[] = $assessmentType; |
| 356 | unset( $categories[$i] ); |
| 357 | } |
| 358 | } |
| 359 | } |
| 360 | $categories = array_merge( $categories ); // renumber to avoid holes in array |
| 361 | return array_unique( $assessments ); // potd/poty can happen multiple times |
| 362 | } |
| 363 | |
| 364 | /** |
| 365 | * Receives a list of metadata arrays and selects the first one to use. |
| 366 | * @param array $arrays an array of arrays of metdata fields in fieldname => value form |
| 367 | * @return array an array of metadata fields in fieldname => value form |
| 368 | */ |
| 369 | protected function selectFirst( $arrays ) { |
| 370 | // multiple metadata values for the same fields on the same image would not make much sense, |
| 371 | // so use the first value |
| 372 | return $arrays ? $arrays[0] : []; |
| 373 | } |
| 374 | |
| 375 | /** |
| 376 | * Receives the list of information templates found by the template parser and selects which one |
| 377 | * to use. Also collects all the authors to make sure attribution requirements are honored. |
| 378 | * @param array $informationTemplates an array of information templates, |
| 379 | * each is an array of metdata fields in fieldname => value form |
| 380 | * @return array an array of metdata fields in fieldname => value form |
| 381 | */ |
| 382 | protected function selectInformationTemplate( array $informationTemplates ) { |
| 383 | if ( !$informationTemplates ) { |
| 384 | return []; |
| 385 | } |
| 386 | |
| 387 | $authorCount = 0; |
| 388 | foreach ( $informationTemplates as $template ) { |
| 389 | if ( isset( $template['Artist'] ) ) { |
| 390 | $authorCount++; |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | if ( $authorCount > 1 ) { |
| 395 | $informationTemplates[0]['AuthorCount'] = $authorCount; |
| 396 | } |
| 397 | return $informationTemplates[0]; |
| 398 | } |
| 399 | |
| 400 | /** |
| 401 | * Receives the list of licenses found by the template parser and selects which one to use. |
| 402 | * @param array $licenses an array of licenses, each is an array of metadata fields |
| 403 | * in fieldname => value form |
| 404 | * @return array an array of metadata fields in fieldname => value form |
| 405 | */ |
| 406 | protected function selectLicense( array $licenses ) { |
| 407 | if ( !$licenses ) { |
| 408 | return []; |
| 409 | } |
| 410 | |
| 411 | $sortedLicenses = $this->licenseParser->sortDataByLicensePriority( $licenses, |
| 412 | static function ( $license ) { |
| 413 | if ( !isset( $license['LicenseShortName'] ) ) { |
| 414 | return null; |
| 415 | } |
| 416 | return $license['LicenseShortName']; |
| 417 | } |
| 418 | ); |
| 419 | |
| 420 | // sortDataByLicensePriority puts things in right order but also rearranges the keys |
| 421 | // - we don't want that |
| 422 | $sortedLicenses = array_values( $sortedLicenses ); |
| 423 | |
| 424 | if ( !$sortedLicenses ) { |
| 425 | return []; |
| 426 | } |
| 427 | |
| 428 | // T131896 - if any license template is marked nonfree, the image is probably nonfree |
| 429 | foreach ( $sortedLicenses as $license ) { |
| 430 | if ( !empty( $license['NonFree'] ) ) { |
| 431 | $sortedLicenses[0]['NonFree'] = $license['NonFree']; |
| 432 | break; |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | return $sortedLicenses[0]; |
| 437 | } |
| 438 | |
| 439 | /** |
| 440 | * Normalizes the metadata to wfTimestamp()'s TS_DB format |
| 441 | * @param array &$metadata |
| 442 | */ |
| 443 | protected function normalizeMetadataTimestamps( array &$metadata ) { |
| 444 | $fieldsToNormalize = [ 'DateTime', 'DateTimeOriginal' ]; |
| 445 | foreach ( $fieldsToNormalize as $field ) { |
| 446 | if ( |
| 447 | isset( $metadata[$field] ) && |
| 448 | isset( $metadata[$field]['value'] ) && |
| 449 | // Multilang values can get down here, which are arrays with |
| 450 | // '_type' => 'lang'. We don't want to pass an array to |
| 451 | // wfTimestamp: it won't work and will annoy PHP. |
| 452 | // @phan-suppress-next-line PhanTypeArraySuspicious |
| 453 | !isset( $metadata[$field]['value']['_type'] ) |
| 454 | ) { |
| 455 | $parsedTs = wfTimestamp( TS_DB, $metadata[$field]['value'] ); |
| 456 | if ( $parsedTs ) { |
| 457 | $metadata[$field]['value'] = $parsedTs; |
| 458 | } |
| 459 | } |
| 460 | } |
| 461 | } |
| 462 | } |