Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
94.12% |
80 / 85 |
|
60.00% |
3 / 5 |
CRAP | |
0.00% |
0 / 1 |
| DocumentSizeLimiter | |
94.12% |
80 / 85 |
|
60.00% |
3 / 5 |
29.17 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
2 | |||
| estimateDataSize | |
33.33% |
1 / 3 |
|
0.00% |
0 / 1 |
3.19 | |||
| resize | |
100.00% |
21 / 21 |
|
100.00% |
1 / 1 |
7 | |||
| truncateField | |
93.62% |
44 / 47 |
|
0.00% |
0 / 1 |
15.06 | |||
| markWithTemplate | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
3 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch\BuildDocument; |
| 4 | |
| 5 | use CirrusSearch\Search\CirrusIndexField; |
| 6 | use Elastica\Document; |
| 7 | use Elastica\JSON; |
| 8 | |
| 9 | /** |
| 10 | * An approximate, incomplete and rather dangerous algorithm to reduce the size of a CirrusSearch |
| 11 | * document. |
| 12 | * |
| 13 | * This class is meant to reduce the size of abnormally large documents. What we can consider |
| 14 | * abnormally large is certainly prone to interpretation but this class was designed with numbers |
| 15 | * like 1Mb considered as extremely large. You should not expect this class to be byte precise |
| 16 | * and there is no guarantee that the resulting size after the operation will be below the expected |
| 17 | * max. There might be various reasons for this: |
| 18 | * - there are other fields than the ones listed above that take a lot of space |
| 19 | * - the expected size is so low that it does not even allow the json overhead to be present |
| 20 | * |
| 21 | * If the use-case is to ensure that the resulting json representation is below a size S you should |
| 22 | * definitely account for some overhead and ask this class to reduce the document to something smaller |
| 23 | * than S (i.e. S*0.9). |
| 24 | * |
| 25 | * Limiter heuristics are controlled by a profile that supports the following criteria: |
| 26 | * - max_size (int): the target maximum size of the document (when serialized as json) |
| 27 | * - field_types (array<string, string>): field name as key, the type of field (text or keyword) as value |
| 28 | * - max_field_size (array<string, int>): field name as key, max size as value, truncate these fields |
| 29 | * to the appropriate size |
| 30 | * - fields (array<string, int>): field name as key, min size as value, truncate these fields up to this |
| 31 | * minimal size as long as the document size is above max_size |
| 32 | * - markup_template (string): mark the document with this template if it was oversize. |
| 33 | * |
| 34 | * Text fields are truncated using mb_strcut, if the string is part of an array and it becomes empty |
| 35 | * after the truncation it's removed from the array, if the string is a "keyword" (non tokenized |
| 36 | * field) it's not truncated and simply removed from its array. |
| 37 | * |
| 38 | * If an array is mixing string and non-string data it's ignored. |
| 39 | */ |
| 40 | class DocumentSizeLimiter { |
| 41 | public const MANDATORY_REDUCTION_BUCKET = "mandatory_reduction"; |
| 42 | public const OVERSIZE_REDUCTION_REDUCTION_BUCKET = "oversize_reduction"; |
| 43 | public const HINT_DOC_SIZE_LIMITER_STATS = 'DocumentSizeLimiter_stats'; |
| 44 | |
| 45 | /** @var int */ |
| 46 | private $maxDocSize; |
| 47 | /** @var int */ |
| 48 | private $docLength; |
| 49 | /** @var Document */ |
| 50 | private $document; |
| 51 | /** @var string[] */ |
| 52 | private $fieldTypes; |
| 53 | /** @var int[] list of max field length */ |
| 54 | private $maxFieldSize; |
| 55 | /** @var int[] list of fields to truncate when the doc is oversize, value is the min length to keep */ |
| 56 | private $fields; |
| 57 | /** @var array<string,array<string,int>> */ |
| 58 | private $stats; |
| 59 | /** @var mixed|null */ |
| 60 | private $markupTemplate; |
| 61 | /** @var int the actual max size a truncated document can (takes into account the markup template that has to be added) */ |
| 62 | private $actualMaxDocSize; |
| 63 | |
| 64 | public function __construct( array $profile ) { |
| 65 | $this->maxDocSize = $profile['max_size'] ?? PHP_INT_MAX; |
| 66 | $this->fieldTypes = $profile['field_types'] ?? []; |
| 67 | $this->maxFieldSize = $profile["max_field_size"] ?? []; |
| 68 | $this->fields = $profile["fields"] ?? []; |
| 69 | $this->markupTemplate = $profile["markup_template"] ?? null; |
| 70 | $this->actualMaxDocSize = $this->maxDocSize; |
| 71 | if ( $this->markupTemplate !== null ) { |
| 72 | $this->actualMaxDocSize -= strlen( $this->markupTemplate ) + 3; // 3 is 2 " and a comma |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | public static function estimateDataSize( Document $document ): int { |
| 77 | try { |
| 78 | return strlen( JSON::stringify( $document->getData(), \JSON_UNESCAPED_SLASHES | \JSON_UNESCAPED_UNICODE ) ); |
| 79 | } catch ( \JsonException $je ) { |
| 80 | // Ignore, consider this of length 0, process is likely to fail at later point |
| 81 | } |
| 82 | return 0; |
| 83 | } |
| 84 | |
| 85 | /** |
| 86 | * Truncate some textual data from the input Document. |
| 87 | * @param Document $document |
| 88 | * @return array some statistics about the process. |
| 89 | */ |
| 90 | public function resize( Document $document ): array { |
| 91 | $this->stats = []; |
| 92 | $this->document = $document; |
| 93 | $originalDocLength = self::estimateDataSize( $document ); |
| 94 | $this->docLength = $originalDocLength; |
| 95 | // first pass to force some fields |
| 96 | foreach ( $this->maxFieldSize as $field => $len ) { |
| 97 | $this->truncateField( $field, ( $this->fieldTypes[$field] ?? "text" ) === "keyword", |
| 98 | $len, 0, self::MANDATORY_REDUCTION_BUCKET ); |
| 99 | } |
| 100 | |
| 101 | // second pass applied only if the doc is oversize |
| 102 | if ( $this->docLength > $this->maxDocSize ) { |
| 103 | foreach ( $this->fields as $field => $len ) { |
| 104 | if ( $this->docLength <= $this->actualMaxDocSize ) { |
| 105 | break; |
| 106 | } |
| 107 | $this->truncateField( $field, ( $this->fieldTypes[$field] ?? "text" ) === "keyword", |
| 108 | $len, $this->actualMaxDocSize, self::OVERSIZE_REDUCTION_REDUCTION_BUCKET ); |
| 109 | } |
| 110 | } |
| 111 | /** @phan-suppress-next-line PhanRedundantCondition */ |
| 112 | if ( $this->markupTemplate != null && !empty( $this->stats[self::OVERSIZE_REDUCTION_REDUCTION_BUCKET] ) ) { |
| 113 | $this->markWithTemplate( $document ); |
| 114 | } |
| 115 | $this->stats["document"] = [ |
| 116 | "original_length" => $originalDocLength, |
| 117 | "new_length" => $this->docLength, |
| 118 | ]; |
| 119 | CirrusIndexField::setHint( $document, self::HINT_DOC_SIZE_LIMITER_STATS, $this->stats ); |
| 120 | return $this->stats; |
| 121 | } |
| 122 | |
| 123 | private function truncateField( string $field, bool $keyword, int $minFieldLength, int $maxDocSize, string $statBucket ): void { |
| 124 | if ( !$this->document->has( $field ) ) { |
| 125 | return; |
| 126 | } |
| 127 | $fieldData = $this->document->get( $field ); |
| 128 | $plainString = false; |
| 129 | |
| 130 | // If the field is a plain string but is marked as a keyword we prefer to not touch it. |
| 131 | // It is probable that such fields are not of variable length (IDs, mimetypes) and thus |
| 132 | // it would make little to have a profile that tries to truncate those. But out of caution |
| 133 | // we simply skip those. |
| 134 | if ( is_string( $fieldData ) && !$keyword ) { |
| 135 | // wrap and plain string into an array to reuse the same loop as string[] fields. |
| 136 | $fieldData = [ $fieldData ]; |
| 137 | $plainString = true; |
| 138 | } |
| 139 | if ( !is_array( $fieldData ) ) { |
| 140 | return; |
| 141 | } |
| 142 | |
| 143 | $onlyStrings = array_reduce( $fieldData, static function ( $isString, $str ) { |
| 144 | return $isString && is_string( $str ); |
| 145 | }, true ); |
| 146 | |
| 147 | $onlyStrings = array_reduce( $fieldData, static function ( $isString, $str ) { |
| 148 | return $isString && is_string( $str ); |
| 149 | }, true ); |
| 150 | if ( !$onlyStrings ) { |
| 151 | // not messing-up with mixed-types |
| 152 | return; |
| 153 | } |
| 154 | |
| 155 | $fieldLen = array_reduce( $fieldData, static function ( $siz, $str ) { |
| 156 | return $siz + strlen( $str ); |
| 157 | }, 0 ); |
| 158 | $sizeReduction = 0; |
| 159 | // Since we generally truncate the end of a text we also remove array elements from the end. |
| 160 | for ( $index = count( $fieldData ) - 1; $index >= 0; $index-- ) { |
| 161 | $remainingFieldLen = $fieldLen - $sizeReduction; |
| 162 | $maxSizeToRemove = $this->docLength - $sizeReduction - $maxDocSize; |
| 163 | if ( $remainingFieldLen <= $minFieldLength ) { |
| 164 | break; |
| 165 | } |
| 166 | if ( $maxSizeToRemove <= 0 ) { |
| 167 | break; |
| 168 | } |
| 169 | if ( $remainingFieldLen <= 0 ) { |
| 170 | break; |
| 171 | } |
| 172 | $data = &$fieldData[$index]; |
| 173 | $len = strlen( $data ); |
| 174 | if ( $keyword ) { |
| 175 | $sizeReduction += strlen( $data ); |
| 176 | unset( $fieldData[$index] ); |
| 177 | } else { |
| 178 | $removableLen = $remainingFieldLen - $minFieldLength; |
| 179 | |
| 180 | $newLen = $len - max( min( $maxSizeToRemove, $len, $removableLen ), 0 ); |
| 181 | $data = mb_strcut( $data, 0, $newLen ); |
| 182 | $sizeReduction += $len - strlen( $data ); |
| 183 | if ( $data === "" ) { |
| 184 | unset( $fieldData[$index] ); |
| 185 | } |
| 186 | } |
| 187 | } |
| 188 | $this->docLength -= $sizeReduction; |
| 189 | $fieldData = array_values( $fieldData ); |
| 190 | if ( $plainString ) { |
| 191 | $fieldData = array_pop( $fieldData ) ?? ""; // prefers empty string over null |
| 192 | } |
| 193 | $this->document->set( $field, $fieldData ); |
| 194 | $this->stats[$statBucket][$field] = $sizeReduction; |
| 195 | } |
| 196 | |
| 197 | private function markWithTemplate( Document $document ) { |
| 198 | $templates = []; |
| 199 | if ( $document->has( "template" ) ) { |
| 200 | $templates = $document->get( "template" ); |
| 201 | } |
| 202 | // add this markup to the main NS to avoid pulling Title and the ns text service |
| 203 | // this will be searchable via hastemplate::the_markup_template |
| 204 | $templates[] = $this->markupTemplate; |
| 205 | $this->docLength += strlen( $this->markupTemplate ) + 2 + ( count( $templates ) > 1 ? 1 : 0 ); |
| 206 | $document->set( "template", $templates ); |
| 207 | } |
| 208 | } |