Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
94.12% |
80 / 85 |
|
60.00% |
3 / 5 |
CRAP | |
0.00% |
0 / 1 |
DocumentSizeLimiter | |
94.12% |
80 / 85 |
|
60.00% |
3 / 5 |
29.17 | |
0.00% |
0 / 1 |
__construct | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
2 | |||
estimateDataSize | |
33.33% |
1 / 3 |
|
0.00% |
0 / 1 |
3.19 | |||
resize | |
100.00% |
21 / 21 |
|
100.00% |
1 / 1 |
7 | |||
truncateField | |
93.62% |
44 / 47 |
|
0.00% |
0 / 1 |
15.06 | |||
markWithTemplate | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
3 |
1 | <?php |
2 | |
3 | namespace CirrusSearch\BuildDocument; |
4 | |
5 | use CirrusSearch\Search\CirrusIndexField; |
6 | use Elastica\Document; |
7 | use Elastica\JSON; |
8 | |
9 | /** |
10 | * An approximate, incomplete and rather dangerous algorithm to reduce the size of a CirrusSearch |
11 | * document. |
12 | * |
13 | * This class is meant to reduce the size of abnormally large documents. What we can consider |
14 | * abnormally large is certainly prone to interpretation but this class was designed with numbers |
15 | * like 1Mb considered as extremely large. You should not expect this class to be byte precise |
16 | * and there is no guarantee that the resulting size after the operation will be below the expected |
17 | * max. There might be various reasons for this: |
18 | * - there are other fields than the ones listed above that take a lot of space |
19 | * - the expected size is so low that it does not even allow the json overhead to be present |
20 | * |
21 | * If the use-case is to ensure that the resulting json representation is below a size S you should |
22 | * definitely account for some overhead and ask this class to reduce the document to something smaller |
23 | * than S (i.e. S*0.9). |
24 | * |
25 | * Limiter heuristics are controlled by a profile that supports the following criteria: |
26 | * - max_size (int): the target maximum size of the document (when serialized as json) |
27 | * - field_types (array<string, string>): field name as key, the type of field (text or keyword) as value |
28 | * - max_field_size (array<string, int>): field name as key, max size as value, truncate these fields |
29 | * to the appropriate size |
30 | * - fields (array<string, int>): field name as key, min size as value, truncate these fields up to this |
31 | * minimal size as long as the document size is above max_size |
32 | * - markup_template (string): mark the document with this template if it was oversize. |
33 | * |
34 | * Text fields are truncated using mb_strcut, if the string is part of an array and it becomes empty |
35 | * after the truncation it's removed from the array, if the string is a "keyword" (non tokenized |
36 | * field) it's not truncated and simply removed from its array. |
37 | * |
38 | * If an array is mixing string and non-string data it's ignored. |
39 | */ |
40 | class DocumentSizeLimiter { |
41 | public const MANDATORY_REDUCTION_BUCKET = "mandatory_reduction"; |
42 | public const OVERSIZE_REDUCTION_REDUCTION_BUCKET = "oversize_reduction"; |
43 | public const HINT_DOC_SIZE_LIMITER_STATS = 'DocumentSizeLimiter_stats'; |
44 | |
45 | /** @var int */ |
46 | private $maxDocSize; |
47 | /** @var int */ |
48 | private $docLength; |
49 | /** @var Document */ |
50 | private $document; |
51 | /** @var string[] */ |
52 | private $fieldTypes; |
53 | /** @var int[] list of max field length */ |
54 | private $maxFieldSize; |
55 | /** @var int[] list of fields to truncate when the doc is oversize, value is the min length to keep */ |
56 | private $fields; |
57 | /** @var array<string,array<string,int>> */ |
58 | private $stats; |
59 | /** @var mixed|null */ |
60 | private $markupTemplate; |
61 | /** @var int the actual max size a truncated document can (takes into account the markup template that has to be added) */ |
62 | private $actualMaxDocSize; |
63 | |
64 | public function __construct( array $profile ) { |
65 | $this->maxDocSize = $profile['max_size'] ?? PHP_INT_MAX; |
66 | $this->fieldTypes = $profile['field_types'] ?? []; |
67 | $this->maxFieldSize = $profile["max_field_size"] ?? []; |
68 | $this->fields = $profile["fields"] ?? []; |
69 | $this->markupTemplate = $profile["markup_template"] ?? null; |
70 | $this->actualMaxDocSize = $this->maxDocSize; |
71 | if ( $this->markupTemplate !== null ) { |
72 | $this->actualMaxDocSize -= strlen( $this->markupTemplate ) + 3; // 3 is 2 " and a comma |
73 | } |
74 | } |
75 | |
76 | public static function estimateDataSize( Document $document ): int { |
77 | try { |
78 | return strlen( JSON::stringify( $document->getData(), \JSON_UNESCAPED_SLASHES | \JSON_UNESCAPED_UNICODE ) ); |
79 | } catch ( \JsonException $je ) { |
80 | // Ignore, consider this of length 0, process is likely to fail at later point |
81 | } |
82 | return 0; |
83 | } |
84 | |
85 | /** |
86 | * Truncate some textual data from the input Document. |
87 | * @param Document $document |
88 | * @return array some statistics about the process. |
89 | */ |
90 | public function resize( Document $document ): array { |
91 | $this->stats = []; |
92 | $this->document = $document; |
93 | $originalDocLength = self::estimateDataSize( $document ); |
94 | $this->docLength = $originalDocLength; |
95 | // first pass to force some fields |
96 | foreach ( $this->maxFieldSize as $field => $len ) { |
97 | $this->truncateField( $field, ( $this->fieldTypes[$field] ?? "text" ) === "keyword", |
98 | $len, 0, self::MANDATORY_REDUCTION_BUCKET ); |
99 | } |
100 | |
101 | // second pass applied only if the doc is oversize |
102 | if ( $this->docLength > $this->maxDocSize ) { |
103 | foreach ( $this->fields as $field => $len ) { |
104 | if ( $this->docLength <= $this->actualMaxDocSize ) { |
105 | break; |
106 | } |
107 | $this->truncateField( $field, ( $this->fieldTypes[$field] ?? "text" ) === "keyword", |
108 | $len, $this->actualMaxDocSize, self::OVERSIZE_REDUCTION_REDUCTION_BUCKET ); |
109 | } |
110 | } |
111 | /** @phan-suppress-next-line PhanRedundantCondition */ |
112 | if ( $this->markupTemplate != null && !empty( $this->stats[self::OVERSIZE_REDUCTION_REDUCTION_BUCKET] ) ) { |
113 | $this->markWithTemplate( $document ); |
114 | } |
115 | $this->stats["document"] = [ |
116 | "original_length" => $originalDocLength, |
117 | "new_length" => $this->docLength, |
118 | ]; |
119 | CirrusIndexField::setHint( $document, self::HINT_DOC_SIZE_LIMITER_STATS, $this->stats ); |
120 | return $this->stats; |
121 | } |
122 | |
123 | private function truncateField( string $field, bool $keyword, int $minFieldLength, int $maxDocSize, string $statBucket ): void { |
124 | if ( !$this->document->has( $field ) ) { |
125 | return; |
126 | } |
127 | $fieldData = $this->document->get( $field ); |
128 | $plainString = false; |
129 | |
130 | // If the field is a plain string but is marked as a keyword we prefer to not touch it. |
131 | // It is probable that such fields are not of variable length (IDs, mimetypes) and thus |
132 | // it would make little to have a profile that tries to truncate those. But out of caution |
133 | // we simply skip those. |
134 | if ( is_string( $fieldData ) && !$keyword ) { |
135 | // wrap and plain string into an array to reuse the same loop as string[] fields. |
136 | $fieldData = [ $fieldData ]; |
137 | $plainString = true; |
138 | } |
139 | if ( !is_array( $fieldData ) ) { |
140 | return; |
141 | } |
142 | |
143 | $onlyStrings = array_reduce( $fieldData, static function ( $isString, $str ) { |
144 | return $isString && is_string( $str ); |
145 | }, true ); |
146 | |
147 | $onlyStrings = array_reduce( $fieldData, static function ( $isString, $str ) { |
148 | return $isString && is_string( $str ); |
149 | }, true ); |
150 | if ( !$onlyStrings ) { |
151 | // not messing-up with mixed-types |
152 | return; |
153 | } |
154 | |
155 | $fieldLen = array_reduce( $fieldData, static function ( $siz, $str ) { |
156 | return $siz + strlen( $str ); |
157 | }, 0 ); |
158 | $sizeReduction = 0; |
159 | // Since we generally truncate the end of a text we also remove array elements from the end. |
160 | for ( $index = count( $fieldData ) - 1; $index >= 0; $index-- ) { |
161 | $remainingFieldLen = $fieldLen - $sizeReduction; |
162 | $maxSizeToRemove = $this->docLength - $sizeReduction - $maxDocSize; |
163 | if ( $remainingFieldLen <= $minFieldLength ) { |
164 | break; |
165 | } |
166 | if ( $maxSizeToRemove <= 0 ) { |
167 | break; |
168 | } |
169 | if ( $remainingFieldLen <= 0 ) { |
170 | break; |
171 | } |
172 | $data = &$fieldData[$index]; |
173 | $len = strlen( $data ); |
174 | if ( $keyword ) { |
175 | $sizeReduction += strlen( $data ); |
176 | unset( $fieldData[$index] ); |
177 | } else { |
178 | $removableLen = $remainingFieldLen - $minFieldLength; |
179 | |
180 | $newLen = $len - max( min( $maxSizeToRemove, $len, $removableLen ), 0 ); |
181 | $data = mb_strcut( $data, 0, $newLen ); |
182 | $sizeReduction += $len - strlen( $data ); |
183 | if ( $data === "" ) { |
184 | unset( $fieldData[$index] ); |
185 | } |
186 | } |
187 | } |
188 | $this->docLength -= $sizeReduction; |
189 | $fieldData = array_values( $fieldData ); |
190 | if ( $plainString ) { |
191 | $fieldData = array_pop( $fieldData ) ?? ""; // prefers empty string over null |
192 | } |
193 | $this->document->set( $field, $fieldData ); |
194 | $this->stats[$statBucket][$field] = $sizeReduction; |
195 | } |
196 | |
197 | private function markWithTemplate( Document $document ) { |
198 | $templates = []; |
199 | if ( $document->has( "template" ) ) { |
200 | $templates = $document->get( "template" ); |
201 | } |
202 | // add this markup to the main NS to avoid pulling Title and the ns text service |
203 | // this will be searchable via hastemplate::the_markup_template |
204 | $templates[] = $this->markupTemplate; |
205 | $this->docLength += strlen( $this->markupTemplate ) + 2 + ( count( $templates ) > 1 ? 1 : 0 ); |
206 | $document->set( "template", $templates ); |
207 | } |
208 | } |