Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
77.19% |
176 / 228 |
|
50.00% |
11 / 22 |
CRAP | |
0.00% |
0 / 1 |
| GlobalIdGenerator | |
77.19% |
176 / 228 |
|
50.00% |
11 / 22 |
117.68 | |
0.00% |
0 / 1 |
| __construct | |
55.00% |
11 / 20 |
|
0.00% |
0 / 1 |
9.28 | |||
| newTimestampedUID88 | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
1 | |||
| getTimestampedID88 | |
87.50% |
7 / 8 |
|
0.00% |
0 / 1 |
2.01 | |||
| newTimestampedUID128 | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
1 | |||
| getTimestampedID128 | |
90.00% |
9 / 10 |
|
0.00% |
0 / 1 |
2.00 | |||
| newUUIDv1 | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getUUIDv1 | |
95.65% |
22 / 23 |
|
0.00% |
0 / 1 |
2 | |||
| newRawUUIDv1 | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| newUUIDv4 | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
1 | |||
| newRawUUIDv4 | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| newSequentialPerNodeID | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| newSequentialPerNodeIDs | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getTimestampFromUUIDv1 | |
100.00% |
10 / 10 |
|
100.00% |
1 / 1 |
2 | |||
| getSequentialPerNodeIDs | |
81.48% |
22 / 27 |
|
0.00% |
0 / 1 |
9.51 | |||
| getTimeAndDelay | |
81.25% |
39 / 48 |
|
0.00% |
0 / 1 |
9.53 | |||
| timeWaitUntil | |
66.67% |
4 / 6 |
|
0.00% |
0 / 1 |
2.15 | |||
| millisecondsSinceEpochBinary | |
66.67% |
4 / 6 |
|
0.00% |
0 / 1 |
2.15 | |||
| intervalsSinceGregorianBinary | |
31.58% |
6 / 19 |
|
0.00% |
0 / 1 |
9.12 | |||
| load | |
65.22% |
15 / 23 |
|
0.00% |
0 / 1 |
12.41 | |||
| getNodeId32 | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| getNodeId48 | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| deleteCacheFiles | n/a |
0 / 0 |
n/a |
0 / 0 |
5 | |||||
| unitTestTearDown | n/a |
0 / 0 |
n/a |
0 / 0 |
1 | |||||
| __destruct | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License as published by |
| 5 | * the Free Software Foundation; either version 2 of the License, or |
| 6 | * (at your option) any later version. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 11 | * GNU General Public License for more details. |
| 12 | * |
| 13 | * You should have received a copy of the GNU General Public License along |
| 14 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 16 | * http://www.gnu.org/copyleft/gpl.html |
| 17 | * |
| 18 | * @file |
| 19 | */ |
| 20 | |
| 21 | namespace Wikimedia\UUID; |
| 22 | |
| 23 | use InvalidArgumentException; |
| 24 | use RuntimeException; |
| 25 | use Wikimedia\Assert\Assert; |
| 26 | use Wikimedia\AtEase\AtEase; |
| 27 | use Wikimedia\Timestamp\ConvertibleTimestamp; |
| 28 | |
| 29 | /** |
| 30 | * Class for getting statistically unique IDs without a central coordinator |
| 31 | * |
| 32 | * @since 1.35 |
| 33 | */ |
| 34 | class GlobalIdGenerator { |
| 35 | /** @var callable Callback for running shell commands */ |
| 36 | protected $shellCallback; |
| 37 | |
| 38 | /** @var string Temporary directory */ |
| 39 | protected $tmpDir; |
| 40 | /** @var string |
| 41 | * File prefix containing user ID to prevent collisions |
| 42 | * if multiple users run MediaWiki (T268420) and getmyuid() is enabled |
| 43 | */ |
| 44 | protected $uniqueFilePrefix; |
| 45 | /** @var string Local file path */ |
| 46 | protected $nodeIdFile; |
| 47 | /** @var string Node ID in binary (32 bits) */ |
| 48 | protected $nodeId32; |
| 49 | /** @var string Node ID in binary (48 bits) */ |
| 50 | protected $nodeId48; |
| 51 | |
| 52 | /** @var bool Whether initialization completed */ |
| 53 | protected $loaded = false; |
| 54 | /** @var string Local file path */ |
| 55 | protected $lockFile88; |
| 56 | /** @var string Local file path */ |
| 57 | protected $lockFile128; |
| 58 | /** @var string Local file path */ |
| 59 | protected $lockFileUUID; |
| 60 | |
| 61 | /** @var array Cached file handles */ |
| 62 | protected $fileHandles = []; |
| 63 | |
| 64 | /** |
| 65 | * Avoid using __CLASS__ so namespace separators aren't interpreted |
| 66 | * as path components on Windows (T259693) |
| 67 | */ |
| 68 | private const FILE_PREFIX = 'mw-GlobalIdGenerator'; |
| 69 | |
| 70 | /** Key used in the serialized clock state map that is stored on disk */ |
| 71 | private const CLOCK_TIME = 'time'; |
| 72 | /** Key used in the serialized clock state map that is stored on disk */ |
| 73 | private const CLOCK_COUNTER = 'counter'; |
| 74 | /** Key used in the serialized clock state map that is stored on disk */ |
| 75 | private const CLOCK_SEQUENCE = 'clkSeq'; |
| 76 | /** Key used in the serialized clock state map that is stored on disk */ |
| 77 | private const CLOCK_OFFSET = 'offset'; |
| 78 | /** Key used in the serialized clock state map that is stored on disk */ |
| 79 | private const CLOCK_OFFSET_COUNTER = 'offsetCounter'; |
| 80 | |
| 81 | /** |
| 82 | * @param string|bool $tempDirectory A writable temporary directory |
| 83 | * @param callback $shellCallback A callback that takes a shell command and returns the output |
| 84 | */ |
| 85 | public function __construct( $tempDirectory, $shellCallback ) { |
| 86 | if ( func_num_args() >= 3 && !is_callable( $shellCallback ) ) { |
| 87 | trigger_error( |
| 88 | __CLASS__ . ' with a BagOStuff instance was deprecated in MediaWiki 1.37.', |
| 89 | E_USER_DEPRECATED |
| 90 | ); |
| 91 | $shellCallback = func_get_arg( 2 ); |
| 92 | } |
| 93 | if ( !strlen( $tempDirectory ) ) { |
| 94 | throw new InvalidArgumentException( "No temp directory provided" ); |
| 95 | } |
| 96 | $this->tmpDir = $tempDirectory; |
| 97 | // Include the UID in the filename (T268420, T358768) |
| 98 | if ( function_exists( 'posix_geteuid' ) ) { |
| 99 | $fileSuffix = posix_geteuid(); |
| 100 | } elseif ( function_exists( 'getmyuid' ) ) { |
| 101 | $fileSuffix = getmyuid(); |
| 102 | } else { |
| 103 | $fileSuffix = ''; |
| 104 | } |
| 105 | $this->uniqueFilePrefix = self::FILE_PREFIX . $fileSuffix; |
| 106 | $this->nodeIdFile = $tempDirectory . '/' . $this->uniqueFilePrefix . '-UID-nodeid'; |
| 107 | // If different processes run as different users, they may have different temp dirs. |
| 108 | // This is dealt with by initializing the clock sequence number and counters randomly. |
| 109 | $this->lockFile88 = $tempDirectory . '/' . $this->uniqueFilePrefix . '-UID-88'; |
| 110 | $this->lockFile128 = $tempDirectory . '/' . $this->uniqueFilePrefix . '-UID-128'; |
| 111 | $this->lockFileUUID = $tempDirectory . '/' . $this->uniqueFilePrefix . '-UUID-128'; |
| 112 | |
| 113 | $this->shellCallback = $shellCallback; |
| 114 | } |
| 115 | |
| 116 | /** |
| 117 | * Get a statistically unique 88-bit unsigned integer ID string. |
| 118 | * The bits of the UID are prefixed with the time (down to the millisecond). |
| 119 | * |
| 120 | * These IDs are suitable as values for the shard key of distributed data. |
| 121 | * If a column uses these as values, it should be declared UNIQUE to handle collisions. |
| 122 | * New rows almost always have higher UIDs, which makes B-TREE updates on INSERT fast. |
| 123 | * They can also be stored "DECIMAL(27) UNSIGNED" or BINARY(11) in MySQL. |
| 124 | * |
| 125 | * UID generation is serialized on each server (as the node ID is for the whole machine). |
| 126 | * |
| 127 | * @param int $base Specifies a base other than 10 |
| 128 | * @return string Number |
| 129 | * @throws RuntimeException |
| 130 | */ |
| 131 | public function newTimestampedUID88( int $base = 10 ) { |
| 132 | Assert::parameter( $base <= 36, '$base', 'must be <= 36' ); |
| 133 | Assert::parameter( $base >= 2, '$base', 'must be >= 2' ); |
| 134 | |
| 135 | $info = $this->getTimeAndDelay( 'lockFile88', 1, 1024, 1024 ); |
| 136 | $info[self::CLOCK_OFFSET_COUNTER] %= 1024; |
| 137 | |
| 138 | return \Wikimedia\base_convert( $this->getTimestampedID88( $info ), 2, $base ); |
| 139 | } |
| 140 | |
| 141 | /** |
| 142 | * @param array $info result of GlobalIdGenerator::getTimeAndDelay() |
| 143 | * @return string 88 bits |
| 144 | * @throws RuntimeException |
| 145 | */ |
| 146 | protected function getTimestampedID88( array $info ) { |
| 147 | $time = $info[self::CLOCK_TIME]; |
| 148 | $counter = $info[self::CLOCK_OFFSET_COUNTER]; |
| 149 | // Take the 46 LSBs of "milliseconds since epoch" |
| 150 | $id_bin = $this->millisecondsSinceEpochBinary( $time ); |
| 151 | // Add a 10 bit counter resulting in 56 bits total |
| 152 | $id_bin .= str_pad( decbin( $counter ), 10, '0', STR_PAD_LEFT ); |
| 153 | // Add the 32 bit node ID resulting in 88 bits total |
| 154 | $id_bin .= $this->getNodeId32(); |
| 155 | // Convert to a 1-27 digit integer string |
| 156 | if ( strlen( $id_bin ) !== 88 ) { |
| 157 | throw new RuntimeException( "Detected overflow for millisecond timestamp." ); |
| 158 | } |
| 159 | |
| 160 | return $id_bin; |
| 161 | } |
| 162 | |
| 163 | /** |
| 164 | * Get a statistically unique 128-bit unsigned integer ID string. |
| 165 | * The bits of the UID are prefixed with the time (down to the millisecond). |
| 166 | * |
| 167 | * These IDs are suitable as globally unique IDs, without any enforced uniqueness. |
| 168 | * New rows almost always have higher UIDs, which makes B-TREE updates on INSERT fast. |
| 169 | * They can also be stored as "DECIMAL(39) UNSIGNED" or BINARY(16) in MySQL. |
| 170 | * |
| 171 | * UID generation is serialized on each server (as the node ID is for the whole machine). |
| 172 | * |
| 173 | * @param int $base Specifies a base other than 10 |
| 174 | * @return string Number |
| 175 | * @throws RuntimeException |
| 176 | */ |
| 177 | public function newTimestampedUID128( int $base = 10 ) { |
| 178 | Assert::parameter( $base <= 36, '$base', 'must be <= 36' ); |
| 179 | Assert::parameter( $base >= 2, '$base', 'must be >= 2' ); |
| 180 | |
| 181 | $info = $this->getTimeAndDelay( 'lockFile128', 16384, 1_048_576, 1_048_576 ); |
| 182 | $info[self::CLOCK_OFFSET_COUNTER] %= 1_048_576; |
| 183 | |
| 184 | return \Wikimedia\base_convert( $this->getTimestampedID128( $info ), 2, $base ); |
| 185 | } |
| 186 | |
| 187 | /** |
| 188 | * @param array $info The result of GlobalIdGenerator::getTimeAndDelay() |
| 189 | * @return string 128 bits |
| 190 | * @throws RuntimeException |
| 191 | */ |
| 192 | protected function getTimestampedID128( array $info ) { |
| 193 | $time = $info[self::CLOCK_TIME]; |
| 194 | $counter = $info[self::CLOCK_OFFSET_COUNTER]; |
| 195 | $clkSeq = $info[self::CLOCK_SEQUENCE]; |
| 196 | // Take the 46 LSBs of "milliseconds since epoch" |
| 197 | $id_bin = $this->millisecondsSinceEpochBinary( $time ); |
| 198 | // Add a 20 bit counter resulting in 66 bits total |
| 199 | $id_bin .= str_pad( decbin( $counter ), 20, '0', STR_PAD_LEFT ); |
| 200 | // Add a 14 bit clock sequence number resulting in 80 bits total |
| 201 | $id_bin .= str_pad( decbin( $clkSeq ), 14, '0', STR_PAD_LEFT ); |
| 202 | // Add the 48 bit node ID resulting in 128 bits total |
| 203 | $id_bin .= $this->getNodeId48(); |
| 204 | // Convert to a 1-39 digit integer string |
| 205 | if ( strlen( $id_bin ) !== 128 ) { |
| 206 | throw new RuntimeException( "Detected overflow for millisecond timestamp." ); |
| 207 | } |
| 208 | |
| 209 | return $id_bin; |
| 210 | } |
| 211 | |
| 212 | /** |
| 213 | * Return an RFC4122 compliant v1 UUID |
| 214 | * |
| 215 | * @return string |
| 216 | * @throws RuntimeException |
| 217 | */ |
| 218 | public function newUUIDv1() { |
| 219 | // There can be up to 10000 intervals for the same millisecond timestamp. |
| 220 | // [0,4999] counter + [0,5000] offset is in [0,9999] for the offset counter. |
| 221 | // Add this onto the timestamp to allow making up to 5000 IDs per second. |
| 222 | return $this->getUUIDv1( $this->getTimeAndDelay( 'lockFileUUID', 16384, 5000, 5001 ) ); |
| 223 | } |
| 224 | |
| 225 | /** |
| 226 | * @param array $info Result of GlobalIdGenerator::getTimeAndDelay() |
| 227 | * @return string 128 bits |
| 228 | */ |
| 229 | protected function getUUIDv1( array $info ) { |
| 230 | $clkSeq_bin = \Wikimedia\base_convert( $info[self::CLOCK_SEQUENCE], 10, 2, 14 ); |
| 231 | $time_bin = $this->intervalsSinceGregorianBinary( |
| 232 | $info[self::CLOCK_TIME], |
| 233 | $info[self::CLOCK_OFFSET_COUNTER] |
| 234 | ); |
| 235 | // Take the 32 bits of "time low" |
| 236 | $id_bin = substr( $time_bin, 28, 32 ); |
| 237 | // Add 16 bits of "time mid" resulting in 48 bits total |
| 238 | $id_bin .= substr( $time_bin, 12, 16 ); |
| 239 | // Add 4 bit version resulting in 52 bits total |
| 240 | $id_bin .= '0001'; |
| 241 | // Add 12 bits of "time high" resulting in 64 bits total |
| 242 | $id_bin .= substr( $time_bin, 0, 12 ); |
| 243 | // Add 2 bits of "variant" resulting in 66 bits total |
| 244 | $id_bin .= '10'; |
| 245 | // Add 6 bits of "clock seq high" resulting in 72 bits total |
| 246 | $id_bin .= substr( $clkSeq_bin, 0, 6 ); |
| 247 | // Add 8 bits of "clock seq low" resulting in 80 bits total |
| 248 | $id_bin .= substr( $clkSeq_bin, 6, 8 ); |
| 249 | // Add the 48 bit node ID resulting in 128 bits total |
| 250 | $id_bin .= $this->getNodeId48(); |
| 251 | // Convert to a 32 char hex string with dashes |
| 252 | if ( strlen( $id_bin ) !== 128 ) { |
| 253 | throw new RuntimeException( "Detected overflow for millisecond timestamp." ); |
| 254 | } |
| 255 | $hex = \Wikimedia\base_convert( $id_bin, 2, 16, 32 ); |
| 256 | return sprintf( '%s-%s-%s-%s-%s', |
| 257 | // "time_low" (32 bits) |
| 258 | substr( $hex, 0, 8 ), |
| 259 | // "time_mid" (16 bits) |
| 260 | substr( $hex, 8, 4 ), |
| 261 | // "time_hi_and_version" (16 bits) |
| 262 | substr( $hex, 12, 4 ), |
| 263 | // "clk_seq_hi_res" (8 bits) and "clk_seq_low" (8 bits) |
| 264 | substr( $hex, 16, 4 ), |
| 265 | // "node" (48 bits) |
| 266 | substr( $hex, 20, 12 ) |
| 267 | ); |
| 268 | } |
| 269 | |
| 270 | /** |
| 271 | * Return an RFC4122 compliant v1 UUID |
| 272 | * |
| 273 | * @return string 32 hex characters with no hyphens |
| 274 | * @throws RuntimeException |
| 275 | */ |
| 276 | public function newRawUUIDv1() { |
| 277 | return str_replace( '-', '', $this->newUUIDv1() ); |
| 278 | } |
| 279 | |
| 280 | /** |
| 281 | * Return an RFC4122 compliant v4 UUID |
| 282 | * |
| 283 | * @return string |
| 284 | * @throws RuntimeException |
| 285 | */ |
| 286 | public function newUUIDv4() { |
| 287 | $hex = bin2hex( random_bytes( 32 / 2 ) ); |
| 288 | |
| 289 | return sprintf( '%s-%s-%s-%s-%s', |
| 290 | // "time_low" (32 bits) |
| 291 | substr( $hex, 0, 8 ), |
| 292 | // "time_mid" (16 bits) |
| 293 | substr( $hex, 8, 4 ), |
| 294 | // "time_hi_and_version" (16 bits) |
| 295 | '4' . substr( $hex, 12, 3 ), |
| 296 | // "clk_seq_hi_res" (8 bits, variant is binary 10x) and "clk_seq_low" (8 bits) |
| 297 | dechex( 0x8 | ( hexdec( $hex[15] ) & 0x3 ) ) . $hex[16] . substr( $hex, 17, 2 ), |
| 298 | // "node" (48 bits) |
| 299 | substr( $hex, 19, 12 ) |
| 300 | ); |
| 301 | } |
| 302 | |
| 303 | /** |
| 304 | * Return an RFC4122 compliant v4 UUID |
| 305 | * |
| 306 | * @return string 32 hex characters with no hyphens |
| 307 | * @throws RuntimeException |
| 308 | */ |
| 309 | public function newRawUUIDv4() { |
| 310 | return str_replace( '-', '', $this->newUUIDv4() ); |
| 311 | } |
| 312 | |
| 313 | /** |
| 314 | * Return an ID that is sequential *only* for this node and bucket |
| 315 | * |
| 316 | * These IDs are suitable for per-host sequence numbers, e.g. for some packet protocols. |
| 317 | * |
| 318 | * @param string $bucket Arbitrary bucket name (should be ASCII) |
| 319 | * @param int $bits Bit size (<=48) of resulting numbers before wrap-around |
| 320 | * @return float Integer value as float |
| 321 | */ |
| 322 | public function newSequentialPerNodeID( $bucket, $bits = 48 ) { |
| 323 | return current( $this->newSequentialPerNodeIDs( $bucket, $bits, 1 ) ); |
| 324 | } |
| 325 | |
| 326 | /** |
| 327 | * Return IDs that are sequential *only* for this node and bucket |
| 328 | * |
| 329 | * @param string $bucket Arbitrary bucket name (should be ASCII) |
| 330 | * @param int $bits Bit size (16 to 48) of resulting numbers before wrap-around |
| 331 | * @param int $count Number of IDs to return |
| 332 | * @return array Ordered list of float integer values |
| 333 | * @see GlobalIdGenerator::newSequentialPerNodeID() |
| 334 | */ |
| 335 | public function newSequentialPerNodeIDs( $bucket, $bits, $count ) { |
| 336 | return $this->getSequentialPerNodeIDs( $bucket, $bits, $count ); |
| 337 | } |
| 338 | |
| 339 | /** |
| 340 | * Get timestamp in a specified format from UUIDv1 |
| 341 | * |
| 342 | * @param string $uuid the UUID to get the timestamp from |
| 343 | * @param int $format the format to convert the timestamp to. Default: TS_MW |
| 344 | * @return string|false timestamp in requested format or false |
| 345 | */ |
| 346 | public function getTimestampFromUUIDv1( string $uuid, int $format = TS_MW ) { |
| 347 | $components = []; |
| 348 | if ( !preg_match( |
| 349 | '/^([0-9a-f]{8})-([0-9a-f]{4})-(1[0-9a-f]{3})-([89ab][0-9a-f]{3})-([0-9a-f]{12})$/', |
| 350 | $uuid, |
| 351 | $components |
| 352 | ) ) { |
| 353 | throw new InvalidArgumentException( "Invalid UUIDv1 {$uuid}" ); |
| 354 | } |
| 355 | |
| 356 | $timestamp = hexdec( substr( $components[3], 1 ) . $components[2] . $components[1] ); |
| 357 | // The 60 bit timestamp value is constructed from fields of this UUID. |
| 358 | // The timestamp is measured in 100-nanosecond units since midnight, October 15, 1582 UTC. |
| 359 | $unixTime = ( $timestamp - 0x01b21dd213814000 ) / 1e7; |
| 360 | |
| 361 | return ConvertibleTimestamp::convert( $format, $unixTime ); |
| 362 | } |
| 363 | |
| 364 | /** |
| 365 | * Return IDs that are sequential *only* for this node and bucket |
| 366 | * |
| 367 | * @param string $bucket Arbitrary bucket name (should be ASCII) |
| 368 | * @param int $bits Bit size (16 to 48) of resulting numbers before wrap-around |
| 369 | * @param int $count Number of IDs to return |
| 370 | * @return array Ordered list of float integer values |
| 371 | * @throws RuntimeException |
| 372 | * @see GlobalIdGenerator::newSequentialPerNodeID() |
| 373 | */ |
| 374 | protected function getSequentialPerNodeIDs( $bucket, $bits, $count ) { |
| 375 | if ( $count <= 0 ) { |
| 376 | return []; |
| 377 | } |
| 378 | if ( $bits < 16 || $bits > 48 ) { |
| 379 | throw new RuntimeException( "Requested bit size ($bits) is out of range." ); |
| 380 | } |
| 381 | |
| 382 | $path = $this->tmpDir . '/' . $this->uniqueFilePrefix . '-' . rawurlencode( $bucket ) . '-48'; |
| 383 | // Get the UID lock file handle |
| 384 | if ( isset( $this->fileHandles[$path] ) ) { |
| 385 | $handle = $this->fileHandles[$path]; |
| 386 | } else { |
| 387 | $handle = fopen( $path, 'cb+' ); |
| 388 | $this->fileHandles[$path] = $handle ?: null; |
| 389 | } |
| 390 | // Acquire the UID lock file |
| 391 | if ( $handle === false ) { |
| 392 | throw new RuntimeException( "Could not open '{$path}'." ); |
| 393 | } |
| 394 | if ( !flock( $handle, LOCK_EX ) ) { |
| 395 | fclose( $handle ); |
| 396 | throw new RuntimeException( "Could not acquire '{$path}'." ); |
| 397 | } |
| 398 | // Fetch the counter value and increment it... |
| 399 | rewind( $handle ); |
| 400 | |
| 401 | // fetch as float |
| 402 | $counter = floor( (float)trim( fgets( $handle ) ) ) + $count; |
| 403 | |
| 404 | // Write back the new counter value |
| 405 | ftruncate( $handle, 0 ); |
| 406 | rewind( $handle ); |
| 407 | |
| 408 | // Use fmod() to avoid "division by zero" on 32 bit machines |
| 409 | // warp-around as needed |
| 410 | fwrite( $handle, (string)fmod( $counter, 2 ** 48 ) ); |
| 411 | fflush( $handle ); |
| 412 | |
| 413 | // Release the UID lock file |
| 414 | flock( $handle, LOCK_UN ); |
| 415 | |
| 416 | $ids = []; |
| 417 | $divisor = 2 ** $bits; |
| 418 | |
| 419 | // pre-increment counter value |
| 420 | $currentId = floor( $counter - $count ); |
| 421 | for ( $i = 0; $i < $count; ++$i ) { |
| 422 | // Use fmod() to avoid "division by zero" on 32 bit machines |
| 423 | $ids[] = fmod( ++$currentId, $divisor ); |
| 424 | } |
| 425 | |
| 426 | return $ids; |
| 427 | } |
| 428 | |
| 429 | /** |
| 430 | * Get a (time,counter,clock sequence) where (time,counter) is higher |
| 431 | * than any previous (time,counter) value for the given clock sequence. |
| 432 | * This is useful for making UIDs sequential on a per-node basis. |
| 433 | * |
| 434 | * @param string $lockFile Name of a local lock file |
| 435 | * @param int $clockSeqSize The number of possible clock sequence values |
| 436 | * @param int $counterSize The number of possible counter values |
| 437 | * @param int $offsetSize The number of possible offset values |
| 438 | * @return array Array with the following keys: |
| 439 | * - GlobalIdGenerator::CLOCK_TIME: (integer seconds, integer milliseconds) array |
| 440 | * - GlobalIdGenerator::CLOCK_COUNTER: integer millisecond tie-breaking counter |
| 441 | * - GlobalIdGenerator::CLOCK_SEQUENCE: integer clock identifier that is local to the node |
| 442 | * - GlobalIdGenerator::CLOCK_OFFSET: integer offset for millisecond tie-breaking counter |
| 443 | * - GlobalIdGenerator::CLOCK_OFFSET_COUNTER: integer; CLOCK_COUNTER with CLOCK_OFFSET applied |
| 444 | * @throws RuntimeException |
| 445 | */ |
| 446 | protected function getTimeAndDelay( $lockFile, $clockSeqSize, $counterSize, $offsetSize ) { |
| 447 | // Get the UID lock file handle |
| 448 | if ( isset( $this->fileHandles[$this->$lockFile] ) ) { |
| 449 | $handle = $this->fileHandles[$this->$lockFile]; |
| 450 | } else { |
| 451 | $handle = fopen( $this->$lockFile, 'cb+' ); |
| 452 | $this->fileHandles[$this->$lockFile] = $handle ?: null; |
| 453 | } |
| 454 | // Acquire the UID lock file |
| 455 | if ( $handle === false ) { |
| 456 | throw new RuntimeException( "Could not open '{$this->$lockFile}'." ); |
| 457 | } |
| 458 | if ( !flock( $handle, LOCK_EX ) ) { |
| 459 | fclose( $handle ); |
| 460 | throw new RuntimeException( "Could not acquire '{$this->$lockFile}'." ); |
| 461 | } |
| 462 | |
| 463 | // The formatters that use this method expect a timestamp with millisecond |
| 464 | // precision and a counter upto a certain size. When more IDs than the counter |
| 465 | // size are generated during the same timestamp, an exception is thrown as we |
| 466 | // cannot increment further, because the formatted ID would not have enough |
| 467 | // bits to fit the counter. |
| 468 | // |
| 469 | // To orchestrate this between independent PHP processes on the same host, |
| 470 | // we must have a common sense of time so that we only have to maintain |
| 471 | // a single counter in a single lock file. |
| 472 | // |
| 473 | // Given that: |
| 474 | // * The system clock can be observed via time(), without milliseconds. |
| 475 | // * Some other clock can be observed via microtime(), which also offers |
| 476 | // millisecond precision. |
| 477 | // * microtime() drifts in-process further and further away from the system |
| 478 | // clock the longer a process runs for. |
| 479 | // For example, on 2018-10-03 an HHVM 3.18 JobQueue process at WMF, |
| 480 | // that ran for 9 min 55 sec, microtime drifted by 7 seconds. |
| 481 | // time() does not have this problem. See https://bugs.php.net/bug.php?id=42659. |
| 482 | // |
| 483 | // We have two choices: |
| 484 | // |
| 485 | // 1. Use microtime() with the following caveats: |
| 486 | // - The last stored time may be in the future, or our current time may be in the |
| 487 | // past, in which case we'll frequently enter the slow timeWaitUntil() method to |
| 488 | // try and "sync" the current process with the previous process. |
| 489 | // We mustn't block for long though, max 10ms? |
| 490 | // - For any drift above 10ms, we pretend that the clock went backwards, and treat |
| 491 | // it the same way as after an NTP sync, by incrementing clock sequence instead. |
| 492 | // Given the sequence rolls over automatically, and silently, and is meant to be |
| 493 | // rare, this essentially sacrifices a reasonable guarantee of uniqueness. |
| 494 | // - For long running processes (e.g. longer than a few seconds) the drift can |
| 495 | // easily be more than 2 seconds. Because we only have a single lock file |
| 496 | // and don't want to keep too many counters and deal with clearing those, |
| 497 | // we fatal the user and refuse to make an ID. (T94522) |
| 498 | // - This offers terrible service availability. |
| 499 | // 2. Use time() instead, and expand the counter size by 1000x and use its |
| 500 | // digits as if they were the millisecond fraction of our timestamp. |
| 501 | // Known caveats or perf impact: None. We still need to read-write our |
| 502 | // lock file on each generation, so might as well make the most of it. |
| 503 | // |
| 504 | // We choose the latter. |
| 505 | $msecCounterSize = $counterSize * 1000; |
| 506 | |
| 507 | rewind( $handle ); |
| 508 | // Format of lock file contents: |
| 509 | // "<clk seq> <sec> <msec counter> <rand offset>" |
| 510 | $data = explode( ' ', fgets( $handle ) ); |
| 511 | |
| 512 | if ( count( $data ) === 4 ) { |
| 513 | // The UID lock file was already initialized |
| 514 | $clkSeq = (int)$data[0] % $clockSeqSize; |
| 515 | $prevSec = (int)$data[1]; |
| 516 | $prevMsecCounter = (int)$data[2] % $msecCounterSize; |
| 517 | $randOffset = (int)$data[3] % $counterSize; |
| 518 | // If the system clock moved back or inter-process clock drift caused the last |
| 519 | // writer process to record a higher time than the current process time, then |
| 520 | // briefly wait for the current process clock to catch up. |
| 521 | $sec = $this->timeWaitUntil( $prevSec ); |
| 522 | if ( $sec === false ) { |
| 523 | // There was too much clock drift to wait. Bump the clock sequence number to |
| 524 | // avoid collisions between new and already-generated IDs with the same time. |
| 525 | $clkSeq = ( $clkSeq + 1 ) % $clockSeqSize; |
| 526 | $sec = time(); |
| 527 | $msecCounter = 0; |
| 528 | $randOffset = random_int( 0, $offsetSize - 1 ); |
| 529 | trigger_error( "Clock was set back; sequence number incremented." ); |
| 530 | } elseif ( $sec === $prevSec ) { |
| 531 | // The time matches the last ID. Bump the tie-breaking counter. |
| 532 | $msecCounter = $prevMsecCounter + 1; |
| 533 | if ( $msecCounter >= $msecCounterSize ) { |
| 534 | // More IDs generated with the same time than counterSize can accommodate |
| 535 | flock( $handle, LOCK_UN ); |
| 536 | throw new RuntimeException( "Counter overflow for timestamp value." ); |
| 537 | } |
| 538 | } else { |
| 539 | // The time is higher than the last ID. Reset the tie-breaking counter. |
| 540 | $msecCounter = 0; |
| 541 | } |
| 542 | } else { |
| 543 | // Initialize UID lock file information |
| 544 | $clkSeq = random_int( 0, $clockSeqSize - 1 ); |
| 545 | $sec = time(); |
| 546 | $msecCounter = 0; |
| 547 | $randOffset = random_int( 0, $offsetSize - 1 ); |
| 548 | } |
| 549 | |
| 550 | // Update and release the UID lock file |
| 551 | ftruncate( $handle, 0 ); |
| 552 | rewind( $handle ); |
| 553 | fwrite( $handle, "{$clkSeq} {$sec} {$msecCounter} {$randOffset}" ); |
| 554 | fflush( $handle ); |
| 555 | flock( $handle, LOCK_UN ); |
| 556 | |
| 557 | // Split msecCounter back into msec and counter |
| 558 | $msec = (int)( $msecCounter / 1000 ); |
| 559 | $counter = $msecCounter % 1000; |
| 560 | |
| 561 | return [ |
| 562 | self::CLOCK_TIME => [ $sec, $msec ], |
| 563 | self::CLOCK_COUNTER => $counter, |
| 564 | self::CLOCK_SEQUENCE => $clkSeq, |
| 565 | self::CLOCK_OFFSET => $randOffset, |
| 566 | self::CLOCK_OFFSET_COUNTER => $counter + $randOffset, |
| 567 | ]; |
| 568 | } |
| 569 | |
| 570 | /** |
| 571 | * Wait till the current timestamp reaches $time and return the current |
| 572 | * timestamp. This returns false if it would have to wait more than 10ms. |
| 573 | * |
| 574 | * @param int $time Result of time() |
| 575 | * @return int|bool Timestamp or false |
| 576 | */ |
| 577 | protected function timeWaitUntil( $time ) { |
| 578 | $start = microtime( true ); |
| 579 | do { |
| 580 | $ct = time(); |
| 581 | // https://www.php.net/manual/en/language.operators.comparison.php |
| 582 | if ( $ct >= $time ) { |
| 583 | // current time is higher than or equal to than $time |
| 584 | return $ct; |
| 585 | } |
| 586 | // up to 10ms |
| 587 | } while ( ( microtime( true ) - $start ) <= 0.010 ); |
| 588 | |
| 589 | return false; |
| 590 | } |
| 591 | |
| 592 | /** |
| 593 | * @param array $time Array of second and millisecond integers |
| 594 | * @return string 46 LSBs of "milliseconds since epoch" in binary (rolls over in 4201) |
| 595 | * @throws RuntimeException |
| 596 | */ |
| 597 | protected function millisecondsSinceEpochBinary( array $time ) { |
| 598 | [ $sec, $msec ] = $time; |
| 599 | $ts = 1000 * $sec + $msec; |
| 600 | if ( $ts > 2 ** 52 ) { |
| 601 | throw new RuntimeException( __METHOD__ . |
| 602 | ': sorry, this function doesn\'t work after the year 144680' ); |
| 603 | } |
| 604 | |
| 605 | return substr( \Wikimedia\base_convert( (string)$ts, 10, 2, 46 ), -46 ); |
| 606 | } |
| 607 | |
| 608 | /** |
| 609 | * @param array $time Array of second and millisecond integers |
| 610 | * @param int $delta Number of intervals to add on to the timestamp |
| 611 | * @return string 60 bits of "100ns intervals since 15 October 1582" (rolls over in 3400) |
| 612 | * @throws RuntimeException |
| 613 | */ |
| 614 | protected function intervalsSinceGregorianBinary( array $time, $delta = 0 ) { |
| 615 | [ $sec, $msec ] = $time; |
| 616 | $offset = '122192928000000000'; |
| 617 | |
| 618 | // 64 bit integers |
| 619 | if ( PHP_INT_SIZE >= 8 ) { |
| 620 | $ts = ( 1000 * $sec + $msec ) * 10000 + (int)$offset + $delta; |
| 621 | $id_bin = str_pad( decbin( $ts % ( 2 ** 60 ) ), 60, '0', STR_PAD_LEFT ); |
| 622 | } elseif ( extension_loaded( 'gmp' ) ) { |
| 623 | // ms |
| 624 | $ts = gmp_add( gmp_mul( (string)$sec, '1000' ), (string)$msec ); |
| 625 | // 100ns intervals |
| 626 | $ts = gmp_add( gmp_mul( $ts, '10000' ), $offset ); |
| 627 | $ts = gmp_add( $ts, (string)$delta ); |
| 628 | // wrap around |
| 629 | $ts = gmp_mod( $ts, gmp_pow( '2', 60 ) ); |
| 630 | $id_bin = str_pad( gmp_strval( $ts, 2 ), 60, '0', STR_PAD_LEFT ); |
| 631 | } elseif ( extension_loaded( 'bcmath' ) ) { |
| 632 | // ms |
| 633 | $ts = bcadd( bcmul( $sec, '1000' ), $msec ); |
| 634 | // 100ns intervals |
| 635 | $ts = bcadd( bcmul( $ts, '10000' ), $offset ); |
| 636 | $ts = bcadd( $ts, (string)$delta ); |
| 637 | // wrap around |
| 638 | $ts = bcmod( $ts, bcpow( '2', '60' ) ); |
| 639 | $id_bin = \Wikimedia\base_convert( $ts, 10, 2, 60 ); |
| 640 | } else { |
| 641 | throw new RuntimeException( 'bcmath or gmp extension required for 32 bit machines.' ); |
| 642 | } |
| 643 | return $id_bin; |
| 644 | } |
| 645 | |
| 646 | /** |
| 647 | * Load the node ID information |
| 648 | */ |
| 649 | private function load() { |
| 650 | if ( $this->loaded ) { |
| 651 | return; |
| 652 | } |
| 653 | |
| 654 | $this->loaded = true; |
| 655 | |
| 656 | // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged |
| 657 | $nodeId = @file_get_contents( $this->nodeIdFile ) ?: ''; |
| 658 | // Try to get some ID that uniquely identifies this machine (RFC 4122)... |
| 659 | if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) { |
| 660 | AtEase::suppressWarnings(); |
| 661 | if ( PHP_OS_FAMILY === 'Windows' ) { |
| 662 | // https://technet.microsoft.com/en-us/library/bb490913.aspx |
| 663 | $csv = trim( ( $this->shellCallback )( 'getmac /NH /FO CSV' ) ); |
| 664 | $line = substr( $csv, 0, strcspn( $csv, "\n" ) ); |
| 665 | $info = str_getcsv( $line, ",", "\"", "\\" ); |
| 666 | // @phan-suppress-next-line PhanTypeMismatchArgumentNullableInternal False positive |
| 667 | $nodeId = isset( $info[0] ) ? str_replace( '-', '', $info[0] ) : ''; |
| 668 | } elseif ( is_executable( '/sbin/ifconfig' ) ) { |
| 669 | // Linux/BSD/Solaris/OS X |
| 670 | // See https://linux.die.net/man/8/ifconfig |
| 671 | $m = []; |
| 672 | preg_match( '/\s([0-9a-f]{2}(?::[0-9a-f]{2}){5})\s/', |
| 673 | ( $this->shellCallback )( '/sbin/ifconfig -a' ), $m ); |
| 674 | $nodeId = isset( $m[1] ) ? str_replace( ':', '', $m[1] ) : ''; |
| 675 | } |
| 676 | AtEase::restoreWarnings(); |
| 677 | if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) { |
| 678 | $nodeId = bin2hex( random_bytes( 12 / 2 ) ); |
| 679 | // set multicast bit |
| 680 | $nodeId[1] = dechex( hexdec( $nodeId[1] ) | 0x1 ); |
| 681 | } |
| 682 | file_put_contents( $this->nodeIdFile, $nodeId ); |
| 683 | } |
| 684 | $this->nodeId32 = \Wikimedia\base_convert( substr( sha1( $nodeId ), 0, 8 ), 16, 2, 32 ); |
| 685 | $this->nodeId48 = \Wikimedia\base_convert( $nodeId, 16, 2, 48 ); |
| 686 | } |
| 687 | |
| 688 | /** |
| 689 | * @return string |
| 690 | */ |
| 691 | private function getNodeId32() { |
| 692 | $this->load(); |
| 693 | |
| 694 | return $this->nodeId32; |
| 695 | } |
| 696 | |
| 697 | /** |
| 698 | * @return string |
| 699 | */ |
| 700 | private function getNodeId48() { |
| 701 | $this->load(); |
| 702 | |
| 703 | return $this->nodeId48; |
| 704 | } |
| 705 | |
| 706 | /** |
| 707 | * Delete all cache files that have been created (T46850) |
| 708 | * |
| 709 | * This is a cleanup method primarily meant to be used from unit tests to |
| 710 | * avoid polluting the local filesystem. If used outside of a unit test |
| 711 | * environment it should be used with caution as it may destroy state saved |
| 712 | * in the files. |
| 713 | * |
| 714 | * @see unitTestTearDown |
| 715 | * @codeCoverageIgnore |
| 716 | */ |
| 717 | private function deleteCacheFiles() { |
| 718 | foreach ( $this->fileHandles as $path => $handle ) { |
| 719 | if ( $handle !== null ) { |
| 720 | fclose( $handle ); |
| 721 | } |
| 722 | if ( is_file( $path ) ) { |
| 723 | unlink( $path ); |
| 724 | } |
| 725 | unset( $this->fileHandles[$path] ); |
| 726 | } |
| 727 | if ( is_file( $this->nodeIdFile ) ) { |
| 728 | unlink( $this->nodeIdFile ); |
| 729 | } |
| 730 | } |
| 731 | |
| 732 | /** |
| 733 | * Cleanup resources when tearing down after a unit test (T46850) |
| 734 | * |
| 735 | * This is a cleanup method primarily meant to be used from unit tests to |
| 736 | * avoid polluting the local filesystem. If used outside of a unit test |
| 737 | * environment it should be used with caution as it may destroy state saved |
| 738 | * in the files. |
| 739 | * |
| 740 | * @internal For use by unit tests |
| 741 | * @see deleteCacheFiles |
| 742 | * @codeCoverageIgnore |
| 743 | */ |
| 744 | public function unitTestTearDown() { |
| 745 | $this->deleteCacheFiles(); |
| 746 | } |
| 747 | |
| 748 | public function __destruct() { |
| 749 | // @phan-suppress-next-line PhanPluginUseReturnValueInternalKnown |
| 750 | array_map( 'fclose', array_filter( $this->fileHandles ) ); |
| 751 | } |
| 752 | } |