MediaWiki REL1_35
moveToExternal.php
Go to the documentation of this file.
1<?php
25
26define( 'REPORTING_INTERVAL', 1 );
27
28if ( !defined( 'MEDIAWIKI' ) ) {
29 $optionsWithArgs = [ 'e', 's' ];
30 require_once __DIR__ . '/../commandLine.inc';
31 require_once 'resolveStubs.php';
32
33 $fname = 'moveToExternal';
34
35 if ( !isset( $args[1] ) ) {
36 print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <type> <location>\n";
37 exit;
38 }
39
40 $type = $args[0]; // e.g. "DB" or "mwstore"
41 $location = $args[1]; // e.g. "cluster12" or "global-swift"
42 $dbw = wfGetDB( DB_MASTER );
43
44 $maxID = $options['e'] ?? $dbw->selectField( 'text', 'MAX(old_id)', '', $fname );
45 $minID = $options['s'] ?? 1;
46
47 moveToExternal( $type, $location, $maxID, $minID );
48}
49
50function moveToExternal( $type, $location, $maxID, $minID = 1 ) {
51 $fname = 'moveToExternal';
52 $dbw = wfGetDB( DB_MASTER );
54
55 $count = $maxID - $minID + 1;
56 $blockSize = 1000;
57 $numBlocks = ceil( $count / $blockSize );
58 print "Moving text rows from $minID to $maxID to external storage\n";
59
60 $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
61 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
62 $extStore = $esFactory->getStore( $type );
63 $numMoved = 0;
64
65 for ( $block = 0; $block < $numBlocks; $block++ ) {
66 $blockStart = $block * $blockSize + $minID;
67 $blockEnd = $blockStart + $blockSize - 1;
68
69 if ( !( $block % REPORTING_INTERVAL ) ) {
70 print "oldid=$blockStart, moved=$numMoved\n";
71 $lbFactory->waitForReplication();
72 }
73
74 $res = $dbr->select( 'text', [ 'old_id', 'old_flags', 'old_text' ],
75 [
76 "old_id BETWEEN $blockStart AND $blockEnd",
77 'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
78 ], $fname
79 );
80 foreach ( $res as $row ) {
81 # Resolve stubs
82 $text = $row->old_text;
83 $id = $row->old_id;
84 if ( $row->old_flags === '' ) {
85 $flags = 'external';
86 } else {
87 $flags = "{$row->old_flags},external";
88 }
89
90 if ( strpos( $flags, 'object' ) !== false ) {
91 $obj = unserialize( $text );
92 $className = strtolower( get_class( $obj ) );
93 if ( $className == 'historyblobstub' ) {
94 # resolveStub( $id, $row->old_text, $row->old_flags );
95 # $numStubs++;
96 continue;
97 } elseif ( $className == 'historyblobcurstub' ) {
98 $text = gzdeflate( $obj->getText() );
99 $flags = 'utf-8,gzip,external';
100 } elseif ( $className == 'concatenatedgziphistoryblob' ) {
101 // Do nothing
102 } else {
103 print "Warning: unrecognised object class \"$className\"\n";
104 continue;
105 }
106 } else {
107 $className = false;
108 }
109
110 if ( strlen( $text ) < 100 && $className === false ) {
111 // Don't move tiny revisions
112 continue;
113 }
114
115 # print "Storing " . strlen( $text ) . " bytes to $url\n";
116 # print "old_id=$id\n";
117
118 $url = $extStore->store( $location, $text );
119 if ( !$url ) {
120 print "Error writing to external storage\n";
121 exit;
122 }
123 $dbw->update( 'text',
124 [ 'old_flags' => $flags, 'old_text' => $url ],
125 [ 'old_id' => $id ], $fname );
126 $numMoved++;
127 }
128 }
129}
unserialize( $serialized)
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
MediaWikiServices is the service locator for the application scope of MediaWiki.
global $optionsWithArgs
while(( $__line=Maintenance::readconsole()) !==false) print
Definition eval.php:64
if( $line===false) $args
Definition mcc.php:124
if(!defined('MEDIAWIKI')) moveToExternal( $type, $location, $maxID, $minID=1)
const REPORTING_INTERVAL
const DB_REPLICA
Definition defines.php:25
const DB_MASTER
Definition defines.php:29