MediaWiki  master
moveToExternal.php
Go to the documentation of this file.
1 <?php
24 // NO_AUTOLOAD -- file scope code
25 
27 
28 define( 'REPORTING_INTERVAL', 1 );
29 
30 if ( !defined( 'MEDIAWIKI' ) ) {
31  $optionsWithArgs = [ 'e', 's' ];
32  require_once __DIR__ . '/../CommandLineInc.php';
33  require_once 'resolveStubs.php';
34 
35  $fname = 'moveToExternal';
36 
37  if ( !isset( $args[1] ) ) {
38  print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <type> <location>\n";
39  exit;
40  }
41 
42  $type = $args[0]; // e.g. "DB" or "mwstore"
43  $location = $args[1]; // e.g. "cluster12" or "global-swift"
44  $dbw = wfGetDB( DB_PRIMARY );
45 
46  $maxID = (int)( $options['e'] ?? $dbw->selectField( 'text', 'MAX(old_id)', '', $fname ) );
47  $minID = (int)( $options['s'] ?? 1 );
48 
49  moveToExternal( $type, $location, $maxID, $minID );
50 }
51 
52 function moveToExternal( $type, $location, $maxID, $minID = 1 ) {
53  $fname = 'moveToExternal';
54  $dbw = wfGetDB( DB_PRIMARY );
55  $dbr = wfGetDB( DB_REPLICA );
56 
57  $count = $maxID - $minID + 1;
58  $blockSize = 1000;
59  $numBlocks = ceil( $count / $blockSize );
60  print "Moving text rows from $minID to $maxID to external storage\n";
61 
62  $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
63  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
64  $extStore = $esFactory->getStore( $type );
65  $numMoved = 0;
66 
67  for ( $block = 0; $block < $numBlocks; $block++ ) {
68  $blockStart = $block * $blockSize + $minID;
69  $blockEnd = $blockStart + $blockSize - 1;
70 
71  if ( !( $block % REPORTING_INTERVAL ) ) {
72  print "oldid=$blockStart, moved=$numMoved\n";
73  $lbFactory->waitForReplication();
74  }
75 
76  $res = $dbr->select( 'text', [ 'old_id', 'old_flags', 'old_text' ],
77  [
78  "old_id BETWEEN $blockStart AND $blockEnd",
79  'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
80  ], $fname
81  );
82  foreach ( $res as $row ) {
83  # Resolve stubs
84  $text = $row->old_text;
85  $id = $row->old_id;
86  if ( $row->old_flags === '' ) {
87  $flags = 'external';
88  } else {
89  $flags = "{$row->old_flags},external";
90  }
91 
92  if ( strpos( $flags, 'object' ) !== false ) {
93  $obj = unserialize( $text );
94  $className = strtolower( get_class( $obj ) );
95  if ( $className == 'historyblobstub' ) {
96  # resolveStub( $id, $row->old_text, $row->old_flags );
97  # $numStubs++;
98  continue;
99  } elseif ( $className == 'historyblobcurstub' ) {
100  $text = gzdeflate( $obj->getText() );
101  $flags = 'utf-8,gzip,external';
102  } elseif ( $className == 'concatenatedgziphistoryblob' ) {
103  // Do nothing
104  } else {
105  print "Warning: unrecognised object class \"$className\"\n";
106  continue;
107  }
108  } else {
109  $className = false;
110  }
111 
112  if ( strlen( $text ) < 100 && $className === false ) {
113  // Don't move tiny revisions
114  continue;
115  }
116 
117  # print "Storing " . strlen( $text ) . " bytes to $url\n";
118  # print "old_id=$id\n";
119 
120  $url = $extStore->store( $location, $text );
121  if ( !$url ) {
122  print "Error writing to external storage\n";
123  exit;
124  }
125  $dbw->update( 'text',
126  [ 'old_flags' => $flags, 'old_text' => $url ],
127  [ 'old_id' => $id ], $fname );
128  $numMoved++;
129  }
130  }
131 }
moveToExternal
if(!defined( 'MEDIAWIKI')) moveToExternal( $type, $location, $maxID, $minID=1)
Definition: moveToExternal.php:52
$optionsWithArgs
global $optionsWithArgs
Definition: CommandLineInc.php:26
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:193
$res
$res
Definition: testCompression.php:57
$dbr
$dbr
Definition: testCompression.php:54
REPORTING_INTERVAL
const REPORTING_INTERVAL
Definition: moveToExternal.php:28
wfGetDB
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
Definition: GlobalFunctions.php:2203
$args
if( $line===false) $args
Definition: mcc.php:124
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
$type
$type
Definition: testCompression.php:52