MediaWiki  master
moveToExternal.php
Go to the documentation of this file.
1 <?php
25 
26 define( 'REPORTING_INTERVAL', 1 );
27 
28 if ( !defined( 'MEDIAWIKI' ) ) {
29  $optionsWithArgs = [ 'e', 's' ];
30  require_once __DIR__ . '/../commandLine.inc';
31  require_once 'resolveStubs.php';
32 
33  $fname = 'moveToExternal';
34 
35  if ( !isset( $args[1] ) ) {
36  print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <type> <location>\n";
37  exit;
38  }
39 
40  $type = $args[0]; // e.g. "DB" or "mwstore"
41  $location = $args[1]; // e.g. "cluster12" or "global-swift"
42  $dbw = wfGetDB( DB_MASTER );
43 
44  $maxID = $options['e'] ?? $dbw->selectField( 'text', 'MAX(old_id)', '', $fname );
45  $minID = $options['s'] ?? 1;
46 
47  moveToExternal( $type, $location, $maxID, $minID );
48 }
49 
50 function moveToExternal( $type, $location, $maxID, $minID = 1 ) {
51  $fname = 'moveToExternal';
52  $dbw = wfGetDB( DB_MASTER );
53  $dbr = wfGetDB( DB_REPLICA );
54 
55  $count = $maxID - $minID + 1;
56  $blockSize = 1000;
57  $numBlocks = ceil( $count / $blockSize );
58  print "Moving text rows from $minID to $maxID to external storage\n";
59 
60  $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
61  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
62  $extStore = $esFactory->getStore( $type );
63  $numMoved = 0;
64 
65  for ( $block = 0; $block < $numBlocks; $block++ ) {
66  $blockStart = $block * $blockSize + $minID;
67  $blockEnd = $blockStart + $blockSize - 1;
68 
69  if ( !( $block % REPORTING_INTERVAL ) ) {
70  print "oldid=$blockStart, moved=$numMoved\n";
71  $lbFactory->waitForReplication();
72  }
73 
74  $res = $dbr->select( 'text', [ 'old_id', 'old_flags', 'old_text' ],
75  [
76  "old_id BETWEEN $blockStart AND $blockEnd",
77  'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
78  ], $fname
79  );
80  foreach ( $res as $row ) {
81  # Resolve stubs
82  $text = $row->old_text;
83  $id = $row->old_id;
84  if ( $row->old_flags === '' ) {
85  $flags = 'external';
86  } else {
87  $flags = "{$row->old_flags},external";
88  }
89 
90  if ( strpos( $flags, 'object' ) !== false ) {
91  $obj = unserialize( $text );
92  $className = strtolower( get_class( $obj ) );
93  if ( $className == 'historyblobstub' ) {
94  # resolveStub( $id, $row->old_text, $row->old_flags );
95  # $numStubs++;
96  continue;
97  } elseif ( $className == 'historyblobcurstub' ) {
98  $text = gzdeflate( $obj->getText() );
99  $flags = 'utf-8,gzip,external';
100  } elseif ( $className == 'concatenatedgziphistoryblob' ) {
101  // Do nothing
102  } else {
103  print "Warning: unrecognised object class \"$className\"\n";
104  continue;
105  }
106  } else {
107  $className = false;
108  }
109 
110  if ( strlen( $text ) < 100 && $className === false ) {
111  // Don't move tiny revisions
112  continue;
113  }
114 
115  # print "Storing " . strlen( $text ) . " bytes to $url\n";
116  # print "old_id=$id\n";
117 
118  $url = $extStore->store( $location, $text );
119  if ( !$url ) {
120  print "Error writing to external storage\n";
121  exit;
122  }
123  $dbw->update( 'text',
124  [ 'old_flags' => $flags, 'old_text' => $url ],
125  [ 'old_id' => $id ], $fname );
126  $numMoved++;
127  }
128  }
129 }
moveToExternal
if(!defined( 'MEDIAWIKI')) moveToExternal( $type, $location, $maxID, $minID=1)
Definition: moveToExternal.php:50
$optionsWithArgs
global $optionsWithArgs
Definition: commandLine.inc:24
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:146
$res
$res
Definition: testCompression.php:57
$dbr
$dbr
Definition: testCompression.php:54
REPORTING_INTERVAL
const REPORTING_INTERVAL
Definition: moveToExternal.php:26
wfGetDB
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
Definition: GlobalFunctions.php:2463
$args
if( $line===false) $args
Definition: mcc.php:124
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_MASTER
const DB_MASTER
Definition: defines.php:26
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
print
print
Definition: opensearch_desc.php:53
$type
$type
Definition: testCompression.php:52