DatabaseUpdater.php

Go to the documentation of this file.

<?php
use MediaWiki\MediaWikiServices;

require_once __DIR__ . '/../../maintenance/Maintenance.php';

abstract class DatabaseUpdater {
    protected static $updateCounter = 0;

    protected $updates = [];

    protected $updatesSkipped = [];

    protected $extensionUpdates = [];

    protected $db;

    protected $shared = false;

    protected $postDatabaseUpdateMaintenance = [
        DeleteDefaultMessages::class,
        PopulateRevisionLength::class,
        PopulateRevisionSha1::class,
        PopulateImageSha1::class,
        FixExtLinksProtocolRelative::class,
        PopulateFilearchiveSha1::class,
        PopulateBacklinkNamespace::class,
        FixDefaultJsonContentPages::class,
        CleanupEmptyCategories::class,
        AddRFCAndPMIDInterwiki::class,
    ];

    protected $fileHandle = null;

    protected $skipSchema = false;

    protected $holdContentHandlerUseDB = true;

    protected function __construct( Database &$db, $shared, Maintenance $maintenance = null ) {
        $this->db = $db;
        $this->db->setFlag( DBO_DDLMODE ); // For Oracle's handling of schema files
        $this->shared = $shared;
        if ( $maintenance ) {
            $this->maintenance = $maintenance;
            $this->fileHandle = $maintenance->fileHandle;
        } else {
            $this->maintenance = new FakeMaintenance;
        }
        $this->maintenance->setDB( $db );
        $this->initOldGlobals();
        $this->loadExtensions();
        Hooks::run( 'LoadExtensionSchemaUpdates', [ $this ] );
    }

    private function initOldGlobals() {
        global $wgExtNewTables, $wgExtNewFields, $wgExtPGNewFields,
            $wgExtPGAlteredFields, $wgExtNewIndexes, $wgExtModifiedFields;

        # For extensions only, should be populated via hooks
        # $wgDBtype should be checked to specifiy the proper file
        $wgExtNewTables = []; // table, dir
        $wgExtNewFields = []; // table, column, dir
        $wgExtPGNewFields = []; // table, column, column attributes; for PostgreSQL
        $wgExtPGAlteredFields = []; // table, column, new type, conversion method; for PostgreSQL
        $wgExtNewIndexes = []; // table, index, dir
        $wgExtModifiedFields = []; // table, index, dir
    }

    private function loadExtensions() {
        if ( !defined( 'MEDIAWIKI_INSTALL' ) ) {
            return; // already loaded
        }
        $vars = Installer::getExistingLocalSettings();

        $registry = ExtensionRegistry::getInstance();
        $queue = $registry->getQueue();
        // Don't accidentally load extensions in the future
        $registry->clearQueue();

        // This will automatically add "AutoloadClasses" to $wgAutoloadClasses
        $data = $registry->readFromQueue( $queue );
        $hooks = [ 'wgHooks' => [ 'LoadExtensionSchemaUpdates' => [] ] ];
        if ( isset( $data['globals']['wgHooks']['LoadExtensionSchemaUpdates'] ) ) {
            $hooks = $data['globals']['wgHooks']['LoadExtensionSchemaUpdates'];
        }
        if ( $vars && isset( $vars['wgHooks']['LoadExtensionSchemaUpdates'] ) ) {
            $hooks = array_merge_recursive( $hooks, $vars['wgHooks']['LoadExtensionSchemaUpdates'] );
        }
        global $wgHooks, $wgAutoloadClasses;
        $wgHooks['LoadExtensionSchemaUpdates'] = $hooks;
        if ( $vars && isset( $vars['wgAutoloadClasses'] ) ) {
            $wgAutoloadClasses += $vars['wgAutoloadClasses'];
        }
    }

    public static function newForDB( Database $db, $shared = false, $maintenance = null ) {
        $type = $db->getType();
        if ( in_array( $type, Installer::getDBTypes() ) ) {
            $class = ucfirst( $type ) . 'Updater';

            return new $class( $db, $shared, $maintenance );
        } else {
            throw new MWException( __METHOD__ . ' called for unsupported $wgDBtype' );
        }
    }

    public function getDB() {
        return $this->db;
    }

    public function output( $str ) {
        if ( $this->maintenance->isQuiet() ) {
            return;
        }
        global $wgCommandLineMode;
        if ( !$wgCommandLineMode ) {
            $str = htmlspecialchars( $str );
        }
        echo $str;
        flush();
    }

    public function addExtensionUpdate( array $update ) {
        $this->extensionUpdates[] = $update;
    }

    public function addExtensionTable( $tableName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'addTable', $tableName, $sqlPath, true ];
    }

    public function addExtensionIndex( $tableName, $indexName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'addIndex', $tableName, $indexName, $sqlPath, true ];
    }

    public function addExtensionField( $tableName, $columnName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'addField', $tableName, $columnName, $sqlPath, true ];
    }

    public function dropExtensionField( $tableName, $columnName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'dropField', $tableName, $columnName, $sqlPath, true ];
    }

    public function dropExtensionIndex( $tableName, $indexName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'dropIndex', $tableName, $indexName, $sqlPath, true ];
    }

    public function dropExtensionTable( $tableName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'dropTable', $tableName, $sqlPath, true ];
    }

    public function renameExtensionIndex( $tableName, $oldIndexName, $newIndexName,
        $sqlPath, $skipBothIndexExistWarning = false
    ) {
        $this->extensionUpdates[] = [
            'renameIndex',
            $tableName,
            $oldIndexName,
            $newIndexName,
            $skipBothIndexExistWarning,
            $sqlPath,
            true
        ];
    }

    public function modifyExtensionField( $tableName, $fieldName, $sqlPath ) {
        $this->extensionUpdates[] = [ 'modifyField', $tableName, $fieldName, $sqlPath, true ];
    }

    public function tableExists( $tableName ) {
        return ( $this->db->tableExists( $tableName, __METHOD__ ) );
    }

    public function addPostDatabaseUpdateMaintenance( $class ) {
        $this->postDatabaseUpdateMaintenance[] = $class;
    }

    protected function getExtensionUpdates() {
        return $this->extensionUpdates;
    }

    public function getPostDatabaseUpdateMaintenance() {
        return $this->postDatabaseUpdateMaintenance;
    }

    private function writeSchemaUpdateFile( $schemaUpdate = [] ) {
        $updates = $this->updatesSkipped;
        $this->updatesSkipped = [];

        foreach ( $updates as $funcList ) {
            $func = $funcList[0];
            $arg = $funcList[1];
            $origParams = $funcList[2];
            call_user_func_array( $func, $arg );
            flush();
            $this->updatesSkipped[] = $origParams;
        }
    }

    public function getSchemaVars() {
        return []; // DB-type specific
    }

    public function doUpdates( $what = [ 'core', 'extensions', 'stats' ] ) {
        global $wgVersion;

        $this->db->setSchemaVars( $this->getSchemaVars() );

        $what = array_flip( $what );
        $this->skipSchema = isset( $what['noschema'] ) || $this->fileHandle !== null;
        if ( isset( $what['core'] ) ) {
            $this->runUpdates( $this->getCoreUpdateList(), false );
        }
        if ( isset( $what['extensions'] ) ) {
            $this->runUpdates( $this->getOldGlobalUpdates(), false );
            $this->runUpdates( $this->getExtensionUpdates(), true );
        }

        if ( isset( $what['stats'] ) ) {
            $this->checkStats();
        }

        $this->setAppliedUpdates( $wgVersion, $this->updates );

        if ( $this->fileHandle ) {
            $this->skipSchema = false;
            $this->writeSchemaUpdateFile();
            $this->setAppliedUpdates( "$wgVersion-schema", $this->updatesSkipped );
        }
    }

    private function runUpdates( array $updates, $passSelf ) {
        $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();

        $updatesDone = [];
        $updatesSkipped = [];
        foreach ( $updates as $params ) {
            $origParams = $params;
            $func = array_shift( $params );
            if ( !is_array( $func ) && method_exists( $this, $func ) ) {
                $func = [ $this, $func ];
            } elseif ( $passSelf ) {
                array_unshift( $params, $this );
            }
            $ret = call_user_func_array( $func, $params );
            flush();
            if ( $ret !== false ) {
                $updatesDone[] = $origParams;
                $lbFactory->waitForReplication();
            } else {
                $updatesSkipped[] = [ $func, $params, $origParams ];
            }
        }
        $this->updatesSkipped = array_merge( $this->updatesSkipped, $updatesSkipped );
        $this->updates = array_merge( $this->updates, $updatesDone );
    }

    protected function setAppliedUpdates( $version, $updates = [] ) {
        $this->db->clearFlag( DBO_DDLMODE );
        if ( !$this->canUseNewUpdatelog() ) {
            return;
        }
        $key = "updatelist-$version-" . time() . self::$updateCounter;
        self::$updateCounter++;
        $this->db->insert( 'updatelog',
            [ 'ul_key' => $key, 'ul_value' => serialize( $updates ) ],
            __METHOD__ );
        $this->db->setFlag( DBO_DDLMODE );
    }

    public function updateRowExists( $key ) {
        $row = $this->db->selectRow(
            'updatelog',
            # Bug 65813
            '1 AS X',
            [ 'ul_key' => $key ],
            __METHOD__
        );

        return (bool)$row;
    }

    public function insertUpdateRow( $key, $val = null ) {
        $this->db->clearFlag( DBO_DDLMODE );
        $values = [ 'ul_key' => $key ];
        if ( $val && $this->canUseNewUpdatelog() ) {
            $values['ul_value'] = $val;
        }
        $this->db->insert( 'updatelog', $values, __METHOD__, 'IGNORE' );
        $this->db->setFlag( DBO_DDLMODE );
    }

    protected function canUseNewUpdatelog() {
        return $this->db->tableExists( 'updatelog', __METHOD__ ) &&
            $this->db->fieldExists( 'updatelog', 'ul_value', __METHOD__ );
    }

    protected function doTable( $name ) {
        global $wgSharedDB, $wgSharedTables;

        // Don't bother to check $wgSharedTables if there isn't a shared database
        // or the user actually also wants to do updates on the shared database.
        if ( $wgSharedDB === null || $this->shared ) {
            return true;
        }

        if ( in_array( $name, $wgSharedTables ) ) {
            $this->output( "...skipping update to shared table $name.\n" );
            return false;
        } else {
            return true;
        }
    }

    protected function getOldGlobalUpdates() {
        global $wgExtNewFields, $wgExtNewTables, $wgExtModifiedFields,
            $wgExtNewIndexes;

        $updates = [];

        foreach ( $wgExtNewTables as $tableRecord ) {
            $updates[] = [
                'addTable', $tableRecord[0], $tableRecord[1], true
            ];
        }

        foreach ( $wgExtNewFields as $fieldRecord ) {
            $updates[] = [
                'addField', $fieldRecord[0], $fieldRecord[1],
                $fieldRecord[2], true
            ];
        }

        foreach ( $wgExtNewIndexes as $fieldRecord ) {
            $updates[] = [
                'addIndex', $fieldRecord[0], $fieldRecord[1],
                $fieldRecord[2], true
            ];
        }

        foreach ( $wgExtModifiedFields as $fieldRecord ) {
            $updates[] = [
                'modifyField', $fieldRecord[0], $fieldRecord[1],
                $fieldRecord[2], true
            ];
        }

        return $updates;
    }

    abstract protected function getCoreUpdateList();

    public function copyFile( $filename ) {
        $this->db->sourceFile(
            $filename,
            null,
            null,
            __METHOD__,
            [ $this, 'appendLine' ]
        );
    }

    public function appendLine( $line ) {
        $line = rtrim( $line ) . ";\n";
        if ( fwrite( $this->fileHandle, $line ) === false ) {
            throw new MWException( "trouble writing file" );
        }

        return false;
    }

    protected function applyPatch( $path, $isFullPath = false, $msg = null ) {
        if ( $msg === null ) {
            $msg = "Applying $path patch";
        }
        if ( $this->skipSchema ) {
            $this->output( "...skipping schema change ($msg).\n" );

            return false;
        }

        $this->output( "$msg ..." );

        if ( !$isFullPath ) {
            $path = $this->patchPath( $this->db, $path );
        }
        if ( $this->fileHandle !== null ) {
            $this->copyFile( $path );
        } else {
            $this->db->sourceFile( $path );
        }
        $this->output( "done.\n" );

        return true;
    }

    public function patchPath( IDatabase $db, $patch ) {
        global $IP;

        $dbType = $db->getType();
        if ( file_exists( "$IP/maintenance/$dbType/archives/$patch" ) ) {
            return "$IP/maintenance/$dbType/archives/$patch";
        } else {
            return "$IP/maintenance/archives/$patch";
        }
    }

    protected function addTable( $name, $patch, $fullpath = false ) {
        if ( !$this->doTable( $name ) ) {
            return true;
        }

        if ( $this->db->tableExists( $name, __METHOD__ ) ) {
            $this->output( "...$name table already exists.\n" );
        } else {
            return $this->applyPatch( $patch, $fullpath, "Creating $name table" );
        }

        return true;
    }

    protected function addField( $table, $field, $patch, $fullpath = false ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        if ( !$this->db->tableExists( $table, __METHOD__ ) ) {
            $this->output( "...$table table does not exist, skipping new field patch.\n" );
        } elseif ( $this->db->fieldExists( $table, $field, __METHOD__ ) ) {
            $this->output( "...have $field field in $table table.\n" );
        } else {
            return $this->applyPatch( $patch, $fullpath, "Adding $field field to table $table" );
        }

        return true;
    }

    protected function addIndex( $table, $index, $patch, $fullpath = false ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        if ( !$this->db->tableExists( $table, __METHOD__ ) ) {
            $this->output( "...skipping: '$table' table doesn't exist yet.\n" );
        } elseif ( $this->db->indexExists( $table, $index, __METHOD__ ) ) {
            $this->output( "...index $index already set on $table table.\n" );
        } else {
            return $this->applyPatch( $patch, $fullpath, "Adding index $index to table $table" );
        }

        return true;
    }

    protected function dropField( $table, $field, $patch, $fullpath = false ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        if ( $this->db->fieldExists( $table, $field, __METHOD__ ) ) {
            return $this->applyPatch( $patch, $fullpath, "Table $table contains $field field. Dropping" );
        } else {
            $this->output( "...$table table does not contain $field field.\n" );
        }

        return true;
    }

    protected function dropIndex( $table, $index, $patch, $fullpath = false ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        if ( $this->db->indexExists( $table, $index, __METHOD__ ) ) {
            return $this->applyPatch( $patch, $fullpath, "Dropping $index index from table $table" );
        } else {
            $this->output( "...$index key doesn't exist.\n" );
        }

        return true;
    }

    protected function renameIndex( $table, $oldIndex, $newIndex,
        $skipBothIndexExistWarning, $patch, $fullpath = false
    ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        // First requirement: the table must exist
        if ( !$this->db->tableExists( $table, __METHOD__ ) ) {
            $this->output( "...skipping: '$table' table doesn't exist yet.\n" );

            return true;
        }

        // Second requirement: the new index must be missing
        if ( $this->db->indexExists( $table, $newIndex, __METHOD__ ) ) {
            $this->output( "...index $newIndex already set on $table table.\n" );
            if ( !$skipBothIndexExistWarning &&
                $this->db->indexExists( $table, $oldIndex, __METHOD__ )
            ) {
                $this->output( "...WARNING: $oldIndex still exists, despite it has " .
                    "been renamed into $newIndex (which also exists).\n" .
                    "            $oldIndex should be manually removed if not needed anymore.\n" );
            }

            return true;
        }

        // Third requirement: the old index must exist
        if ( !$this->db->indexExists( $table, $oldIndex, __METHOD__ ) ) {
            $this->output( "...skipping: index $oldIndex doesn't exist.\n" );

            return true;
        }

        // Requirements have been satisfied, patch can be applied
        return $this->applyPatch(
            $patch,
            $fullpath,
            "Renaming index $oldIndex into $newIndex to table $table"
        );
    }

    public function dropTable( $table, $patch = false, $fullpath = false ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        if ( $this->db->tableExists( $table, __METHOD__ ) ) {
            $msg = "Dropping table $table";

            if ( $patch === false ) {
                $this->output( "$msg ..." );
                $this->db->dropTable( $table, __METHOD__ );
                $this->output( "done.\n" );
            } else {
                return $this->applyPatch( $patch, $fullpath, $msg );
            }
        } else {
            $this->output( "...$table doesn't exist.\n" );
        }

        return true;
    }

    public function modifyField( $table, $field, $patch, $fullpath = false ) {
        if ( !$this->doTable( $table ) ) {
            return true;
        }

        $updateKey = "$table-$field-$patch";
        if ( !$this->db->tableExists( $table, __METHOD__ ) ) {
            $this->output( "...$table table does not exist, skipping modify field patch.\n" );
        } elseif ( !$this->db->fieldExists( $table, $field, __METHOD__ ) ) {
            $this->output( "...$field field does not exist in $table table, " .
                "skipping modify field patch.\n" );
        } elseif ( $this->updateRowExists( $updateKey ) ) {
            $this->output( "...$field in table $table already modified by patch $patch.\n" );
        } else {
            $this->insertUpdateRow( $updateKey );

            return $this->applyPatch( $patch, $fullpath, "Modifying $field field of table $table" );
        }

        return true;
    }

    public function setFileAccess() {
        $repo = RepoGroup::singleton()->getLocalRepo();
        $zonePath = $repo->getZonePath( 'temp' );
        if ( $repo->getBackend()->directoryExists( [ 'dir' => $zonePath ] ) ) {
            // If the directory was never made, then it will have the right ACLs when it is made
            $status = $repo->getBackend()->secure( [
                'dir' => $zonePath,
                'noAccess' => true,
                'noListing' => true
            ] );
            if ( $status->isOK() ) {
                $this->output( "Set the local repo temp zone container to be private.\n" );
            } else {
                $this->output( "Failed to set the local repo temp zone container to be private.\n" );
            }
        }
    }

    public function purgeCache() {
        global $wgLocalisationCacheConf;
        # We can't guarantee that the user will be able to use TRUNCATE,
        # but we know that DELETE is available to us
        $this->output( "Purging caches..." );
        $this->db->delete( 'objectcache', '*', __METHOD__ );
        if ( $wgLocalisationCacheConf['manualRecache'] ) {
            $this->rebuildLocalisationCache();
        }
        $blobStore = new MessageBlobStore();
        $blobStore->clear();
        $this->db->delete( 'module_deps', '*', __METHOD__ );
        $this->output( "done.\n" );
    }

    protected function checkStats() {
        $this->output( "...site_stats is populated..." );
        $row = $this->db->selectRow( 'site_stats', '*', [ 'ss_row_id' => 1 ], __METHOD__ );
        if ( $row === false ) {
            $this->output( "data is missing! rebuilding...\n" );
        } elseif ( isset( $row->site_stats ) && $row->ss_total_pages == -1 ) {
            $this->output( "missing ss_total_pages, rebuilding...\n" );
        } else {
            $this->output( "done.\n" );

            return;
        }
        SiteStatsInit::doAllAndCommit( $this->db );
    }

    # Common updater functions

    protected function doActiveUsersInit() {
        $activeUsers = $this->db->selectField( 'site_stats', 'ss_active_users', false, __METHOD__ );
        if ( $activeUsers == -1 ) {
            $activeUsers = $this->db->selectField( 'recentchanges',
                'COUNT( DISTINCT rc_user_text )',
                [ 'rc_user != 0', 'rc_bot' => 0, "rc_log_type != 'newusers'" ], __METHOD__
            );
            $this->db->update( 'site_stats',
                [ 'ss_active_users' => intval( $activeUsers ) ],
                [ 'ss_row_id' => 1 ], __METHOD__, [ 'LIMIT' => 1 ]
            );
        }
        $this->output( "...ss_active_users user count set...\n" );
    }

    protected function doLogUsertextPopulation() {
        if ( !$this->updateRowExists( 'populate log_usertext' ) ) {
            $this->output(
                "Populating log_user_text field, printing progress markers. For large\n" .
                "databases, you may want to hit Ctrl-C and do this manually with\n" .
                "maintenance/populateLogUsertext.php.\n"
            );

            $task = $this->maintenance->runChild( 'PopulateLogUsertext' );
            $task->execute();
            $this->output( "done.\n" );
        }
    }

    protected function doLogSearchPopulation() {
        if ( !$this->updateRowExists( 'populate log_search' ) ) {
            $this->output(
                "Populating log_search table, printing progress markers. For large\n" .
                "databases, you may want to hit Ctrl-C and do this manually with\n" .
                "maintenance/populateLogSearch.php.\n" );

            $task = $this->maintenance->runChild( 'PopulateLogSearch' );
            $task->execute();
            $this->output( "done.\n" );
        }
    }

    protected function doUpdateTranscacheField() {
        if ( $this->updateRowExists( 'convert transcache field' ) ) {
            $this->output( "...transcache tc_time already converted.\n" );

            return true;
        }

        return $this->applyPatch( 'patch-tc-timestamp.sql', false,
            "Converting tc_time from UNIX epoch to MediaWiki timestamp" );
    }

    protected function doCollationUpdate() {
        global $wgCategoryCollation;
        if ( $this->db->fieldExists( 'categorylinks', 'cl_collation', __METHOD__ ) ) {
            if ( $this->db->selectField(
                'categorylinks',
                'COUNT(*)',
                'cl_collation != ' . $this->db->addQuotes( $wgCategoryCollation ),
                __METHOD__
                ) == 0
            ) {
                $this->output( "...collations up-to-date.\n" );

                return;
            }

            $this->output( "Updating category collations..." );
            $task = $this->maintenance->runChild( 'UpdateCollation' );
            $task->execute();
            $this->output( "...done.\n" );
        }
    }

    protected function doMigrateUserOptions() {
        if ( $this->db->tableExists( 'user_properties' ) ) {
            $cl = $this->maintenance->runChild( 'ConvertUserOptions', 'convertUserOptions.php' );
            $cl->execute();
            $this->output( "done.\n" );
        }
    }

    protected function doEnableProfiling() {
        global $wgProfiler;

        if ( !$this->doTable( 'profiling' ) ) {
            return;
        }

        $profileToDb = false;
        if ( isset( $wgProfiler['output'] ) ) {
            $out = $wgProfiler['output'];
            if ( $out === 'db' ) {
                $profileToDb = true;
            } elseif ( is_array( $out ) && in_array( 'db', $out ) ) {
                $profileToDb = true;
            }
        }

        if ( $profileToDb && !$this->db->tableExists( 'profiling', __METHOD__ ) ) {
            $this->applyPatch( 'patch-profiling.sql', false, 'Add profiling table' );
        }
    }

    protected function rebuildLocalisationCache() {
        $cl = $this->maintenance->runChild( 'RebuildLocalisationCache', 'rebuildLocalisationCache.php' );
        $this->output( "Rebuilding localisation cache...\n" );
        $cl->setForce();
        $cl->execute();
        $this->output( "done.\n" );
    }

    protected function disableContentHandlerUseDB() {
        global $wgContentHandlerUseDB;

        if ( $wgContentHandlerUseDB ) {
            $this->output( "Turning off Content Handler DB fields for this part of upgrade.\n" );
            $this->holdContentHandlerUseDB = $wgContentHandlerUseDB;
            $wgContentHandlerUseDB = false;
        }
    }

    protected function enableContentHandlerUseDB() {
        global $wgContentHandlerUseDB;

        if ( $this->holdContentHandlerUseDB ) {
            $this->output( "Content Handler DB fields should be usable now.\n" );
            $wgContentHandlerUseDB = $this->holdContentHandlerUseDB;
        }
    }
}