mwbot/

lib.rs

/*
Copyright (C) 2021 Kunal Mehta <legoktm@debian.org>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

//! A MediaWiki bot and tool framework
//!
//! `mwbot` aims to provide a batteries-included framework for building bots
//! and tools for MediaWiki wikis. It builds on top of the [mwapi](https://docs.rs/mwapi)
//! and [parsoid](https://docs.rs/parsoid) crates, which offer lower-level APIs.
//!
//! ## Quickstart
//! ### Configuration
//! Create a `~/.config/mwbot.toml` file with the following structure:
//! ```toml
//! wiki_url = "https://en.wikipedia.org/w/"
//! ```
//!
//! If want to authenticate, add an auth section:
//! ```toml
//! [auth]
//! username = "Example"
//! oauth2_token = "[...]"
//! ```
//! See [the OAuth documentation](https://www.mediawiki.org/wiki/OAuth/For_Developers#OAuth_2)
//! for how to get an OAuth 2 token. Using an [owner-only consumer](https://www.mediawiki.org/wiki/OAuth/Owner-only_consumers#OAuth_2)
//! is the easiest way to do so.
//!
//! <div class="warning">
//!
//! On UNIX-like systems, for security, you must make sure the config's
//! file permissions don't permit reading by other users, or authorization
//! will fail with a [`ReadableConfig`][ConfigError::ReadableConfig] error. In this case,
//! `chmod 600 ~/.config/mwbot.toml` can be used to correct the permissions and
//! make sure the file is only readable and writeable to its owner.
//!
//! </div>
//!
//! ### Reading a page
//! ```
//! # async fn demo() -> mwbot::Result<()> {
//! # use parsoid::prelude::*;
//! let bot = mwbot::Bot::from_default_config().await.unwrap();
//! let page = bot.page("Rust (programming language)")?;
//! let html = page.html().await?.into_mutable();
//! // The lead section is the second p tag in the first section
//! let lead = html.select("section > p")[1].text_contents();
//! assert!(lead.starts_with("Rust is a multi-paradigm, general-purpose programming language"));
//! # Ok(())
//! # }
//! ```
//! Using [`Bot::from_default_config()`] will look in the current directory
//! for `mwbot.toml` before looking in the user's config directory. A
//! custom path can be specified by using `Bot::from_config(...)`.
//!
//! ### Editing a page
//! ```
//! # async fn demo() -> mwbot::Result<()> {
//! # use mwbot::SaveOptions;
//! let bot = mwbot::Bot::from_default_config().await.unwrap();
//! let page = bot.page("Project:Sandbox")?;
//! let wikitext = "This is a test edit!";
//! page.save(wikitext, &SaveOptions::summary("test edit!")).await?;
//! # Ok(())
//! # }
//! ```
//! `Page.save()` accepts both HTML and wikitext and supports the [`{{nobots}}`](https://en.wikipedia.org/wiki/Template:Bots)
//! exclusion mechanism, among other features.
//!
//! ### Next steps
//! Try using one of the offered [page generators](./generators/index.html) to fetch and operate on
//! multiple pages.
//!
//! ## Contributing
//! `mwbot` is the flagship crate of the [`mwbot-rs` project](https://www.mediawiki.org/wiki/Mwbot-rs).
//! We're always looking for new contributors, please [reach out](https://www.mediawiki.org/wiki/Mwbot-rs#Contributing)
//! if you're interested!
#![cfg_attr(docsrs, feature(doc_cfg))]

mod builder;
mod config;
mod edit;
mod error;
#[cfg(feature = "upload")]
#[cfg_attr(docsrs, doc(cfg(feature = "upload")))]
pub mod file;
#[cfg(feature = "generators")]
#[cfg_attr(docsrs, doc(cfg(feature = "generators")))]
pub mod generators;
mod logging;
mod page;
mod siteinfo;
#[cfg(feature = "sitematrix")]
#[cfg_attr(docsrs, doc(cfg(feature = "sitematrix")))]
mod sitematrix;
#[cfg(feature = "upload")]
#[cfg_attr(docsrs, doc(cfg(feature = "upload")))]
pub mod upload;
mod utils;

pub use error::{config::ConfigError, Error};
use fs_mistrust::Mistrust;
pub use mwapi::Client as ApiClient;
use mwapi::{Builder as ApiBuilder, ErrorFormat};
pub use mwtimestamp as timestamp;
pub use mwtitle::{Namespace, Title, TitleCodec};
use once_cell::sync::OnceCell;
pub use parsoid;
use std::{path::Path, sync::Arc};
#[cfg(feature = "sitematrix")]
use tokio::sync::OnceCell as AsyncOnceCell;
use tokio::{sync::Mutex, time};
use tracing::{debug, info};
pub type Result<T, E = Error> = std::result::Result<T, E>;
pub use builder::Builder;
pub use edit::SaveOptions;
pub use logging::init as init_logging;
pub use page::Page;

use parsoid::prelude::*;

/// Main bot class
#[derive(Clone, Debug)]
pub struct Bot {
    api: ApiClient,
    parsoid: ParsoidClient,
    state: BotState,
    config: Arc<BotConfig>,
}

/// Static, read-only settings
#[derive(Clone, Debug)]
struct BotConfig {
    // TODO: figure out something better than this
    username: Option<String>,
    // TODO: is there a better way to track this?
    using_botpassword: bool,
    siteinfo: siteinfo::SiteInfo,
    codec: TitleCodec,
    #[cfg(feature = "sitematrix")]
    sitematrix: AsyncOnceCell<sitematrix::SiteMapping>,
    /// Whether edits should be marked as bot (default: true)
    mark_as_bot: bool,
    /// Whether to respect {{nobots}} (default: true)
    respect_nobots: bool,
}

/// Dynamic state
#[derive(Clone, Debug)]
struct BotState {
    save_timer: Option<Arc<Mutex<time::Interval>>>,
}

impl Bot {
    /// Build a `Bot` instance programmatically.
    ///
    /// The wiki's URL should be the path where api.php and rest.php can be
    /// found (aka the "script path"). By default it is usually something
    /// like `https://wiki.example.org/w/`.
    pub fn builder(wiki_url: String) -> Builder {
        Builder::new(wiki_url)
    }

    /// For older wikis (pre-1.42) that use RESTBase or the old Parsoid rest.php API.
    pub fn builder_with_legacy_rest(
        api_url: String,
        rest_url: String,
    ) -> Builder {
        Builder::new_with_api_url(api_url, rest_url)
    }

    /// Load Bot configuration from a default location, first look at
    /// `mwbot.toml` in the current directory, otherwise look in the
    /// platform's config directory:
    ///
    /// * Linux: `$XDG_CONFIG_HOME` or `$HOME/.config`
    /// * macOS: `$HOME/Library/Application Support`
    /// * Windows: `{FOLDERID_RoamingAppData}`
    ///
    /// Under UNIX-like systems, the configuration file must only be readable by its owner!
    pub async fn from_default_config() -> Result<Self, ConfigError> {
        let path = {
            let first = Path::new("mwbot.toml");
            if first.exists() {
                first.to_path_buf()
            } else {
                dirs::config_dir()
                    .expect("Cannot find config directory")
                    .join("mwbot.toml")
            }
        };
        Self::from_path(&path).await
    }

    /// Load Bot configuration from the specified path.
    ///
    /// Under UNIX-like systems, the configuration file must only be readable by its owner!
    pub async fn from_path(path: &Path) -> Result<Self, ConfigError> {
        debug!("Reading config from {:?}", path);
        let config: config::Config =
            toml::from_str(&std::fs::read_to_string(path)?)?;
        // Check file permissions if there are credentials
        if config.auth.is_some() {
            check_file_permissions(path)?;
        }
        Self::from_config(config).await
    }

    async fn from_config(config: config::Config) -> Result<Self, ConfigError> {
        let mut api = ApiClient::builder(&config.api_url()?)
            .set_maxlag(config.general.maxlag.unwrap_or(5))
            .set_errorformat(ErrorFormat::Wikitext);
        if let Some(limit) = config.general.retry_limit {
            api = api.set_retry_limit(limit);
        }
        let mut user_agent = vec![];
        if let Some(extra) = &config.general.user_agent {
            user_agent.push(extra.to_string());
        }
        // FIXME: do better
        let mut username = None;
        if let Some(auth) = &config.auth {
            match auth {
                config::Auth::BotPassword { username, password } => {
                    info!("Logging in as {} with password", username);
                    api = api.set_botpassword(username, password);
                }
                config::Auth::OAuth2 {
                    username,
                    oauth2_token,
                } => {
                    info!("Logging in as {} with OAuth2 token", username);
                    api = api.set_oauth2_token(oauth2_token);
                }
            }
            let normalized = normalize_username(auth.username());
            user_agent.push(format!("User:{normalized}"));
            username = Some(normalized);
        }
        user_agent.push(format!("mwbot-rs/{}", env!("CARGO_PKG_VERSION")));
        let user_agent = user_agent.join(" ");
        let save_delay = config.edit.save_delay.unwrap_or(10);
        let save_timer = if save_delay > 0 {
            let mut interval =
                time::interval(time::Duration::from_secs(save_delay));
            interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay);
            Some(Arc::new(Mutex::new(interval)))
        } else {
            None
        };
        let api = api
            .set_user_agent(&user_agent)
            .build()
            .await
            .map_err(Error::from)?;
        let (site_info, title_site_info) = siteinfo::get_siteinfo(&api).await?;
        let http = api.http_client().clone();
        Ok(Self {
            api,
            parsoid: ParsoidClient::new_with_client(&config.rest_url()?, http)?,
            config: Arc::new(BotConfig {
                username,
                using_botpassword: matches!(
                    config.auth,
                    Some(config::Auth::BotPassword { .. })
                ),
                siteinfo: site_info,
                codec: TitleCodec::from_site_info(title_site_info)
                    .map_err(Error::from)?,
                #[cfg(feature = "sitematrix")]
                sitematrix: AsyncOnceCell::new(),
                mark_as_bot: config.edit.mark_as_bot.unwrap_or(true),
                respect_nobots: config.edit.respect_nobots.unwrap_or(true),
            }),
            state: BotState { save_timer },
        })
    }

    /// Clone this bot for another wiki in the same family.
    ///
    /// This allows you to use the `Bot` to connect to another wiki in
    /// the same family. For example, if you have a bot for `en.wikipedia.org`,
    /// you can clone it to use with `de.wikipedia.org`.
    ///
    /// Behind the scenes, this will copy over the username, authentication info,
    /// as well as internal HTTP connection pools and save state. Rate limits
    /// like edit speed will apply across both instances.
    pub async fn clone_for_family(
        &self,
        wiki_url: &str,
    ) -> Result<Self, ConfigError> {
        if self.config.using_botpassword {
            return Err(ConfigError::CannotCloneBotPassword);
        }
        let api = ApiBuilder::from_client_with_url(
            &self.api,
            &format!("{wiki_url}api.php"),
        )
        .build()
        .await
        .map_err(Error::from)?;
        let parsoid = ParsoidClient::new_with_client(
            &format!("{wiki_url}rest.php"),
            self.api.http_client().clone(),
        )?;
        let (site_info, title_site_info) = siteinfo::get_siteinfo(&api).await?;

        Ok(Self {
            api,
            parsoid,
            config: Arc::new(BotConfig {
                username: self.config.username.clone(),
                using_botpassword: false,
                siteinfo: site_info,
                codec: TitleCodec::from_site_info(title_site_info)
                    .map_err(Error::from)?,
                #[cfg(feature = "sitematrix")]
                sitematrix: self.config.sitematrix.clone(),
                mark_as_bot: self.config.mark_as_bot,
                respect_nobots: self.config.respect_nobots,
            }),
            state: self.state.clone(),
        })
    }

    #[cfg(feature = "sitematrix")]
    pub async fn clone_for_family_via_dbname(
        &self,
        dbname: &str,
    ) -> Result<Self, ConfigError> {
        let mapping = self
            .config
            .sitematrix
            .get_or_try_init(|| async { sitematrix::fetch(&self.api).await })
            .await?;
        let new_url = match mapping.get(dbname) {
            Some(new_url) => new_url,
            None => Err(ConfigError::InvalidSitename(dbname.to_string()))?,
        };
        self.clone_for_family(&format!(
            "{new_url}{}/",
            self.config.siteinfo.general.script_path
        ))
        .await
    }

    /// Get a reference to the underlying [`mwapi::Client`](https://docs.rs/mwapi/latest/mwapi/struct.Client.html)
    /// to make arbitrary API requests
    pub fn api(&self) -> &ApiClient {
        &self.api
    }

    /// Get a reference to the underlying [`parsoid::Client`](https://docs.rs/parsoid/latest/parsoid/struct.Client.html)
    /// to make arbitrary Parsoid API requests
    pub fn parsoid(&self) -> &ParsoidClient {
        &self.parsoid
    }

    /// Get a reference to the internal [`mwtitle::TitleCodec`](https://docs.rs/mwtitle/latest/mwtitle/struct.TitleCodec.html)
    /// to parse and construct arbitrary titles.
    pub fn title_codec(&self) -> &TitleCodec {
        &self.config.codec
    }

    /// Get a `Page` on this wiki. The specified title should be a full title,
    /// including namespace. This does some validation on the
    /// provided input.
    pub fn page(&self, title: &str) -> Result<Page> {
        let title = self.config.codec.new_title(title)?.remove_fragment();
        self.page_from_title(title)
    }

    /// Get a `Page` on this wiki using information from a database row.
    ///
    /// The provided input will be validated and may return an
    /// error if invalid.
    pub fn page_from_database(
        &self,
        namespace: i32,
        dbkey: &str,
    ) -> Result<Page> {
        let title = self
            .config
            .codec
            .new_title_from_database(namespace, dbkey)?
            .remove_fragment();
        self.page_from_title(title)
    }

    /// Get a `Page` for a given [`Title`](https://docs.rs/mwtitle/latest/mwtitle/struct.Title.html)
    pub fn page_from_title(&self, title: Title) -> Result<Page> {
        if !title.is_local_page() {
            return Err(Error::InvalidPage);
        }
        Ok(Page {
            bot: self.clone(),
            title,
            title_text: Default::default(),
            info: Default::default(),
            baserevid: Default::default(),
        })
    }

    /// Get a `Page` on this wiki using page id.
    ///
    /// The provided input will be validated and may return an
    /// error if invalid.
    pub async fn page_from_id(&self, page_id: u64) -> Result<Page> {
        let mut resp: page::InfoResponse = mwapi_responses::query_api(
            &self.api,
            [("pageids", page_id.to_string())],
        )
        .await?;
        let info = resp
            .query
            .pages
            .pop()
            .expect("API response returned 0 pages");
        let title = self.config.codec.new_title(&info.title)?.remove_fragment();
        if !title.is_local_page() {
            return Err(Error::InvalidPage);
        }
        Ok(Page {
            bot: self.clone(),
            title,
            title_text: Default::default(),
            info: Arc::new(info.clone().into()),
            baserevid: info
                .lastrevid
                .map_or_else(OnceCell::new, OnceCell::with_value),
        })
    }

    /// Get the site's server name, which is usually the domain name,
    /// e.g. `"en.wikipedia.org"`
    pub fn server_name(&self) -> &str {
        &self.config.siteinfo.general.server_name
    }

    /// Get the site's "wikiid", which is usually the internal database name,
    /// e.g. `"enwiki"`
    pub fn wiki_id(&self) -> &str {
        &self.config.siteinfo.general.wiki_id
    }

    /// Get the version of MediaWiki that the site is using, e.g. `"1.45.0"`
    pub fn mediawiki_version(&self) -> &str {
        &self.config.siteinfo.general.version
    }

    /// Get the numeric id of a [`Namespace`](https://docs.rs/mwtitle/latest/mwtitle/struct.Namespace.html).
    pub fn namespace_id<'a, N: Into<Namespace<'a>>>(
        &self,
        namespace: N,
    ) -> Option<i32> {
        self.title_codec().namespace_map().get_id(namespace)
    }

    /// Get the name of a [`Namespace`](https://docs.rs/mwtitle/latest/mwtitle/struct.Namespace.html).
    pub fn namespace_name<'a, N: Into<Namespace<'a>>>(
        &self,
        namespace: N,
    ) -> Option<&str> {
        self.title_codec().namespace_map().get_name(namespace)
    }
}

/// Verify file permissions are not obviously misconfigured:
/// * If the file is owned by the current user, it should only be readable to
///   that user
/// * If the file is owned by another user, it should not be world readable
///
/// TODO: support [extended ACLs](https://wiki.archlinux.org/title/Access_Control_Lists)
fn check_file_permissions(path: &Path) -> Result<(), ConfigError> {
    Ok(Mistrust::new()
        .verifier()
        .require_file()
        .all_errors()
        .check(path)?)
}

// TODO: Can we reuse TitleCodec here?
fn normalize_username(original: &str) -> String {
    // MW normalization, underscores to spaces
    let name = original.replace('_', " ");
    // If it's a bot password, strip the @<name> part
    match name.split_once('@') {
        Some((name, _)) => name.to_string(),
        None => name,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    pub(crate) fn is_authenticated() -> bool {
        std::env::var("MWAPI_TOKEN").is_ok()
    }

    pub(crate) async fn testwp() -> Bot {
        let username = std::env::var("MWAPI_USERNAME");
        let token = std::env::var("MWAPI_TOKEN");
        let auth = if let (Ok(username), Ok(oauth2_token)) = (username, token) {
            Some(config::Auth::OAuth2 {
                username,
                oauth2_token,
            })
        } else {
            None
        };

        Bot::from_config(config::Config {
            wiki_url: Some("https://test.wikipedia.org/w/".to_string()),
            api_url: None,
            rest_url: None,
            auth,
            general: Default::default(),
            edit: Default::default(),
        })
        .await
        .unwrap()
    }

    #[cfg(feature = "wikibase")]
    pub(crate) async fn testwikidata() -> Bot {
        let username = std::env::var("MWAPI_USERNAME");
        let token = std::env::var("MWAPI_TOKEN");
        let auth = if let (Ok(username), Ok(oauth2_token)) = (username, token) {
            Some(config::Auth::OAuth2 {
                username,
                oauth2_token,
            })
        } else {
            None
        };

        Bot::from_config(config::Config {
            wiki_url: Some("https://test.wikidata.org/w/".to_string()),
            api_url: None,
            rest_url: None,
            auth,
            general: Default::default(),
            edit: Default::default(),
        })
        .await
        .unwrap()
    }

    // TODO: turn this into a proper public API; for now it's just for tests
    pub(crate) async fn has_userright(bot: &Bot, right: &str) -> bool {
        let resp = bot
            .api()
            .get_value(&[
                ("action", "query"),
                ("meta", "userinfo"),
                ("uiprop", "rights"),
            ])
            .await
            .unwrap();
        resp["query"]["userinfo"]["rights"]
            .as_array()
            .unwrap()
            .iter()
            .any(|r| r == right)
    }

    fn assert_send_sync<T: Send + Sync>() {}

    /// Assert all these types are Send + Sync
    #[test]
    fn test_send_sync() {
        assert_send_sync::<Bot>();
        assert_send_sync::<Page>();
    }

    #[test]
    fn test_normalize_username() {
        assert_eq!(&normalize_username("Foo"), "Foo");
        assert_eq!(&normalize_username("Foo_bar"), "Foo bar");
        assert_eq!(&normalize_username("Foo@bar"), "Foo");
    }

    #[tokio::test]
    async fn test_get_api() {
        let bot = testwp().await;
        // No errors
        bot.api().get_value(&[("action", "query")]).await.unwrap();
    }

    #[tokio::test]
    async fn test_page() {
        let bot = testwp().await;
        let page = bot.page("Example").unwrap();
        assert_eq!(page.title(), "Example");
        assert_eq!(page.namespace(), 0);
        let page = bot.page_from_database(1, "Example").unwrap();
        assert_eq!(page.title(), "Talk:Example");
        assert_eq!(page.namespace(), 1);
        let error = bot.page("mw:External").unwrap_err();
        assert!(matches!(error, Error::InvalidPage));
    }

    #[tokio::test]
    async fn test_page_from_id() {
        let bot = testwp().await;
        let page = bot.page_from_id(122863).await.unwrap();
        assert_eq!(page.title(), "Mwbot-rs");
        assert_eq!(page.namespace(), 0);
        let page = bot.page_from_id(153569).await.unwrap();
        assert_eq!(page.title(), "Talk:Mwbot-rs");
        assert_eq!(page.namespace(), 1);
        let invalid = bot.page_from_id(0).await.unwrap_err();
        dbg!(&invalid);
        // This is the wrong error type, but at least it fails. See T354590.
        assert!(matches!(invalid, Error::InvalidJson(_)));
    }

    #[tokio::test]
    #[ignore] // flaky, see T341906
    async fn test_user_agent() {
        let bot = testwp().await;
        let version = env!("CARGO_PKG_VERSION");
        let resp: serde_json::Value = bot
            .api()
            .http_client()
            .get("https://httpbin.org/user-agent")
            .send()
            .await
            .unwrap()
            .json()
            .await
            .unwrap();
        let user_agent = resp["user-agent"].as_str().unwrap();
        match &bot.config.username {
            Some(username) => {
                assert_eq!(
                    user_agent,
                    &format!("User:{username} mwbot-rs/{version}")
                );
            }
            None => {
                assert_eq!(user_agent, &format!("mwbot-rs/{version}"));
            }
        }
    }

    /// T333423 regression test
    #[tokio::test]
    async fn test_zero_save_delay() {
        Bot::builder("https://test.wikipedia.org/w/".to_string())
            .set_save_delay(0)
            .build()
            .await
            .unwrap();
    }

    #[tokio::test]
    async fn test_siteinfo() {
        let bot = testwp().await;
        assert_eq!(bot.server_name(), "test.wikipedia.org");
        assert_eq!(bot.wiki_id(), "testwiki");
        // testwp should always be on a WMF branch
        assert!(bot.mediawiki_version().contains("0-wmf."));
    }

    #[tokio::test]
    async fn test_namespace_id() {
        let bot = testwp().await;
        assert_eq!(bot.namespace_id(""), Some(0));
        assert_eq!(bot.namespace_id("Wikipedia"), Some(4));
        assert_eq!(bot.namespace_id("MediaWiki"), Some(8));
        assert_eq!(bot.namespace_id("Special"), Some(-1));
        assert_eq!(bot.namespace_id("NamespaceNotExist"), None);
        assert_eq!(bot.namespace_id(-12345), None);
        assert_eq!(bot.namespace_id(-1), Some(-1));
    }

    #[tokio::test]
    async fn test_namespace_name() {
        let bot = testwp().await;
        assert_eq!(bot.namespace_name(""), Some(""));
        assert_eq!(bot.namespace_name("Wikipedia"), Some("Wikipedia"));
        assert_eq!(bot.namespace_name("Special"), Some("Special"));
        assert_eq!(bot.namespace_name("NamespaceNotExist"), None);
        assert_eq!(bot.namespace_name(-12345), None);
        assert_eq!(bot.namespace_name(-1), Some("Special"));
        assert_eq!(bot.namespace_name(0), Some(""));
        assert_eq!(bot.namespace_name(1), Some("Talk"));
        assert_eq!(bot.namespace_name(4), Some("Wikipedia"));
    }
}