parsoid/

api.rs

/*
Copyright (C) 2020-2021 Kunal Mehta <legoktm@debian.org>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

use crate::immutable::ImmutableWikicode;
use crate::private::Sealed;
use crate::{Error, Result, Wikicode};
use lazy_static::lazy_static;
use reqwest::header::HeaderMap;
pub use reqwest::Client as HttpClient;
use reqwest::{header, Response};
use std::fmt::Write as _;
use std::sync::Arc;
use tokio::sync::Semaphore;
use tracing::debug;
#[cfg(feature = "restbase")]
use tracing::warn;
use urlencoding::encode;

/// Version of library embedded in user-agent
const VERSION: &str = env!("CARGO_PKG_VERSION");
/// `Accept` header for [content negotiation](https://www.mediawiki.org/wiki/Parsoid/API#Content_Negotiation)
const ACCEPT_2_8_0: &str = "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/2.8.0\"";

/// Allows transform_to_wikitext to be used by both Wikicode and
/// ImmutableWikicode
pub trait APICode: Sealed {
    fn html(&self) -> String;
    fn title(&self) -> Option<String>;
    fn etag(&self) -> Option<&str>;
    fn revid(&self) -> Option<u64>;
}

impl APICode for ImmutableWikicode {
    fn html(&self) -> String {
        self.html().to_string()
    }

    fn title(&self) -> Option<String> {
        self.title.clone()
    }

    fn etag(&self) -> Option<&str> {
        self.etag()
    }

    fn revid(&self) -> Option<u64> {
        self.revision_id()
    }
}

impl APICode for Wikicode {
    fn html(&self) -> String {
        self.to_string()
    }

    fn title(&self) -> Option<String> {
        self.title()
    }

    fn etag(&self) -> Option<&str> {
        self.etag.as_deref()
    }

    fn revid(&self) -> Option<u64> {
        self.revision_id()
    }
}

/// HTTP client to get Parsoid HTML from MediaWiki's Rest APIs
///
/// Note: This requires the `http` feature is enabled (it is by default).
#[derive(Clone, Debug)]
pub struct Client {
    http: HttpClient,
    base_url: String,
    endpoint_kind: ParsoidEndpointKind,
    semaphore: Arc<Semaphore>,
}

/// The kind of Parsoid endpoints
#[derive(Clone, Debug, Copy, PartialEq, Eq)]
pub enum ParsoidEndpointKind {
    /// Provided by MediaWiki Core. New in MW 1.42.
    /// See WMF gerrit change [972322](https://gerrit.wikimedia.org/r/c/mediawiki/core/+/972322).
    Core,
    /// Provided by [the Parsoid extension](https://gerrit.wikimedia.org/g/mediawiki/services/parsoid/).
    ParsoidExtension,
    #[cfg(feature = "restbase")]
    /// Provided by RESTBase. Only available on WMF projects.
    /// The RESTBase endpoint has been deprecated and should be replaced.
    ///
    /// This is only supported when `restbase` feature is enabled (which is enabled by default).
    /// If the `restbase` feature is disabled, RESTBase URLs will be transformed to the Parsoid extension URL.
    RESTBase,
}

impl Client {
    /// Create a new Client. `base_url` should either point to `rest.php` or
    /// RESTBase API.
    ///
    /// For wikis running MediaWiki 1.42 or newer (or Wikimedia projects),
    /// it could be a core REST API like: `https://wiki.example.org/w/rest.php`
    ///
    /// For wikis with Parsoid extension installed, it might be like:
    /// `https://wiki.example.org/w/rest.php/wiki.example.org/v3`.
    /// This kind of endpoint is not available
    /// on WMF projects (except for Parsoid cluster).
    ///
    /// For Wikimedia projects, it might also be a RESTBase API like:
    /// `https://en.wikipedia.org/api/rest_v1`.
    /// The RESTBase endpoint has been deprecated and should be replaced with
    /// MediaWiki REST API (the `rest.php` endpoint).
    ///
    /// By default, the `restbase` feature is enabled.
    /// If it is disabled, Client will always try to transform
    /// RESTBase API URL to core REST API.
    ///
    /// (Note: no trailing slash on either endpoint style.)
    pub fn new(base_url: &str, user_agent: &str) -> Result<Self> {
        let mut http = HttpClient::builder();
        let ua = format!("parsoid-rs/{VERSION} {user_agent}");

        #[cfg(target_arch = "wasm32")]
        {
            let mut headers = header::HeaderMap::new();
            headers
                .insert("Api-User-Agent", header::HeaderValue::from_str(&ua)?);
            http = http.default_headers(headers);
        }

        #[cfg(not(target_arch = "wasm32"))]
        {
            http = http.user_agent(ua);
        }

        Self::new_with_client(base_url, http.build()?)
    }

    /// Create a new Client using an existing [`reqwest::Client`]. See the
    /// documentation for [`Client::new()`] for what `base_url` should be. This is
    /// primarily useful when you are already making calls to the wiki and
    /// want to share connection pools and cookie state.
    pub fn new_with_client(base_url: &str, http: HttpClient) -> Result<Self> {
        #[cfg(not(feature = "restbase"))]
        let base_url = if base_url.contains("/api/rest_v1") {
            // convert Wikimedia RESTBase API to Core REST API
            base_url.replace("/api/rest_v1", "/w/rest.php")
        } else {
            base_url.to_string()
        };
        #[cfg(feature = "restbase")]
        let base_url = base_url.to_string();

        let endpoint_kind = if base_url.contains("/api/rest_v1") {
            #[cfg(not(feature = "restbase"))]
            unreachable!();
            #[cfg(feature = "restbase")]
            {
                warn!("RESTBase is deprecated, Use rest.php instead.");
                ParsoidEndpointKind::RESTBase
            }
        } else if base_url.contains("/rest.php") && base_url.ends_with("v3") {
            ParsoidEndpointKind::ParsoidExtension
        } else if base_url.contains("/rest.php") {
            ParsoidEndpointKind::Core
        } else {
            return Err(Error::UnrecognizedParsoidEndpoint());
        };

        Ok(Client {
            http,
            base_url,
            endpoint_kind,
            semaphore: Arc::new(Semaphore::new(10)),
        })
    }

    fn default_headers(&self) -> HeaderMap {
        lazy_static! {
            static ref HEADERMAP: HeaderMap = {
                let mut headers = header::HeaderMap::new();
                headers.insert(
                    header::ACCEPT,
                    ACCEPT_2_8_0
                        .parse()
                        .expect("Unable to parse Accept header"),
                );
                headers
            };
        }

        (*HEADERMAP).clone()
    }

    /// Helper to get a page's HTML
    /// When revid is provided, page is required on RESTBase API,
    /// but not on MediaWiki REST API.
    async fn page_html(
        &self,
        page: &str,
        revid: Option<u64>,
    ) -> Result<Response> {
        let url = match self.endpoint_kind {
            ParsoidEndpointKind::Core => {
                if let Some(revid) = revid {
                    format!(
                        "{}/v1/revision/{}/html?redirect=no",
                        self.base_url, revid
                    )
                } else {
                    format!(
                        "{}/v1/page/{}/html?redirect=no",
                        self.base_url,
                        encode(page)
                    )
                }
            }
            ParsoidEndpointKind::ParsoidExtension => {
                if let Some(revid) = revid {
                    format!(
                        "{}/page/html/{}/{}?redirect=false",
                        self.base_url,
                        encode(page),
                        revid
                    )
                } else {
                    format!(
                        "{}/page/html/{}?redirect=false",
                        self.base_url,
                        encode(page)
                    )
                }
            }
            #[cfg(feature = "restbase")]
            ParsoidEndpointKind::RESTBase => {
                if let Some(revid) = revid {
                    format!(
                        "{}/page/html/{}/{}?redirect=false",
                        self.base_url,
                        encode(page),
                        revid
                    )
                } else {
                    format!(
                        "{}/page/html/{}?redirect=false",
                        self.base_url,
                        encode(page)
                    )
                }
            }
        };
        let req = self.http.get(url).headers(self.default_headers()).build()?;
        let _lock = self.semaphore.acquire().await?;
        debug!(?req);
        // TODO: improve error handling
        let resp = self.http.execute(req).await?;
        debug!(?resp);
        drop(_lock);
        if resp.status() == 404 {
            Err(Error::PageDoesNotExist(page.to_string()))
        } else {
            Ok(resp.error_for_status()?)
        }
    }

    /// Get a `Wikicode` instance for the specified page
    pub async fn get(&self, page: &str) -> Result<ImmutableWikicode> {
        let resp = self.page_html(page, None).await?;
        let etag = match &resp.headers().get("etag") {
            Some(etag) => match etag.to_str() {
                Ok(etag) => Some(etag.to_string()),
                Err(_) => return Err(Error::InvalidEtag),
            },
            None => None,
        };
        // We go through Wikicode -> ImmutableWikicode to parse the revid out
        // of the HTML. TODO: there's probably a better way.
        let code = {
            let mut code = Wikicode::new(&resp.text().await?);
            code.etag = etag;
            code.title = Some(page.to_string());
            code.into_immutable()
        };
        Ok(code)
    }

    /// Get a `Wikicode` instance for the specified page at the specified revision
    pub async fn get_revision(
        &self,
        page: &str,
        revid: u64,
    ) -> Result<ImmutableWikicode> {
        let resp = self.page_html(page, Some(revid)).await?;
        let etag = match &resp.headers().get("etag") {
            Some(etag) => match etag.to_str() {
                Ok(etag) => Some(etag.to_string()),
                Err(_) => return Err(Error::InvalidEtag),
            },
            None => None,
        };
        Ok(ImmutableWikicode {
            html: resp.text().await?,
            title: Some(page.to_string()),
            etag,
            revid: Some(revid),
        })
    }

    /// Get the Parsoid HTML for the specified page
    pub async fn get_raw(&self, page: &str) -> Result<String> {
        Ok(self.page_html(page, None).await?.text().await?)
    }

    /// Get the Parsoid HTML for the specified page at the specified revision
    pub async fn get_revision_raw(
        &self,
        page: &str,
        revid: u64,
    ) -> Result<String> {
        Ok(self.page_html(page, Some(revid)).await?.text().await?)
    }

    /// Get a `Wikicode` instance for the specified wikitext
    pub async fn transform_to_html(
        &self,
        wikitext: &str,
    ) -> Result<ImmutableWikicode> {
        let html = self.transform_to_html_raw(wikitext).await?;
        Ok(ImmutableWikicode::new(&html))
    }

    /// Get the Parsoid HTML for the specified wikitext
    pub async fn transform_to_html_raw(
        &self,
        wikitext: &str,
    ) -> Result<String> {
        let req = match self.endpoint_kind {
            ParsoidEndpointKind::Core => {
                let url =
                    format!("{}/v1/transform/wikitext/to/html", self.base_url);
                self.http
                    .post(url)
                    .headers(self.default_headers())
                    .json(&serde_json::json!({ "wikitext": wikitext }))
                    .build()?
            }
            ParsoidEndpointKind::ParsoidExtension => {
                let url =
                    format!("{}/transform/wikitext/to/html", self.base_url);
                self.http
                    .post(url)
                    .headers(self.default_headers())
                    .json(&serde_json::json!({ "wikitext": wikitext }))
                    .build()?
            }
            #[cfg(feature = "restbase")]
            ParsoidEndpointKind::RESTBase => {
                let url =
                    format!("{}/transform/wikitext/to/html", self.base_url);
                self.http
                    .post(url)
                    .headers(self.default_headers())
                    .form(&[("wikitext", wikitext)])
                    .build()?
            }
        };
        let _lock = self.semaphore.acquire().await?;
        debug!(?req);
        let resp = self.http.execute(req).await?;
        debug!(?resp);
        drop(_lock);
        let html = resp.error_for_status()?.text().await?;
        Ok(html)
    }

    /// Get the wikitext for the specified Parsoid HTML
    pub async fn transform_to_wikitext<C: APICode>(
        &self,
        code: &C,
    ) -> Result<String> {
        self.transform_to_wikitext_raw(
            &code.html(),
            code.title().as_deref(),
            code.revid(),
            code.etag(),
        )
        .await
    }

    /// Get the wikitext for the specified Parsoid HTML
    pub async fn transform_to_wikitext_raw(
        &self,
        html: &str,
        title: Option<&str>,
        revid: Option<u64>,
        etag: Option<&str>,
    ) -> Result<String> {
        let mut url = match self.endpoint_kind {
            ParsoidEndpointKind::Core => {
                format!("{}/v1/transform/html/to/wikitext", self.base_url)
            }
            ParsoidEndpointKind::ParsoidExtension => {
                format!("{}/transform/html/to/wikitext", self.base_url)
            }
            #[cfg(feature = "restbase")]
            ParsoidEndpointKind::RESTBase => {
                format!("{}/transform/html/to/wikitext", self.base_url)
            }
        };
        if let Some(title) = title {
            let _ = write!(url, "/{}", encode(title));
            if let Some(revid) = revid {
                let _ = write!(url, "/{revid}");
            }
        }
        let mut header_map = self.default_headers();
        if let Some(etag) = etag {
            header_map.insert(header::IF_MATCH, etag.parse().unwrap());
        }

        let req = self.http.post(url);
        let req = match self.endpoint_kind {
            ParsoidEndpointKind::Core => {
                req.json(&serde_json::json!({ "html": html }))
            }
            ParsoidEndpointKind::ParsoidExtension => {
                req.json(&serde_json::json!({ "html": html }))
            }
            #[cfg(feature = "restbase")]
            ParsoidEndpointKind::RESTBase => req.form(&[("html", html)]),
        };
        let req = req.headers(header_map).build()?;
        let _lock = self.semaphore.acquire().await?;
        debug!(?req);
        let resp = self.http.execute(req).await?;
        debug!(?resp);
        drop(_lock);
        let wikitext = resp.error_for_status()?.text().await?;
        Ok(wikitext)
    }
}