parsoid/
lib.rs

/*
Copyright (C) 2020-2021 Kunal Mehta <legoktm@debian.org>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
//! The `parsoid` crate is a wrapper around [Parsoid HTML](https://www.mediawiki.org/wiki/Specs/HTML/2.8.0)
//! that provides convenient accessors for processing and extraction.
//!
//! Inspired by [mwparserfromhell](https://github.com/earwig/mwparserfromhell/),
//! [parsoid-jsapi](https://github.com/wikimedia/parsoid-jsapi) and built on top
//! of [Kuchiki (朽木)](https://github.com/kuchiki-rs/kuchiki).
//!
//! # Quick starts
//!
//! Fetch HTML and extract the value of a template parameter:
//! ```rust
//! # use parsoid::Result;
//! use parsoid::prelude::*;
//! # #[tokio::main]
//! # async fn main() -> Result<()> {
//! let client = ParsoidClient::new("https://en.wikipedia.org/w/rest.php", "parsoid-rs demo")?;
//! let code = client.get("Taylor_Swift").await?.into_mutable();
//! for template in code.filter_templates()? {
//!     if template.name() == "Template:Infobox person" {
//!         let birth_name = template.param("birth_name").unwrap();
//!         assert_eq!(birth_name, "Taylor Alison Swift");
//!     }
//! }
//! # Ok(())
//! # }
//! ```
//!
//! Add a link to a page and convert it to wikitext:
//! ```rust
//! # use parsoid::Result;
//! use parsoid::prelude::*;
//! # #[tokio::main]
//! # async fn main() -> Result<()> {
//! let client = ParsoidClient::new("https://en.wikipedia.org/w/rest.php", "parsoid-rs demo")?;
//! let code = client.get("Wikipedia:Sandbox").await?.into_mutable();
//! let link = WikiLink::new(
//!     "./Special:Random",
//!     &Wikicode::new_text("Visit a random page")
//! );
//! code.append(&link);
//! let wikitext = client.transform_to_wikitext(&code).await?;
//! assert!(wikitext.ends_with("[[Special:Random|Visit a random page]]"));
//! # Ok(())
//! # }
//! ```
//! This crate provides no functionality for actually saving a page, you'll
//! need to use something like [`mwbot`](https://docs.rs/mwbot).
//!
//! ## Architecture
//! Conceptually this crate provides wiki-related types on top of an HTML processing
//! library. There are three primary constructs to be aware of: `Wikicode`,
//! `Wikinode`, and `Template`.
//!
//! `Wikicode` represents a container of an entire wiki page, equivalent to a
//! `<html>` or `<body>` node. It provides some convenience functions like
//! `filter_links()` to easily operate on and mutate a specific Wikinode.
//! (For convenience, `Wikicode` is also a `Wikinode`.)
//!
//! ```rust
//! # use parsoid::Result;
//! use parsoid::prelude::*;
//! # #[tokio::main]
//! # async fn main() -> Result<()> {
//! let client = ParsoidClient::new("https://en.wikipedia.org/w/rest.php", "parsoid-rs demo")?;
//! let code = client.get("Taylor Swift").await?.into_mutable();
//! for link in code.filter_links() {
//!     if link.target() == "You Belong with Me" {
//!         // ...do something
//!     }
//! }
//! # Ok(())
//! # }
//! ```
//!
//! Filter functions are only provided for common types as an optimization,
//! but it's straightforward to implement for other types:
//! ```rust
//! # use parsoid::Result;
//! use parsoid::prelude::*;
//! # #[tokio::main]
//! # async fn main() -> Result<()> {
//! let client = ParsoidClient::new("https://en.wikipedia.org/w/rest.php", "parsoid-rs demo")?;
//! let code = client.get("Taylor Swift").await?.into_mutable();
//! let entities: Vec<HtmlEntity> = code
//!     .descendants()
//!     .filter_map(|node| node.as_html_entity())
//!     .collect();
//! # Ok(())
//! # }
//! ```
//!
//! `Wikinode` is an enum representing all of the different types of Wikinodes,
//! mostly to enable functions that accept/return various types of nodes.
//!
//! A Wikinode provides convenience functions for working with specific
//! types of MediaWiki constructs. For example, the `WikiLink` type wraps around
//! a node of `<a rel="mw:WikiLink" href="...">...</a>`. It provides functions
//! for accessing or mutating the `href` attribute. To access the link text
//! you would need to use `.children()` and modify or append to those nodes.
//! Standard mutators like `.append()` and `.insert_after()` are part of the
//! `WikinodeIterator` trait, which is automatically imported in the prelude.
//!
//! The following nodes have been implemented so far:
//! * `BehaviorSwitch`: `__TOC__`
//! * `Category`: `[[Category:Foo]]`
//! * `Comment`: `<!-- ... -->`
//! * `ExtLink`: `[https://example.org Text]`
//! * `Heading`: `== Some text ==`
//! * `HtmlEntity`: `&nbsp;`
//! * `IncludeOnly`: `<includeonly>foo</includeonly>`
//! * `InterwikiLink`: `[[:en:Foo]]`
//! * `LanguageLink`: `[[en:Foo]]`
//! * `Nowiki`: `<nowiki>[[foo]]</nowiki>`
//! * `Redirect`: `#REDIRECT [[Foo]]`
//! * `Reference`: the contents inside `<ref>` tags
//! * `ReferenceLink`: the position of `<ref>`
//! * `ReferenceList`: `<references/>`
//! * `Section`: Contains a `Heading` and its contents
//! * `WikiLink`: `[[Foo|bar]]`
//! * `Generic` - any node that we don't have a more specific type for.
//!
//! Each Wikinode is effectively a wrapper around `Rc<Node>`, making it cheap to
//! clone around.
//!
//! ## Templates
//! Unlike Wikinodes, Templates do not have a 1:1 mapping with a HTML node, it's
//! possible to have multiple templates in one node. The main way to get
//! `Template` instances is to call `Wikicode::filter_templates()`.
//!
//! See the [`Template`](./struct.Template.html) documentation for more details
//! and examples.
//!
//! ## noinclude and onlyinclude
//! Similar to Templates, `<noinclude>` and `<onlyinclude>` do not have a
//! 1:1 mapping with a single HTML node, as they may span multiple. The main
//! way to get `NoInclude` or `OnlyInclude` instances is to call
//! `filter_noinclude()` and `filter_onlyinclude()` respectively.
//!
//! See the [module-level](./inclusion/) documentation for more details and
//! examples.
//!
//! ## Safety
//! This library is implemented using only safe Rust and should not panic.
//! However, the HTML is expected to meet some level of well-formedness. For
//! example, if a node has `rel="mw:WikiLink"`, it is assumed it is an `<a>`
//! element. This is not designed to be fully defensive for arbitrary HTML
//! and should only be used with HTML from Parsoid itself or mutated by
//! this or another similar library (contributions to improve this will gladly
//! be welcomed!).
//!
//! Additionally `Wikicode` does not implement [`Send`](https://doc.rust-lang.org/std/marker/trait.Send.html),
//! which means it cannot be safely shared across threads. This is a
//! limitation of the underlying kuchikiki library being used.
//!
//! A `ImmutableWikicode` is provided as a workaround - it is `Send` and
//! contains all the same information `Wikicode` does, but is immutable.
//! Switching between the two is straightforward by using `into_immutable()` and
//! `into_mutable()` or by using the standard `From` and `Into` traits.
//!
//! ## Testing
//! Use the `build_corpus` example to download the first 500 featured articles
//! on the English Wikipedia to create a test corpus.
//!
//! The `featured_articles` example will iterate through those downloaded examples
//! to test the parsing code, clean roundtripping, etc.
//!
//! ## Contributing
//! `parsoid` is a part of the [`mwbot-rs` project](https://www.mediawiki.org/wiki/Mwbot-rs).
//! We're always looking for new contributors, please [reach out](https://www.mediawiki.org/wiki/Mwbot-rs#Contributing)
//! if you're interested!
#![deny(clippy::all)]
#![deny(rustdoc::all)]
#![cfg_attr(docsrs, feature(doc_cfg))]

#[cfg(feature = "http")]
#[cfg_attr(docsrs, doc(cfg(feature = "http")))]
mod api;
mod iter;
/// Re-export of IndexMap
pub mod map {
    pub use indexmap::IndexMap;
}
pub mod cite;
mod error;
mod gallery;
pub mod image;
mod immutable;
pub mod inclusion;
mod indicator;
mod multinode;
pub mod node;
mod template;

#[cfg(test)]
mod tests;

pub(crate) mod private {
    pub trait Sealed {}
}

/// Prelude to import to pull in traits and useful types
///
/// ```rust
/// use parsoid::prelude::*;
/// ```
pub mod prelude {
    #[cfg(feature = "http")]
    #[cfg_attr(docsrs, doc(cfg(feature = "http")))]
    pub use crate::api::Client as ParsoidClient;
    pub use crate::cite::{Reference, ReferenceLink, ReferenceList};
    pub use crate::gallery::Gallery;
    pub use crate::image::Image;
    pub use crate::immutable::ImmutableWikicode;
    pub use crate::inclusion::{NoInclude, OnlyInclude};
    pub use crate::iter::WikinodeIterator;
    pub use crate::map;
    pub use crate::multinode::WikiMultinode;
    pub use crate::node::{
        BehaviorSwitch, Category, Comment, ExtLink, Heading, HtmlEntity,
        IncludeOnly, Indicator, InterwikiLink, LanguageLink, Nowiki, Redirect,
        Section, WikiLink, Wikinode,
    };
    pub use crate::template::Template;
    pub use crate::Wikicode;
}

#[cfg(feature = "http")]
#[cfg_attr(docsrs, doc(cfg(feature = "http")))]
pub use crate::api::Client;
pub use crate::iter::WikinodeIterator;
pub use crate::multinode::WikiMultinode;
use crate::node::{Comment, Redirect, Wikinode};
pub use crate::template::Template;
pub use error::Error;
use kuchikiki::iter::NodeEdge;
use kuchikiki::traits::*;
use kuchikiki::NodeData;
use kuchikiki::NodeRef;
use markup5ever::{LocalName, QualName};
use std::ops::Deref;
pub type Result<T> = std::result::Result<T, Error>;
pub use crate::immutable::ImmutableWikicode;
use percent_encoding::percent_decode_str;

#[macro_use]
extern crate markup5ever;

/// Helper wrapper to create a new `QualName`
fn build_qual_name(tag: LocalName) -> QualName {
    QualName::new(None, ns!(html), tag)
}

/// Container for HTML, usually represents the entire page
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Wikicode {
    document: NodeRef,
    pub(crate) title: Option<String>,
    pub(crate) etag: Option<String>,
}

impl Deref for Wikicode {
    type Target = NodeRef;

    fn deref(&self) -> &Self::Target {
        &self.document
    }
}

impl Wikicode {
    /// Create a new `Wikicode` instance from raw Parsoid HTML.
    pub fn new(body: &str) -> Self {
        let document = kuchikiki::parse_html().one(body);
        // Remove all `data-parsoid`` attributes
        // This is purely Parsoid-internal information and may be changed without notice.
        // It should not be used by bots because of its unstablity.
        // https://www.mediawiki.org/wiki/Parsoid/Internals/data-parsoid
        let doc_iter = document.traverse_inclusive();
        for node in doc_iter {
            if let NodeEdge::Start(node) = node {
                if let NodeData::Element(data) = node.data() {
                    _ = data.attributes.borrow_mut().remove("data-parsoid");
                }
            }
        }
        Self {
            document,
            title: None,
            etag: None,
        }
    }

    /// Create a new HTML node with the given tag
    /// ```
    /// # use parsoid::prelude::*;
    /// let node = Wikicode::new_node("b");
    /// // Append your list items
    /// node.append(&Wikicode::new_text("bolded text"));
    /// assert_eq!(&node.to_string(), "<b>bolded text</b>")
    /// ```
    pub fn new_node(tag: &str) -> Self {
        let document =
            NodeRef::new_element(crate::build_qual_name(tag.into()), vec![]);
        Self {
            document,
            title: None,
            etag: None,
        }
    }

    /// Create a text node with the given contents
    /// ```
    /// # use parsoid::prelude::*;
    /// let node = Wikicode::new_text("foo bar");
    /// assert_eq!(&node.to_string(), "foo bar");
    /// // Tags will be escaped
    /// let weird_node = Wikicode::new_text("foo <bar>");
    /// assert_eq!(&weird_node.to_string(), "foo &lt;bar&gt;");
    /// ```
    pub fn new_text(text: &str) -> Self {
        let document = NodeRef::new_text(text);
        Self {
            document,
            title: None,
            etag: None,
        }
    }

    /// Create a new `Wikicode` instance from a node we already have
    fn new_from_node(node: &NodeRef) -> Self {
        Wikicode {
            document: node.clone(),
            title: None,
            etag: None,
        }
    }

    /// Get the HTML spec version for this document, if it's available
    pub fn spec_version(&self) -> Option<String> {
        match self
            .document
            .select_first("meta[property=\"mw:htmlVersion\"]")
        {
            Ok(element) => Some(
                element
                    .attributes
                    .borrow()
                    .get("content")
                    .unwrap()
                    .to_string(),
            ),
            Err(_) => None,
        }
    }

    /// Get the revision id associated with the Parsoid HTML, if it has one.
    pub fn revision_id(&self) -> Option<u64> {
        match self.html_element() {
            Some(element) => element
                .as_element()
                .unwrap()
                .attributes
                .borrow()
                .get("about")
                .map(|url| {
                    url.split('/').last().unwrap().to_string().parse().unwrap()
                }),
            None => None,
        }
    }

    /// Get the title associated with the Parsoid HTML, if it has one.
    pub fn title(&self) -> Option<String> {
        self.title.clone()
    }

    pub fn redirect(&self) -> Option<Redirect> {
        match self.document.select_first(Redirect::SELECTOR) {
            Ok(element) => Some(Redirect::new_from_node(element.as_node())),
            Err(_) => None,
        }
    }

    /// Get the root <html> element if it exists, primarily intended to help
    /// getting metadata.
    fn html_element(&self) -> Option<NodeRef> {
        match self.document.select_first("html") {
            Ok(element) => Some(element.as_node().clone()),
            Err(_) => None,
        }
    }

    /// Get the <body> element or everything if there is no body. This is
    /// intented to help ensure nodes (e.g. comments) just come from the body.
    fn body_element(&self) -> Wikinode {
        match self.document.select_first("body") {
            // Just the <body>
            Ok(element) => Wikinode::new_from_node(element.as_node()),
            // Everything as there is no <body>
            Err(_) => Wikinode::Generic(self.clone()),
        }
    }

    /// Get a plain text representation of the Parsoid HTML with all markup
    /// stripped.
    pub fn text_contents(&self) -> String {
        self.body_element().text_contents()
    }

    pub fn into_immutable(self) -> immutable::ImmutableWikicode {
        self.into()
    }
}

impl From<Wikinode> for Wikicode {
    fn from(node: Wikinode) -> Self {
        Wikicode::new_from_node(node.as_node())
    }
}

impl private::Sealed for Wikicode {}

impl WikinodeIterator for Wikicode {
    fn as_node(&self) -> &NodeRef {
        &self.document
    }

    fn filter_comments(&self) -> Vec<Comment> {
        // Override as we want to only get comments from under <body>
        self.body_element()
            .inclusive_descendants()
            .filter_map(|node| node.as_comment())
            .collect()
    }
}

fn clean_link(link: &str) -> String {
    let link = link.trim_start_matches("./").replace('_', " ");
    urlencoding::decode(&link)
        .map(|s| s.to_string())
        .unwrap_or(link)
}

fn parse_target(link: &str) -> String {
    // Needs a protocol for parsing, so stick in a placeholder
    let url = url::Url::parse(&format!("https://placeholder.invalid/{}", link))
        .unwrap();
    // Collect the path, but trim the leading /
    let mut title = url.path().trim_start_matches('/').to_string();
    // If there's a fragment, we stick it back on
    if let Some(fragment) = url.fragment() {
        title.push('#');
        title.push_str(fragment);
    }
    title = title.replace('_', " ");
    title = percent_decode_str(&title)
        .decode_utf8()
        .unwrap()
        .to_string();
    title
}

fn full_link(title: &str) -> String {
    let title = title.replace(' ', "_");
    let title = urlencoding::decode(&title)
        .map(|s| s.to_string())
        .unwrap_or(title);
    format!("./{}", title)
}

fn inner_data<T: serde::de::DeserializeOwned>(node: &NodeRef) -> Result<T> {
    Ok(serde_json::from_str(
        node.as_element()
            .unwrap()
            .attributes
            .borrow()
            .get("data-mw")
            .unwrap(),
    )?)
}

fn set_inner_data<T: serde::ser::Serialize>(
    node: &NodeRef,
    data: T,
) -> Result<()> {
    node.as_element()
        .unwrap()
        .attributes
        .borrow_mut()
        .insert("data-mw", serde_json::to_string(&data)?);
    Ok(())
}

fn assert_element(node: &NodeRef) {
    if node.as_element().is_none() {
        unreachable!("Non-element node passed");
    }
}

/// Check whether an attribute contains the specified word. Basically
/// it's the [`[attr~=value]` selector](https://developer.mozilla.org/en-US/docs/Web/CSS/Attribute_selectors#syntax).
fn attribute_contains_word(check: &str, word: &str) -> bool {
    check.split(' ').any(|value| value == word)
}
parsoid/lib.rs

parsoid/
lib.rs
