#!/usr/bin/env python3

"""
Moin wiki format converter.

Copyright (C) 2018-2019, 2021, 2023, 2026 Paul Boddie <paul@boddie.org.uk>

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 3 of the License, or (at your option) any later
version.

This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
details.

You should have received a copy of the GNU General Public License along with
this program.  If not, see <http://www.gnu.org/licenses/>.
"""

from moinformat import collate_page_metadata, copy_attachments, errors, \
                       extract_included_page_sections, Metadata
from moinformat.io import printstr, stderr

from multiprocessing import Pool
from os.path import split
import sys

try:
    from itertools import izip_longest
except ImportError:
    from itertools import zip_longest as izip_longest

try:
    from itertools import imap
except ImportError:
    imap = map



# Long messages.

message_all_with_filenames = """\
Using --all overrides any indicated pagenames. Either --all or the filenames
should be omitted.
"""

message_explicit_pagenames = """\
Explicit pagenames (indicated using --pagename) are only to be specified when
providing filenames without an input directory (indicated using --input-dir).

To indicate pagenames within an input directory, omit any --pagename flags.
"""

message_tree_format_usage = """\
The --tree option cannot be used together with the --format or --output-format
options since the --tree option indicates use of the "pretty" format.
"""

message_history_without_revisions = """\
The --history option can only be used with an input directory providing a
collection of document revisions, indicated using the --input-dir-type option.
"""

message_history_output_support = """\
The --history option can only be used with an output directory able to provide
storage for the revisions of each document, as opposed to a single revision,
indicated using the --output-dir-type option.
"""

message_number_of_processes = """\
An integer must be provided with the --processes or --number-of-processes
option.
"""

message_invalid_options = """\
The following options or combination of options are invalid:

"""



# Options management.

def getmapping(mappings):

    """
    Return the given 'mappings' - a collection of key-then-value items - as a
    dictionary.
    """

    mapping = {}
    key = None

    for arg in mappings:
        if key is None:
            key = arg
        else:
            mapping[key] = arg
            key = None

    return mapping

def getvalue(values, default=None):

    """
    Return the first value from 'values' or 'default' if 'values' is empty or
    the first value tests as false.
    """

    return values and values[0] or default



# Processing functions.

class Processor:

    "A wrapper for the processing function."

    def __init__(self, metadata, tree=False):

        """
        Initialise processing using the specified 'metadata'.

        If 'tree' is set to a true value, output the document tree for each
        processed page instead of writing to any output context.
        """

        self.metadata = metadata
        self.tree = tree

    def process_page(self, pagename_and_filename):

        """
        Process the page with the given 'pagename_and_filename' tuple of the
        form (pagename, filename).

        Return a tuple of the form (pagename, written, page_metadata) where
        written indicates whether the page was processed and written
        successfully.
        """

        pagename, filename = pagename_and_filename

        stderr.write("%r ...\n" % (pagename or filename))

        # Define a pagename if missing.

        pagename = pagename or split(filename)[-1]

        # Copy the metadata to support parallel processing.

        metadata = self.metadata.copy_for_page(pagename)

        input = metadata.get_input()
        output = metadata.get_output()
        theme = metadata.get_theme()

        history = metadata.get("history")

        # Obtain multiple revisions if appropriate. Where versioned input is
        # available but not used, the current revision will be used explicitly.

        if not filename:
            revisions = input.revisions(pagename)

            # Obtain the last revision, which should be the current one.

            if not history:
                revisions = revisions[-1:]

            # If no revisions exist, return the page as unwritten.

            if not revisions:
                return (pagename, [], {})

        # For a specific input file, use a single unspecified revision.

        else:
            revisions = [None]

        # Obtain any page history metadata.

        page_history = input.read_page_history(pagename)
        page_metadata = {}
        written = []

        for revision in revisions:

            # Define a revision, which may be None.

            metadata.set("revision", revision)

            # Read either from a filename or using a pagename.

            if filename:
                pagetext = input.readfile(filename)
            else:
                pagetext = input.read_revision(pagename, revision)

            # Parse the page content.

            parser = metadata.get_parser()
            document = parser.parse(pagetext)

            parser.evaluate_macros()
            parser.update_metadata()

            page_metadata = parser.get_page_metadata()

            # Obtain a serialiser using the configuration.

            serialiser = metadata.get_serialiser()
            outtext = serialiser.serialise(document)

            # Show a document tree for debugging purposes, if requested.

            if self.tree:
                printstr(outtext)
                written.append(revision)
                continue

            # With a theme, apply it to the text.

            if theme:
                outtext = theme.apply(outtext)

            # If reading from a file, show the result. Otherwise, write to the
            # output context.

            if not output.can_write():
                printstr(outtext)
                written.append(revision)
            else:
                # Obtain any revision metadata.

                revision_metadata = page_history and page_history.get(revision)

                # Write any indicated revision for versioned output using a
                # specific name.

                if output.write_revision(outtext, pagename, revision,
                                         revision_metadata=revision_metadata):
                    written.append(revision)

        # Only write additional resources if the output context supports it,
        # typically when writing to directories that support multiple files,
        # as opposed to standard output.

        if output.can_write():

            # Write details of the current revision in versioned output.

            current = input.current_revision(pagename)
            output.write_current_revision(pagename, current)

            # Write any consolidated page history, selecting only the written
            # revisions.

            if written and page_history:
                written = output.write_page_history(page_history, pagename, written,
                                                    page_metadata=page_metadata)

        # Copy attachments for written pages.

        if written:
            stderr.write("%r\n" % pagename)
            copy_attachments(parser, input, output, all=True)
        else:
            stderr.write("%r not written!\n" % pagename)

        return (pagename, written, page_metadata)

# Main program.

def main():

    "Interpret program options and perform the conversion."

    dirname, progname = split(sys.argv[0])
    args = sys.argv[1:]

    if "--help" in args:
        show_help(progname)
        sys.exit(0)

    # Option values.

    attachments_dir = []
    category_pattern = []
    document_indexes = []
    filenames = []
    icon_sets = []
    icon_types = []
    input_formats = []
    input_dir_types = []
    input_dirs = []
    input_encodings = []
    input_page_seps = []
    interwiki = []
    macro_names = []
    mw_page_seps = []
    number_of_processes = []
    output_dir_types = []
    output_dirs = []
    output_encodings = []
    output_formats = []
    output_page_seps = []
    pagenames = []
    revision_names = []
    root_pagenames = []
    theme_names = []
    variables = []

    # Obtain filenames by default.

    l = filenames

    # Flags.

    all = False
    anchor = False
    bundle = False
    common = False
    debug = False
    extract_included = False
    fragment = False
    history = False
    line_break = False
    macros = False
    non_breaking_space = False
    no_inline = False
    pagename_titles = False
    smileys = False
    tree = False
    unlink_missing = False
    urls = False
    verbatim = False
    wikiwords = False

    for arg in args:

        # Flags with no following arguments.

        # Detect anchor syntax usage.

        if arg == "--anchor":
            anchor = True

        # Detect all documents.

        elif arg == "--all":
            all = True

        # Detect resource bundling.

        elif arg == "--bundle":
            bundle = True

        # Detect use of a common attachments directory.

        elif arg in ("--common", "--common-attachments"):
            common = True

        # Detect debugging mode.

        elif arg == "--debug":
            debug = True

        # Detect included region extraction.

        elif arg == "--extract-included":
            extract_included = True

        # Detect fragment output (if serialising).

        elif arg == "--fragment":
            fragment = True

        # Detect history output.

        elif arg == "--history":
            history = True

        # Detect line break syntax usage.

        elif arg == "--line-break":
            line_break = True

        # Detect macro evaluation.

        elif arg == "--macros":
            macros = True

        # Detect non-breaking space syntax usage.

        elif arg == "--nbsp":
            non_breaking_space = True

        # Detect suppression of inline objects.

        elif arg == "--no-inline":
            no_inline = True

        # Detect pagename titles.

        elif arg == "--pagename-titles":
            pagename_titles = True

        # Recognise smileys.

        elif arg == "--smileys":
            smileys = True

        # Detect tree output.

        elif arg == "--tree":
            if output_formats:
                stderr.write(message_tree_format_usage)
                sys.exit(1)
            tree = True

        # Detect verbatim syntax usage.

        elif arg == "--verbatim":
            verbatim = True

        # Recognise wikiwords (including suppressed wikiwords).

        elif arg == "--wikiwords":
            wikiwords = True

        # Suppress links to missing pages.

        elif arg == "--unlink-missing":
            unlink_missing = True

        # Recognise bare URLs.

        elif arg == "--urls":
            urls = True

        # Options with following arguments.

        # Switch to document index.

        elif arg == "--attachments-dir":
            l = attachments_dir
            continue

        # Switch to the category pagename pattern.

        elif arg == "--category-pattern":
            l = category_pattern
            continue

        # Switch to document index.

        elif arg == "--document-index":
            l = document_indexes
            continue

        # Switch to collecting formats.

        elif arg in ("--format", "--output-format"):
            if tree:
                stderr.write(message_tree_format_usage)
                sys.exit(1)
            l = output_formats
            continue

        # Switch to collecting icon sets.

        elif arg == "--icon-set":
            l = icon_sets
            continue

        # Switch to collecting icon types.

        elif arg == "--icon-types":
            l = icon_types
            continue

        # Switch to collecting input locations.

        elif arg == "--input-dir":
            l = input_dirs
            continue

        # Switch to collecting input context types.

        elif arg == "--input-dir-type":
            l = input_dir_types
            continue

        # Switch to collecting input encodings.

        elif arg == "--input-encoding":
            l = input_encodings
            continue

        # Switch to collecting input formats.

        elif arg == "--input-format":
            l = input_formats
            continue

        # Switch to collecting input page hierarchy separators.

        elif arg == "--input-page-sep":
            l = input_page_seps
            continue

        # Switch to collecting interwiki mappings.

        elif arg == "--interwiki":
            l = interwiki
            continue

        # Detect macro evaluation.

        elif arg == "--macro-names":
            l = macro_names
            macros = True
            continue

        # Switch to collecting MediaWiki page hierarchy separators.

        elif arg == "--mw-page_sep":
            l = mw_page_seps
            continue

        # Switch to collecting number of process details.

        elif arg in ("--number-of-processes", "--processes"):
            l = number_of_processes
            continue

        # Switch to collecting output locations.

        elif arg == "--output-dir":
            l = output_dirs
            continue

        # Switch to collecting output context types.

        elif arg == "--output-dir-type":
            l = output_dir_types
            continue

        # Switch to collecting output encodings.

        elif arg == "--output-encoding":
            l = output_encodings
            continue

        # Switch to collecting output page hierarchy separators.

        elif arg == "--output-page-sep":
            l = output_page_seps
            continue

        # Switch to collecting page names.

        elif arg == "--pagename":
            l = pagenames
            continue

        # Switch to revision name.

        elif arg == "--revision-name":
            l = revision_names
            continue

        # Switch to collecting root page names.

        elif arg == "--root":
            l = root_pagenames
            continue

        # Switch to collecting theme names.

        elif arg == "--theme":
            l = theme_names
            continue

        # Switch to collecting variables.

        elif arg == "--variables":
            l = variables
            continue

        # Stop collecting options and arguments.

        elif arg == "-f":
            pass

        # Collect options and arguments.

        else:
            l.append(arg)

            # Collect multiple mappings.

            if l is icon_sets or l is interwiki or l is macro_names or l is variables:
                continue

        # Collect filenames normally.

        l = filenames

    input_format = input_formats and input_formats[0] or "moin"
    output_format = tree and "pretty" or output_formats and output_formats[0] or "html"
    input_dir = getvalue(input_dirs)
    output_dir = getvalue(output_dirs)

    # Define metadata to coordinate the various components.

    metadata = Metadata({
        "anchor"            : anchor,
        "attachments"       : getvalue(attachments_dir, "attachments"),
        "bundle"            : bundle,
        "category_pattern"  : getvalue(category_pattern, Metadata.defaults["category_pattern"]),
        "common_attachments": common,
        "document_index"    : getvalue(document_indexes),
        "extract_included"  : extract_included,
        "fragment"          : fragment,
        "history"           : history,
        "icon_sets"         : icon_sets or Metadata.defaults["icon_sets"],
        "icon_type"         : getvalue(icon_types, Metadata.defaults["icon_type"]),
        "input_context"     : input_dir and \
                              getvalue(input_dir_types, "directory") or \
                              "standalone",
        "input_encoding"    : getvalue(input_encodings),
        "input_filename"    : input_dir,
        "input_format"      : input_format,
        "input_separator"   : getvalue(input_page_seps),
        "interwiki"         : getmapping(interwiki),
        "line_break"        : line_break,
        "link_format"       : output_format,
        "macros"            : macros,
        "macro_names"       : macro_names,
        "mw_separator"      : getvalue(mw_page_seps, Metadata.defaults["mw_separator"]),
        "no_inline"         : no_inline,
        "non_breaking_space": non_breaking_space,
        "output_context"    : output_dir and \
                              getvalue(output_dir_types, "directory") or \
                              "standalone",
        "output_encoding"   : getvalue(output_encodings),
        "output_format"     : output_format,
        "output_filename"   : output_dir,
        "output_separator"  : getvalue(output_page_seps),
        "pagename_titles"   : pagename_titles,
        "revision_name"     : getvalue(revision_names),
        "root_pagename"     : getvalue(root_pagenames, "FrontPage"),
        "smileys"           : smileys,
        "theme_name"        : not fragment and getvalue(theme_names, "default") or None,
        "unlink_missing"    : unlink_missing,
        "urls"              : urls,
        "variables"         : getmapping(variables),
        "verbatim"          : verbatim,
        "wikiwords"         : wikiwords,
        })

    # A mapping from settings to options for error reporting.

    setting_options = {
        "input_context"     : "--input-dir-type",
        "input_format"      : "--input-format",
        "output_context"    : "--output-dir-type",
        "output_format"     : "--output-format",
        }

    # Test the settings.

    invalid = metadata.test_settings()

    if invalid:
        stderr.write(message_invalid_options)

        for setting, value in invalid:
            option = setting_options[setting]

            stderr.write("%s\t%s" % (option, value))
            stderr.write("\n")

        sys.exit(1)

    # Copy the metadata to support parallel processing.

    # Obtain the input and output contexts and theme for top-level processing.

    input = metadata.get_input()
    output = metadata.get_output()
    theme = metadata.get_theme()

    # Test for the presence of revision information when history is required.

    if history and not input.provides_history():
        stderr.write(message_history_without_revisions)
        sys.exit(1)

    # Test for support for history in the output context.

    if history and not output.supports_history():
        stderr.write(message_history_output_support)
        sys.exit(1)

    # Treat indicated filenames as pagenames if an input directory is indicated
    # and if no pagenames are explicitly specified.

    if input_dir:
        if pagenames:
            stderr.write(message_explicit_pagenames)
            sys.exit(1)

        if all:
            if filenames:
                stderr.write(message_all_with_filenames)
                sys.exit(1)
            else:
                filenames = input.all()

        pagenames = filenames
        filenames = []

    # Obtain the number of processes to be used.

    try:
        number_of_processes = int(getvalue(number_of_processes) or 1)
    except ValueError:
        stderr.write(message_number_of_processes)
        sys.exit(1)

    # Open each file or page, parse the content, serialise the document.

    processor = Processor(metadata, tree)

    # For debugging, it can be much more helpful to use the function directly.

    if debug:
        results = []
        for pagename_and_filename in izip_longest(pagenames, filenames):
            results.append(processor.process_page(pagename_and_filename))

    # Use a pool of processes otherwise.

    else:
        pool = Pool(number_of_processes)
        results = pool.map(processor.process_page, izip_longest(pagenames, filenames))

    # Obtain unwritten pages and collect page metadata.

    not_written = []
    all_page_metadata = {}

    for pagename, written, page_metadata in results:
        if not written:
            not_written.append(pagename)
        else:
            all_page_metadata[pagename] = page_metadata

    written_pages = list(all_page_metadata.keys())

    # Install any theme resources.

    if theme:
        theme.install_resources()

    # Report unwritten pages.

    if not_written:
        stderr.write("Not written: %s\n" % ", ".join(map(repr, not_written)))

    if output.can_write():

        # Collate and write page metadata, but only if all pages were processed.

        if all:
            collated_metadata = collate_page_metadata(all_page_metadata)
            output.write_metadata(collated_metadata)

        # Obtain any stored collated metadata otherwise.

        else:
            collated_metadata = output.read_metadata()

        # NOTE: Generate extracted pages here, along with pages from which sections
        # NOTE: are extracted.

        extract_pages = extract_included_page_sections(metadata, collated_metadata, written_pages)

        # NOTE: Possibly regenerate pages employing updated category data.

        # Perform any finalisation involving the written pages.

        output.finalise(written_pages + extract_pages)

def show_help(progname):

    "Show the help text."

    stderr.write(help_text % progname)

help_text = """\
Usage: %s [ <options> ] ( --all | [ -f] <filename>... )

Input file options:

--all               Detect all document files in the specified input directory
-f                  Indicate a specific file, needed if --icon-set, --interwiki,
                    --macro-names or --variables immediately precedes the filename

Input options:

--attachments-dir   Indicate the name of attachments directories
                    (default: attachments)
--category-pattern  Indicate a regular expression identifying category pagenames
                    and employing an optional group called name to isolate the
                    category name within a pagename
                    (default: ^Category(?P<name>.*))
--common            Obtain attachments from a common directory for all pages,
                    rather than each page having its own subdirectory of a
                    top-level attachments directory
--input-dir         Indicate an input directory containing document files
--input-dir-type    Indicate the type of input directory involved
                    (default: directory)
--input-encoding    Indicate the character encoding used in document files
--input-format      Indicate the format of the parsed documents
                    (default: moin)
--input-page-sep    Indicate the separator used in filenames to encode
                    hierarchical relationships (subpages and descendant pages)
--pagename          Indicate the page name corresponding to an indicated
                    filename, with each successive instance of this option
                    corresponding to each successive filename instance

Output options:

--bundle            Bundle resources such as stylesheets within every document,
                    useful for publishing documents that need to be copied or
                    distributed individually
--document-index    Provide a "DocumentIndex" filename to be used to name page
                    files and in links in HTML format output, useful for local
                    file browsing instead of Web-published content
--format            Indicate the format to be used for serialised documents;
                    equivalent to --output-format
                    (default: html)
--fragment          Indicates that an output fragment, not an entire document,
                    is to be generated, skipping any theming activities
--history           Generate the revision history for each page, producing an
                    output document for each revision, where supported by the
                    output context
--icon-set          Indicate one or more icon sets to be searched for icons
                    used in HTML format output
                    (default: Tango mate gnome)
--icon-type         Indicate the icon type or format of icons used in HTML
                    format output
                    (default: png)
--mw-page-sep       Indicate the separator used in MediaWiki attachment
                    filenames to encode hierarchical relationships (subpages and
                    descendant pages)
--no-inline         Suppress inline objects in serialised documents, linking to
                    separate objects instead
--output-dir        Indicate an output directory to contain serialised document
                    files
--output-dir-type   Indicate the type of output directory involved
                    (default: directory)
--output-encoding   Indicate the character encoding used in serialised document
                    files
--output-format     Indicate the format to be used for serialised documents;
                    equivalent to --format
                    (default: html)
--output-page-sep   Indicate the separator used in filenames to encode
                    hierarchical relationships (subpages and descendant pages)
--revision-name     Provide a filename template to be used to name page revision
                    files, where {revision} in the template is substituted for
                    the specific revision number, and where {format} can be
                    indicated to be substituted for the output format
--theme             Indicate a theme for serialised documents, typically
                    requiring an output directory to be useful
--tree              Produce a document tree representation on standard output
                    instead of generating output files
--unlink-missing    Suppress links to absent or missing pages, producing only
                    the link label in the generated output

Parsing and configuration options:

--anchor            Support the MoinLight anchor syntax
--debug             Process one page at a time, permitting keyboard interruption
                    that generates a usable traceback
--extract-included  Extract page text regions included by the Include macro,
                    create dedicated pages for these regions, and reference
                    these pages in replacement Include macros
--interwiki         Indicate name and corresponding URL pairs to be used to
                    translate interwiki links; unlike Moin, interwiki links do
                    not produce links with this tool unless this option is
                    indicated, and then only for interwiki links whose wiki name
                    is present in the indicated mapping
--line-break        Support the MoinLight line break syntax
--macros            Perform macro evaluation/expansion before serialising
                    documents
--macro-names       Indicate specific macros to be evaluated/expanded before
                    serialising documents
--nbsp              Support the MoinLight non-breaking space syntax
--pagename-titles   Use pagenames as titles instead of the first top-level
                    heading, this traditionally being a prettier form of the
                    pagename and a suitable title
--root              Indicate the root page name to be used
                    (default: FrontPage)
--smileys           Recognise smileys
--urls              Recognise bare URLs; unlike Moin, bare URLs do not produce
                    links with this tool unless this option is indicated
--variables         Indicate name and corresponding value pairs to be used to
                    translate variables
--verbatim          Support the MoinLight verbatim syntax
--wikiwords         Recognise wikiwords including suppressed wikiwords, the
                    latter being of the form !WikiWord and, in conventional Moin
                    markup, suppressing the linking of WikiWord to another page;
                    unlike Moin, bare wikiwords do not produce links with this
                    tool unless this option is indicated

"""

if __name__ == "__main__":
    try:
        main()
    except errors.ProcessingError as exc:
        printstr(str(exc))

# vim: tabstop=4 expandtab shiftwidth=4
