1/*  Part of SWI-Prolog
    2
    3    Author:        Jan Wielemaker
    4    E-mail:        J.Wielemaker@vu.nl
    5    WWW:           http://www.swi-prolog.org
    6    Copyright (c)  2006-2019, University of Amsterdam
    7                              VU University Amsterdam
    8                              CWI, Amsterdam
    9    All rights reserved.
   10
   11    Redistribution and use in source and binary forms, with or without
   12    modification, are permitted provided that the following conditions
   13    are met:
   14
   15    1. Redistributions of source code must retain the above copyright
   16       notice, this list of conditions and the following disclaimer.
   17
   18    2. Redistributions in binary form must reproduce the above copyright
   19       notice, this list of conditions and the following disclaimer in
   20       the documentation and/or other materials provided with the
   21       distribution.
   22
   23    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   24    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   25    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   26    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
   27    COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
   28    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   29    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   30    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   31    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   32    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
   33    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   34    POSSIBILITY OF SUCH DAMAGE.
   35*/
   36
   37:- module(pldoc_wiki,
   38          [ wiki_codes_to_dom/3,        % +Codes, +Args, -DOM
   39            wiki_lines_to_dom/3,        % +Lines, +Map, -DOM
   40            section_comment_header/3,   % +Lines, -Header, -RestLines
   41            summary_from_lines/2,       % +Lines, -Codes
   42            indented_lines/3,           % +Text, +PrefixChars, -Lines
   43            strip_leading_par/2,        % +DOM0, -DOM
   44            autolink_extension/2,       % ?Extension, ?Type
   45            autolink_file/2             % +FileName, -Type
   46          ]).   47:- use_module(library(lists)).   48:- use_module(library(debug)).   49:- use_module(library(error)).   50:- use_module(library(pairs)).   51:- use_module(library(option)).   52:- use_module(library(debug)).   53:- use_module(library(apply)).   54:- use_module(library(dcg/basics)).   55
   56:- use_module(doc_util).   57
   58
   59/** <module> PlDoc wiki parser
   60
   61This file defines the PlDoc wiki parser,  which parses both comments and
   62wiki text files. The original version of this SWI-Prolog wiki format was
   63largely modeled after Twiki (http://twiki.org/).  The current version is
   64extended to take many aspects from   markdown, in particular the doxygen
   65refinement thereof.
   66
   67@see http://www.stack.nl/~dimitri/doxygen/manual/markdown.html
   68*/
   69
   70:- multifile
   71    prolog:doc_wiki_face//2,        % -Out, +VarNames
   72    prolog:doc_url_expansion/3,     % +Alias(Rest), -HREF, -Label
   73    prolog:url_expansion_hook/3,    % +Term, -Ref, -Label
   74    prolog:doc_autolink_extension/2.% +Extension, -Type
   75
   76
   77                 /*******************************
   78                 *          WIKI PARSING        *
   79                 *******************************/
   80
   81%!  wiki_lines_to_dom(+Lines:lines, +Args:list(atom), -Term) is det
   82%
   83%   Translate a Wiki text into  an   HTML  term suitable for html//1
   84%   from the html_write library.
   85
   86wiki_lines_to_dom(Lines, Args, HTML) :-
   87    tokenize_lines(Lines, Tokens0),
   88    normalise_indentation(Tokens0, Tokens),
   89    wiki_structure(Tokens, -1, Pars),
   90    wiki_faces(Pars, Args, HTML).
   91
   92
   93%!  wiki_codes_to_dom(+String, +Args, -DOM) is det.
   94%
   95%   Translate a plain text into a DOM term.
   96%
   97%   @param String   Plain text.  Either a string or a list of codes.
   98
   99wiki_codes_to_dom(Codes, Args, DOM) :-
  100    indented_lines(Codes, [], Lines),
  101    wiki_lines_to_dom(Lines, Args, DOM).
  102
  103
  104%!  wiki_structure(+Lines:lines, +BaseIndent,
  105%!                 -Blocks:list(block)) is det
  106%
  107%   Get the structure in terms  of block-level elements: paragraphs,
  108%   lists and tables. This processing uses   a mixture of layout and
  109%   punctuation.
  110
  111wiki_structure([], _, []) :- !.
  112wiki_structure([_-[]|T], BI, Pars) :-          % empty lines
  113    !,
  114    wiki_structure(T, BI, Pars).
  115wiki_structure(Lines, _, [\tags(Tags)]) :-
  116    tags(Lines, Tags),
  117    !.
  118wiki_structure(Lines, BI, [P1|PL]) :-
  119    take_block(Lines, BI, P1, RestLines),
  120    wiki_structure(RestLines, BI, PL).
  121
  122%!  take_block(+Lines, +BaseIndent, ?Block, -RestLines) is semidet.
  123%
  124%   Take a block-structure from the input.  Defined block elements
  125%   are lists, table, hrule, section header and paragraph.
  126
  127take_block([_-[]|Lines], BaseIndent, Block, Rest) :-
  128    !,
  129    take_block(Lines, BaseIndent, Block, Rest).
  130take_block([N-_|_], BaseIndent, _, _) :-
  131    N < BaseIndent,
  132    !,
  133    fail.                           % less indented
  134take_block(Lines, BaseIndent, List, Rest) :-
  135    list_item(Lines, Type, Indent, LI, LIT, Rest0),
  136    !,
  137    Indent > BaseIndent,
  138    rest_list(Rest0, Type, Indent, LIT, [], Rest),
  139    List0 =.. [Type, LI],
  140    (   ul_to_dl(List0, List)
  141    ->  true
  142    ;   List0 = dl(Items)
  143    ->  List = dl(class=wiki, Items)
  144    ;   List = List0
  145    ).
  146take_block([N-['|'|RL1]|LT], _, Table, Rest) :-
  147    phrase(row(R0), RL1),
  148    take_table(LT, N, R0, Table, Rest),
  149    !.
  150take_block([0-[-,-|More]|LT], _, Block, LT) :-  % separation line
  151    maplist(=(-), More),
  152    !,
  153    Block = hr([]).
  154take_block([_-Line|LT], _, Block, LT) :-        % separation line
  155    ruler(Line),
  156    !,
  157    Block = hr([]).
  158take_block([_-[@|_]], _, _, _) :-              % starts @tags section
  159    !,
  160    fail.
  161take_block(Lines, _BaseIndent, Section, RestLines) :-
  162    section_header(Lines, Section, RestLines),
  163    !.
  164take_block([_-Verb|Lines], _, Verb, Lines) :-
  165    verbatim_term(Verb),
  166    !.
  167take_block([I-L1|LT], BaseIndent, Elem, Rest) :-
  168    !,
  169    append(L1, PT, Par),
  170    rest_par(LT, PT, I, BaseIndent, MaxI, Rest),
  171    (   MaxI >= BaseIndent+16
  172    ->  Elem = center(Par)
  173    ;   phrase(blockquote(BQ), Par)
  174    ->  Elem = blockquote(BQ)
  175    ;   Elem = p(Par)
  176    ).
  177take_block([Verb|Lines], _, Verb, Lines).
  178
  179blockquote(Clean) -->
  180    [>, ' '],
  181    bq_lines(Clean).
  182
  183bq_lines([' '|Par]) -->
  184    ['\n'], !, [>,' '],
  185    bq_lines(Par).
  186bq_lines([H|T]) -->
  187    [H],
  188    bq_lines(T).
  189bq_lines([]) -->
  190    [].
  191
  192
  193%!  ruler(+Line) is semidet.
  194%
  195%   True if Line contains 3 ruler chars and otherwise spaces.
  196
  197ruler([C0|Line]) :-
  198    rule_char(C0),
  199    phrase(ruler(C0, 1), Line).
  200
  201ruler(C, N) --> [C], !, { N2 is N+1 }, ruler(C, N2).
  202ruler(C, N) --> [' '], !, ruler(C, N).
  203ruler(_, N) --> { N >= 3 }.
  204
  205rule_char('-').
  206rule_char('_').
  207rule_char('*').
  208
  209%!  list_item(+Lines, ?Type, ?Indent, -LI0, -LIT, -RestLines) is det.
  210%
  211%   Create a list-item. Naturally this should produce a single item,
  212%   but DL lists produce two items, so   we create the list of items
  213%   as a difference list.
  214%
  215%   @tbd    Pass base-indent
  216
  217list_item([Indent-Line|LT], Type, Indent, Items, ItemT, Rest) :-
  218    !,
  219    list_item_prefix(Type, Line, L1),
  220    (   Type == dl
  221    ->  split_dt(L1, DT0, DD1),
  222        append(DD1, LIT, DD),
  223        strip_ws_tokens(DT0, DT),
  224        Items = [dt(DT),dd(DD)|ItemT]
  225    ;   append(L1, LIT, LI0),
  226        Items = [li(LI0)|ItemT]
  227    ),
  228    rest_list_item(LT, Type, Indent, LIT, Rest).
  229
  230%!  rest_list_item(+Lines, +Type, +Indent, -RestItem, -RestLines) is det
  231%
  232%   Extract the remainder (after the first line) of a list item.
  233
  234rest_list_item(Lines, _Type, Indent, RestItem, RestLines) :-
  235    take_blocks_at_indent(Lines, Indent, Blocks, RestLines),
  236    (   Blocks = [p(Par)|MoreBlocks]
  237    ->  append(['\n'|Par], MoreBlocks, RestItem)
  238    ;   RestItem = Blocks
  239    ).
  240
  241%!  take_blocks_at_indent(+Lines, +Indent, -Pars, -RestLines) is det.
  242%
  243%   Process paragraphs and verbatim blocks (==..==) in bullet-lists.
  244
  245take_blocks_at_indent(Lines, _, [], Lines) :-
  246    skip_empty_lines(Lines, Lines1),
  247    section_header(Lines1, _, _),
  248    !.
  249take_blocks_at_indent(Lines, N, [Block|RestBlocks], RestLines) :-
  250    take_block(Lines, N, Block, Rest0),
  251    !,
  252    take_blocks_at_indent(Rest0, N, RestBlocks, RestLines).
  253take_blocks_at_indent(Lines, _, [], Lines).
  254
  255
  256%!  rest_list(+Lines, +Type, +Indent,
  257%!            -Items, -ItemTail, -RestLines) is det.
  258
  259rest_list(Lines, Type, N, Items, IT, Rest) :-
  260    skip_empty_lines(Lines, Lines1),
  261    list_item(Lines1, Type, N, Items, IT0, Rest0),
  262    !,
  263    rest_list(Rest0, Type, N, IT0, IT, Rest).
  264rest_list(Rest, _, _, IT, IT, Rest).
  265
  266%!  list_item_prefix(?Type, +Line, -Rest) is det.
  267
  268list_item_prefix(ul, [*, ' '|T], T) :- !.
  269list_item_prefix(ul, [-, ' '|T], T) :- !.
  270list_item_prefix(dl, [$, ' '|T], T) :-
  271    split_dt(T, _, _),
  272    !.
  273list_item_prefix(ol, [w(N), '.', ' '|T], T) :-
  274    atom_codes(N, [D]),
  275    between(0'0, 0'9, D).
  276
  277%!  split_dt(+LineAfterDollar, -DT, -Rest)
  278%
  279%   First see whether the entire line is the item. This allows
  280%   creating items holding : by using $ <tokens> :\n
  281
  282split_dt(In, DT, []) :-
  283    append(DT, [':'], In),
  284    !.
  285split_dt(In, DT, Rest) :-
  286    append(DT, [':'|Rest0], In),
  287    (   Rest0 == []
  288    ->  Rest = []
  289    ;   Rest0 = [' '|Rest]
  290    ),
  291    !.
  292
  293
  294%!  ul_to_dl(+UL, -DL) is semidet.
  295%
  296%   Translate an UL list into a DL list   if  all entries are of the
  297%   form "* <term> nl, <description>" and at least one <description>
  298%   is   non-empty,   or    all    items     are    of    the   form
  299%   [[PredicateIndicator]].
  300
  301ul_to_dl(ul(Items), Description) :-
  302    term_items(Items, DLItems, []),
  303    (   terms_to_predicate_includes(DLItems, Preds)
  304    ->  Description = dl(class(predicates), Preds)
  305    ;   member(dd(DD), DLItems), DD \== []
  306    ->  Description = dl(class(termlist), DLItems)
  307    ).
  308
  309term_items([], T, T).
  310term_items([LI|LIs], DLItems, Tail) :-
  311    term_item(LI, DLItems, Tail1),
  312    term_items(LIs, Tail1, Tail).
  313
  314%!  term_item(+LI, -DLItem, ?Tail) is semidet.
  315%
  316%   If LI is of the form <Term> followed  by a newline, return it as
  317%   dt-dd  tuple.  The  <dt>  item    contains  a  term
  318%
  319%       \term(Text, Term, Bindings).
  320
  321term_item(li(Tokens),
  322          [ dt(class=term, \term(Text, Term, Bindings)),
  323            dd(Descr)
  324          | Tail
  325          ], Tail) :-
  326    (   (   append(TermTokens, ['\n'|Descr], Tokens)
  327        ->  true
  328        ;   TermTokens = Tokens,
  329            Descr = []
  330        )
  331    ->  with_output_to(string(Tmp),
  332                       ( forall(member(T, TermTokens),
  333                                write_token(T)),
  334                         write(' .\n'))),
  335        E = error(_,_),
  336        catch(setup_call_cleanup(
  337                  open_string(Tmp, In),
  338                  ( read_dt_term(In, Term, Bindings),
  339                    read_dt_term(In, end_of_file, []),
  340                    atom_string(Text, Tmp)
  341                  ),
  342                  close(In)),
  343              E, fail)
  344    ).
  345
  346write_token(w(X)) :-
  347    !,
  348    write(X).
  349write_token(X) :-
  350    write(X).
  351
  352read_dt_term(In, Term, Bindings) :-
  353    read_term(In, Term,
  354              [ variable_names(Bindings),
  355                module(pldoc_modes)
  356              ]).
  357
  358terms_to_predicate_includes([], []).
  359terms_to_predicate_includes([dt(class=term, \term(_, [[PI]], [])), dd([])|T0],
  360                            [\include(PI, predicate, [])|T]) :-
  361    is_pi(PI),
  362    terms_to_predicate_includes(T0, T).
  363
  364is_pi(Name/Arity) :-
  365    atom(Name),
  366    integer(Arity),
  367    between(0, 20, Arity).
  368is_pi(Name//Arity) :-
  369    atom(Name),
  370    integer(Arity),
  371    between(0, 20, Arity).
  372
  373
  374%!  row(-Cells)// is det.
  375
  376row([C0|CL]) -->
  377    cell(C0),
  378    !,
  379    row(CL).
  380row([]) -->
  381    [].
  382
  383cell(td(C)) -->
  384    face_tokens(C0),
  385    ['|'],
  386    !,
  387    { strip_ws_tokens(C0, C)
  388    }.
  389
  390face_tokens([]) -->
  391    [].
  392face_tokens(Tokens) -->
  393    face_token(H),                          % Deal with embedded *|...|*, etc.
  394    token('|'),
  395    face_tokens(Face),
  396    token('|'),
  397    face_token(H),
  398    !,
  399    { append([[H,'|'], Face, ['|', H], Rest], Tokens) },
  400    face_tokens(Rest).
  401face_tokens([H|T]) -->
  402    token(H),
  403    face_tokens(T).
  404
  405face_token(=) --> [=].
  406face_token(*) --> [*].
  407face_token('_') --> ['_'].
  408
  409take_table(Lines, Indent, Row0, Table, Rest) :-
  410    rest_table(Lines, Indent, Rows, Rest),
  411    (   Rows = [align(Align)|Rows1]
  412    ->  maplist(align_row(Align), Rows1, Rows2),
  413        (   maplist(=(td([])), Row0)        % empty header
  414        ->  Table = table(class=wiki, Rows2)
  415        ;   maplist(td_to_th, Row0, Header),
  416            Table = table(class=wiki, [tr(Header)|Rows2])
  417        )
  418    ;   Table = table(class=wiki, [tr(Row0)|Rows])
  419    ).
  420
  421td_to_th(td(X), th(X)) :- !.
  422td_to_th(X, X).
  423
  424align_row(Align, tr(Row0), tr(Row)) :-
  425    align_cells(Align, Row0, Row).
  426
  427align_cells([Align|AT], [Cell0|T0], [Cell|T]) :-
  428    align_cell(Align, Cell0, Cell),
  429    align_cells(AT, T0, T).
  430align_cells(_, Cells, Cells).
  431
  432align_cell(Align, td(Content), td(class=Align, Content)).
  433
  434%!  rest_table(+Lines, +Indent, -Rows, -RestLines).
  435
  436rest_table([N-Line|LT], N, [align(Align)|RL], Rest) :-
  437    phrase(column_alignment(Align), Line),
  438    !,
  439    rest_table2(LT, N, RL, Rest).
  440rest_table(Lines, N, RL, Rest) :-
  441    rest_table2(Lines, N, RL, Rest).
  442
  443rest_table2([N-['|'|RL1]|LT], N, [tr(R0)|RL], Rest) :-
  444    !,
  445    phrase(row(R0), RL1),
  446    rest_table2(LT, N, RL, Rest).
  447rest_table2(Rest, _, [], Rest).
  448
  449%!  column_alignment(-Alignment) is semidet.
  450%
  451%   Process an alignment line.
  452
  453column_alignment([H|T]) -->
  454    ['|'],
  455    (   colspec(H)
  456    ->  column_alignment(T)
  457    ;   {T=[]}
  458    ).
  459
  460colspec(Align) -->
  461    ws_tokens, [':'], dashes3,
  462    (   [':']
  463    ->  {Align = center}
  464    ;   {Align = left}
  465    ),
  466    ws_tokens.
  467colspec(Align) -->
  468    ws_tokens, dashes3,
  469    (   [':']
  470    ->  {Align = right}
  471    ;   {Align = left}
  472    ),
  473    ws_tokens.
  474
  475dashes3 -->
  476    [-,-,-],
  477    dashes.
  478
  479dashes --> [-], !, dashes.
  480dashes --> [].
  481
  482ws_tokens --> [' '], !, ws_tokens.
  483ws_tokens --> [].
  484
  485%!  rest_par(+Lines, -Par,
  486%!           +BaseIndent, +MaxI0, -MaxI, -RestLines) is det.
  487%
  488%   Take the rest of a paragraph. Paragraphs   are  ended by a blank
  489%   line or the start of a list-item.   The latter is a bit dubious.
  490%   Why not a general  block-level   object?  The current definition
  491%   allows for writing lists without a blank line between the items.
  492
  493rest_par([], [], BI, MaxI0, MaxI, []) :-
  494    !,
  495    MaxI is max(BI, MaxI0).
  496rest_par([_-[]|Rest], [], _, MaxI, MaxI, Rest) :- !.
  497rest_par(Lines, [], _, MaxI, MaxI, Lines) :-
  498    Lines = [_-Verb|_],
  499    verbatim_term(Verb),
  500    !.
  501rest_par([I-L|Rest], [], _, MaxI, MaxI, [I-L|Rest]) :-
  502    list_item_prefix(_, L, _),
  503    !.
  504rest_par([I-L1|LT], ['\n'|Par], BI, MaxI0, MaxI, Rest) :-
  505    append(L1, PT, Par),
  506    MaxI1 is max(I, MaxI0),
  507    rest_par(LT, PT, BI, MaxI1, MaxI, Rest).
  508
  509
  510%!  section_header(+Lines, -Section, -RestLines) is semidet.
  511%
  512%   Get a section line from the input.
  513
  514section_header([_-L1|LT], Section, LT) :-
  515    twiki_section_line(L1, Section),
  516    !.
  517section_header([0-L1|LT], Section, LT) :-
  518    md_section_line(L1, Section),
  519    !.
  520section_header([_-L1,0-L2|LT], Section, LT) :-
  521    md_section_line(L1, L2, Section),
  522    !.
  523
  524%!  twiki_section_line(+Tokens, -Section) is semidet.
  525%
  526%   Extract a section using the Twiki   conventions. The section may
  527%   be preceeded by [Word], in which case we generate an anchor name
  528%   Word for the section.
  529
  530twiki_section_line([-,-,-|Rest], Section) :-
  531    plusses(Rest, Section).
  532
  533plusses([+, ' '|Rest], h1(Attrs, Content)) :-
  534    hdr_attributes(Rest, Attrs, Content).
  535plusses([+, +, ' '|Rest], h2(Attrs, Content)) :-
  536    hdr_attributes(Rest, Attrs, Content).
  537plusses([+, +, +, ' '|Rest], h3(Attrs, Content)) :-
  538    hdr_attributes(Rest, Attrs, Content).
  539plusses([+, +, +, +, ' '|Rest], h4(Attrs, Content)) :-
  540    hdr_attributes(Rest, Attrs, Content).
  541
  542hdr_attributes(List, Attrs, Content) :-
  543    strip_leading_ws(List, List2),
  544    (   List2 = ['[',w(Name),']'|List3]
  545    ->  strip_ws_tokens(List3, Content),
  546        Attrs = [class(wiki), id(Name)]
  547    ;   Attrs = class(wiki),
  548        strip_ws_tokens(List, Content)
  549    ).
  550
  551%!  md_section_line(+Tokens, -Section) is semidet.
  552%
  553%   Handle markdown section lines staring with #
  554
  555md_section_line([#, ' '|Rest], h1(Attrs, Content)) :-
  556    md_section_attributes(Rest, Attrs, Content).
  557md_section_line([#, #, ' '|Rest], h2(Attrs, Content)) :-
  558    md_section_attributes(Rest, Attrs, Content).
  559md_section_line([#, #, #, ' '|Rest], h3(Attrs, Content)) :-
  560    md_section_attributes(Rest, Attrs, Content).
  561md_section_line([#, #, #, #, ' '|Rest], h4(Attrs, Content)) :-
  562    md_section_attributes(Rest, Attrs, Content).
  563
  564md_section_attributes(List, Attrs, Content) :-
  565    phrase((tokens(Content), [' '], section_label(Label)), List),
  566    !,
  567    Attrs = [class(wiki), id(Label)].
  568md_section_attributes(Content, Attrs, Content) :-
  569    Attrs = [class(wiki)].
  570
  571section_label(Label) -->
  572    [ '{', '#', w(Name) ],
  573    label_conts(Cont), ['}'],
  574    !,
  575    { atomic_list_concat([Name|Cont], Label) }.
  576
  577label_conts([H|T]) --> label_cont(H), !, label_conts(T).
  578label_conts([]) --> [].
  579
  580label_cont(-) --> [-].
  581label_cont(Name) --> [w(Name)].
  582
  583
  584md_section_line(Line1, Line2, Header) :-
  585    Line1 \== [],
  586    section_underline(Line2, Type),
  587    is_list(Line1),
  588    phrase(wiki_words(_), Line1),  % Should not have structure elements
  589    !,
  590    (   phrase(labeled_section_line(Title, Attrs), Line1)
  591    ->  true
  592    ;   Title = Line1,
  593        Attrs = []
  594    ),
  595    Header =.. [Type, [class(wiki)|Attrs], Title].
  596
  597section_underline([=,=,=|T], h1) :-
  598    maplist(=(=), T),
  599    !.
  600section_underline([-,-,-|T], h2) :-
  601    maplist(=(-), T),
  602    !.
  603
  604labeled_section_line(Title, Attrs) -->
  605    tokens(Title), [' '], section_label(Label),
  606    !,
  607    { Attrs = [id(Label)] }.
  608
  609
  610%!  strip_ws_tokens(+Tokens, -Stripped)
  611%
  612%   Strip leading and trailing whitespace from a token list.  Note
  613%   the the whitespace is already normalised.
  614
  615strip_ws_tokens([' '|T0], T) :-
  616    !,
  617    strip_ws_tokens(T0, T).
  618strip_ws_tokens(L0, L) :-
  619    append(L, [' '], L0),
  620    !.
  621strip_ws_tokens(L, L).
  622
  623
  624%!  strip_leading_ws(+Tokens, -Stripped) is det.
  625%
  626%   Strip leading whitespace from a token list.
  627
  628strip_leading_ws([' '|T], T) :- !.
  629strip_leading_ws(T, T).
  630
  631
  632                 /*******************************
  633                 *             TAGS             *
  634                 *******************************/
  635
  636%!  tags(+Lines:lines, -Tags) is semidet.
  637%
  638%   If the first line is a @tag, read the remainder of the lines to
  639%   a list of \tag(Name, Value) terms.
  640
  641tags(Lines, Tags) :-
  642    collect_tags(Lines, Tags0),
  643    keysort(Tags0, Tags1),
  644    pairs_values(Tags1, Tags2),
  645    combine_tags(Tags2, Tags).
  646
  647%!  collect_tags(+IndentedLines, -Tags) is semidet
  648%
  649%   Create a list Order-tag(Tag,Tokens) for   each @tag encountered.
  650%   Order is the desired position as defined by tag_order/2.
  651%
  652%   @tbd Tag content is  often  poorly   aligned.  We  now  find the
  653%   alignment of subsequent lines  and  assume   the  first  line is
  654%   alligned with the remaining lines.
  655
  656collect_tags([], []).
  657collect_tags([Indent-[@,String|L0]|Lines], [Order-tag(Tag,Value)|Tags]) :-
  658    tag_name(String, Tag, Order),
  659    !,
  660    strip_leading_ws(L0, L),
  661    rest_tag(Lines, Indent, VT, RestLines),
  662    normalise_indentation(VT, VT1),
  663    wiki_structure([0-L|VT1], -1, Value0),
  664    strip_leading_par(Value0, Value),
  665    collect_tags(RestLines, Tags).
  666
  667
  668%!  tag_name(+String, -Tag:atom, -Order:int) is semidet.
  669%
  670%   If String denotes a know tag-name,
  671
  672tag_name(w(Name), Tag, Order) :-
  673    (   renamed_tag(Name, Tag, Level),
  674        tag_order(Tag, Order)
  675    ->  print_message(Level, pldoc(deprecated_tag(Name, Tag)))
  676    ;   tag_order(Name, Order)
  677    ->  Tag = Name
  678    ;   print_message(warning, pldoc(unknown_tag(Name))),
  679        fail
  680    ).
  681
  682
  683rest_tag([], _, [], []) :- !.
  684rest_tag(Lines, Indent, [], Lines) :-
  685    Lines = [Indent-[@,Word|_]|_],
  686    tag_name(Word, _, _),
  687    !.
  688rest_tag([L|Lines0], Indent, [L|VT], Lines) :-
  689    rest_tag(Lines0, Indent, VT, Lines).
  690
  691
  692%!  renamed_tag(+DeprecatedTag:atom, -Tag:atom, -Warn) is semidet.
  693%
  694%   Declaration for deprecated tags.
  695
  696renamed_tag(exception, throws, warning).
  697renamed_tag(param,     arg,    silent).
  698
  699
  700%!  tag_order(+Tag:atom, -Order:int) is semidet.
  701%
  702%   Both declares the know tags and  their expected order. Currently
  703%   the tags are forced into  this   order  without  warning. Future
  704%   versions may issue a warning if the order is inconsistent.
  705
  706:- multifile
  707    pldoc:tag_order/2.  708
  709tag_order(Tag, Order) :-
  710    pldoc:tag_order(Tag, Order),
  711    !.
  712tag_order(arg,         100).
  713tag_order(error,       200).            % same as throw
  714tag_order(throws,      300).
  715tag_order(author,      400).
  716tag_order(version,     500).
  717tag_order(see,         600).
  718tag_order(deprecated,  700).
  719tag_order(compat,      800).            % PlDoc extension
  720tag_order(copyright,   900).
  721tag_order(license,    1000).
  722tag_order(bug,        1100).
  723tag_order(tbd,        1200).
  724
  725%!  combine_tags(+Tags:list(tag(Key, Value)), -Tags:list) is det.
  726%
  727%   Creates the final tag-list.  Tags is a list of
  728%
  729%           * \params(list(param(Name, Descr)))
  730%           * \tag(Name, list(Descr))
  731%
  732%   Descr is a list of tokens.
  733
  734combine_tags([], []).
  735combine_tags([tag(arg, V1)|T0], [\args([P1|PL])|Tags]) :-
  736    !,
  737    arg_tag(V1, P1),
  738    arg_tags(T0, PL, T1),
  739    combine_tags(T1, Tags).
  740combine_tags([tag(Tag,V0)|T0], [\tag(Tag, [V0|Vs])|T]) :-
  741    same_tag(Tag, T0, T1, Vs),
  742    combine_tags(T1, T).
  743
  744arg_tag([PT|Descr0], arg(PN, Descr)) :-
  745    word_of(PT, PN),
  746    strip_leading_ws(Descr0, Descr).
  747
  748word_of(w(W), W) :- !.                  % TBD: check non-word arg
  749word_of(W, W).
  750
  751arg_tags([tag(arg, V1)|T0], [P1|PL], T) :-
  752    !,
  753    arg_tag(V1, P1),
  754    arg_tags(T0, PL, T).
  755arg_tags(T, [], T).
  756
  757same_tag(Tag, [tag(Tag, V)|T0], T, [V|Vs]) :-
  758    !,
  759    same_tag(Tag, T0, T, Vs).
  760same_tag(_, L, L, []).
  761
  762
  763                 /*******************************
  764                 *             FACES            *
  765                 *******************************/
  766
  767%!  wiki_faces(+Structure, +ArgNames, -HTML) is det.
  768%
  769%   Given the wiki structure, analyse the content of the paragraphs,
  770%   list items and table cells and apply font faces and links.
  771
  772wiki_faces([dt(Class, \term(Text, Term, Bindings)), dd(Descr0)|T0],
  773           ArgNames,
  774           [dt(Class, \term(Text, Term, Bindings)), dd(Descr)|T]) :-
  775    !,
  776    varnames(Bindings, VarNames, ArgNames),
  777    wiki_faces(Descr0, VarNames, Descr),
  778    wiki_faces(T0, ArgNames, T).
  779wiki_faces(DOM0, ArgNames, DOM) :-
  780    structure_term(DOM0, Functor, Content0),
  781    !,
  782    wiki_faces_list(Content0, ArgNames, Content),
  783    structure_term(DOM, Functor, Content).
  784wiki_faces(Verb, _, Verb) :-
  785    verbatim_term(Verb),
  786    !.
  787wiki_faces(Content0, ArgNames, Content) :-
  788    assertion(is_list(Content0)),
  789    phrase(wiki_faces(Content, ArgNames), Content0),
  790    !.
  791
  792varnames([], List, List).
  793varnames([Name=_|T0], [Name|T], List) :-
  794    varnames(T0, T, List).
  795
  796wiki_faces_list([], _, []).
  797wiki_faces_list([H0|T0], Args, [H|T]) :-
  798    wiki_faces(H0, Args, H),
  799    wiki_faces_list(T0, Args, T).
  800
  801%!  structure_term(+Term, -Functor, -Content) is semidet.
  802%!  structure_term(-Term, +Functor, +Content) is det.
  803%
  804%   (Un)pack a term describing structure, so  we can process Content
  805%   and re-pack the structure.
  806
  807structure_term(\tags(Tags), tags, [Tags]) :- !.
  808structure_term(\args(Params), args, [Params]) :- !.
  809structure_term(arg(Name,Descr), arg(Name), [Descr]) :- !.
  810structure_term(\tag(Name,Value), tag(Name), [Value]) :- !.
  811structure_term(\include(What,Type,Opts), include(What,Type,Opts), []) :- !.
  812structure_term(dl(Att, Args), dl(Att), [Args]) :- !.
  813structure_term(dt(Att, Args), dt(Att), [Args]) :- !.
  814structure_term(table(Att, Args), table(Att), [Args]) :- !.
  815structure_term(td(Att, Args), td(Att), [Args]) :- !.
  816structure_term(h1(Att, Args), h1(Att), [Args]) :- !.
  817structure_term(h2(Att, Args), h2(Att), [Args]) :- !.
  818structure_term(h3(Att, Args), h3(Att), [Args]) :- !.
  819structure_term(h4(Att, Args), h4(Att), [Args]) :- !.
  820structure_term(hr(Att), hr(Att), []) :- !.
  821structure_term(p(Args), p, [Args]) :- !.
  822structure_term(Term, Functor, Args) :-
  823    structure_term_any(Term, Functor, Args).
  824
  825structure_term(Term) :-
  826    structure_term_any(Term, _Functor, _Args).
  827
  828structure_term_any(Term, Functor, Args) :-
  829    functor(Term, Functor, 1),
  830    structure_tag(Functor),
  831    !,
  832    Term =.. [Functor|Args].
  833
  834structure_tag(ul).
  835structure_tag(ol).
  836structure_tag(dl).
  837structure_tag(li).
  838structure_tag(dt).
  839structure_tag(dd).
  840structure_tag(table).
  841structure_tag(tr).
  842structure_tag(td).
  843structure_tag(th).
  844structure_tag(blockquote).
  845structure_tag(center).
  846
  847
  848%!  verbatim_term(?Term) is det
  849%
  850%   True if Term must be passes verbatim.
  851
  852verbatim_term(pre(_,_)).
  853verbatim_term(\term(_,_,_)).
  854
  855%!  matches(:Goal, -Input, -Last)//
  856%
  857%   True when Goal runs successfully on the DCG input and Input
  858%   is the list of matched tokens.
  859
  860:- meta_predicate matches(2, -, -, ?, ?).  861
  862matches(Goal, Input, Last, List, Rest) :-
  863    call(Goal, List, Rest),
  864    input(List, Rest, Input, Last).
  865
  866input([H|T0], Rest, Input, Last) :-
  867    (   T0 == Rest
  868    ->  Input = [H],
  869        Last = H
  870    ;   Input = [H|T],
  871        input(T0, Rest, T, Last)
  872    ).
  873
  874
  875%!  wiki_faces(-WithFaces, +ArgNames)// is nondet.
  876%!  wiki_faces(-WithFaces, +ArgNames, +Options)// is nondet.
  877%
  878%   Apply font-changes and automatic  links   to  running  text. The
  879%   faces are applied after discovering   the structure (paragraphs,
  880%   lists, tables, keywords).
  881%
  882%   @arg Options is a dict, minimally containing `depth`
  883
  884wiki_faces(WithFaces, ArgNames, List, Rest) :-
  885    default_faces_options(Options),
  886    catch(wiki_faces(WithFaces, ArgNames, Options, List, Rest),
  887          pldoc(depth_limit),
  888          failed_faces(WithFaces, List, Rest)).
  889
  890default_faces_options(_{depth:5}).
  891
  892failed_faces(WithFaces) -->
  893    { debug(markdown(overflow), 'Depth limit exceeded', []) },
  894    wiki_words(WithFaces).
  895
  896wiki_faces([EmphTerm|T], ArgNames, Options) -->
  897    emphasis_seq(EmphTerm, ArgNames, Options),
  898    !,
  899    wiki_faces_int(T, ArgNames).
  900wiki_faces(Faces, ArgNames, Options) -->
  901    wiki_faces_int(Faces, ArgNames, Options).
  902
  903wiki_faces_int(WithFaces, ArgNames) -->
  904    { default_faces_options(Options)
  905    },
  906    wiki_faces_int(WithFaces, ArgNames, Options).
  907
  908wiki_faces_int([], _, _) -->
  909    [].
  910wiki_faces_int(List, ArgNames, Options) -->
  911    wiki_face(H, ArgNames, Options),
  912    !,
  913    {   is_list(H)
  914    ->  append(H, T, List)
  915    ;   List = [H|T]
  916    },
  917    wiki_faces(T, ArgNames, Options).
  918wiki_faces_int([Before,EmphTerm|T], ArgNames, Options) -->
  919    emphasis_before(Before),
  920    emphasis_seq(EmphTerm, ArgNames, Options),
  921    !,
  922    wiki_faces_int(T, ArgNames, Options).
  923wiki_faces_int([H|T], ArgNames, Options) -->
  924    wiki_face_simple(H, ArgNames, Options),
  925    !,
  926    wiki_faces_int(T, ArgNames, Options).
  927
  928next_level(Options0, Options) -->
  929    {   succ(NewDepth, Options0.depth)
  930    ->  Options = Options0.put(depth, NewDepth)
  931    ;   throw(pldoc(depth_limit))
  932    }.
  933
  934%!  prolog:doc_wiki_face(-Out, +VarNames)// is semidet.
  935%!  prolog:doc_wiki_face(-Out, +VarNames, +Options0)// is semidet.
  936%
  937%   Hook that can be  used  to   provide  additional  processing for
  938%   additional _inline_ wiki constructs.  The DCG list is a list of
  939%   tokens.  Defined tokens are:
  940%
  941%     - w(Atom)
  942%     Recognised word (alphanumerical)
  943%     - Atom
  944%     Single character atom representing punctuation marks or the
  945%     atom =|' '|= (space), representing white-space.
  946%
  947%   The  Out  variable  is  input  for    the  backends  defined  in
  948%   doc_latex.pl and doc_html.pl. Roughly, these   are terms similar
  949%   to what html//1 from library(http/html_write) accepts.
  950
  951wiki_face(Out, Args, _) -->
  952    prolog:doc_wiki_face(Out, Args),
  953    !.
  954wiki_face(var(Arg), ArgNames, _) -->
  955    [w(Arg)],
  956    { memberchk(Arg, ArgNames)
  957    },
  958    !.
  959wiki_face(b(Bold), ArgNames, Options) -->
  960    [*,'|'], string(Tokens), ['|',*],
  961    !,
  962    { phrase(wiki_faces(Bold, ArgNames, Options), Tokens) }.
  963wiki_face(i(Italic), ArgNames, Options) -->
  964    ['_','|'], string(Tokens), ['|','_'],
  965    !,
  966    { phrase(wiki_faces(Italic, ArgNames, Options), Tokens) }.
  967wiki_face(code(Code), _, _) -->
  968    [=], eq_code_words(Words), [=],
  969    !,
  970    { atomic_list_concat(Words, Code) }.
  971wiki_face(code(Code), _, _) -->
  972    [=,'|'], wiki_words(Code), ['|',=],
  973    !.
  974wiki_face(PredRef, _, _) -->
  975    ['`'], take_predref(PredRef), ['`'],
  976    !.
  977wiki_face(\nopredref(Pred), _, _) -->
  978    ['`', '`'], take_predref(\predref(Pred)), ['`', '`'],
  979    !.
  980wiki_face([flag, ' ', \flagref(Flag)], _, _) -->
  981    [ w('flag'), ' ', '`', w(Flag), '`' ],
  982    { current_prolog_flag(Flag, _) },
  983    !.
  984wiki_face(code(Code), _, _) -->
  985    ['`','`'], wiki_words(Code), ['`','`'],
  986    !.
  987wiki_face(Code, _, _) -->
  988    (   ['`'], code_words(Words), ['`']
  989    ->  { atomic_list_concat(Words, Text),
  990          E = error(_,_),
  991          catch(atom_to_term(Text, Term, Vars), E, fail),
  992          !,
  993          code_face(Text, Term, Vars, Code)
  994        }
  995    ).
  996wiki_face(Face, _, Options) -->
  997    [ w(Name) ], arg_list(List),
  998    { atomic_list_concat([Name|List], Text),
  999      E = error(_,_),
 1000      catch(atom_to_term(Text, Term, Vars), E, fail),
 1001      term_face(Text, Term, Vars, Face, Options)
 1002    },
 1003    !.
 1004wiki_face(br([]), _, _) -->
 1005    [<,w(br),>,'\n'], !.
 1006wiki_face(br([]), _, _) -->
 1007    [<,w(br),/,>,'\n'], !.
 1008        % Below this, we only do links.
 1009wiki_face(_, _, Options) -->
 1010    { Options.get(link) == false,
 1011      !,
 1012      fail
 1013    }.
 1014wiki_face(PredRef, _, _) -->
 1015    take_predref(PredRef),
 1016    !.
 1017wiki_face(\cite(Citations), _, _) -->
 1018    ['['], citations(Citations), [']'].
 1019wiki_face(\include(Name, Type, Options), _, _) -->
 1020    ['[','['], file_name(Base, Ext), [']',']'],
 1021    { autolink_extension(Ext, Type),
 1022      !,
 1023      file_name_extension(Base, Ext, Name),
 1024      resolve_file(Name, Options, [])
 1025    },
 1026    !.
 1027wiki_face(\include(Name, Type, [caption(Caption)|Options]), _, _) -->
 1028    (   ['!','['], tokens(100, Caption), [']','(']
 1029    ->  file_name(Base, Ext), [')'],
 1030        { autolink_extension(Ext, Type),
 1031          !,
 1032          file_name_extension(Base, Ext, Name),
 1033          resolve_file(Name, Options, [])
 1034        }
 1035    ),
 1036    !.
 1037wiki_face(Link, ArgNames, Options) -->          % TWiki: [[Label][Link]]
 1038    (   ['[','['], wiki_label(Label, ArgNames, Options), [']','[']
 1039    ->  wiki_link(Link, [label(Label), relative(true), end(']')]),
 1040        [']',']'], !
 1041    ).
 1042wiki_face(Link, ArgNames, Options) -->          % Markdown: [Label](Link)
 1043    (   ['['], wiki_label(Label, ArgNames, Options), [']','(']
 1044    ->  wiki_link(Link, [label(Label), relative(true), end(')')]),
 1045        [')'], !
 1046    ).
 1047wiki_face(Link, _ArgNames, _) -->
 1048    wiki_link(Link, []),
 1049    !.
 1050
 1051wiki_label(Label, _ArgNames, _Options) -->
 1052    image_label(Label).
 1053wiki_label(Label, ArgNames, Options) -->
 1054    next_level(Options, NOptions),
 1055    limit(40, wiki_faces(Label, ArgNames, NOptions.put(link,false))).
 1056
 1057%!  wiki_face_simple(-Out, +ArgNames, +Options)
 1058%
 1059%   Skip simple (non-markup) wiki.
 1060
 1061wiki_face_simple(Word, _, _) -->
 1062    [ w(Word) ],
 1063    !.
 1064wiki_face_simple(SpaceOrPunct, _, _) -->
 1065    [ SpaceOrPunct ],
 1066    { atomic(SpaceOrPunct) },
 1067    !.
 1068wiki_face_simple(FT, ArgNames, _) -->
 1069    [Structure],
 1070    { wiki_faces(Structure, ArgNames, FT)
 1071    }.
 1072
 1073wiki_words([]) --> [].
 1074wiki_words([Word|T]) --> [w(Word)], !, wiki_words(T).
 1075wiki_words([Punct|T]) --> [Punct], {atomic(Punct)}, wiki_words(T).
 1076
 1077%!  code_words(-Words)//
 1078%
 1079%   True when Words is the  content   as  it  appears in =|`code`|=,
 1080%   where =|``|= is mapped to =|`|=.
 1081
 1082code_words([]) --> [].
 1083code_words([Word|T]) --> [w(Word)], code_words(T).
 1084code_words(CodeL) --> ['`','`'], {CodeL = ['`'|T]}, code_words(T).
 1085code_words([Punct|T]) --> [Punct], {atomic(Punct)}, code_words(T).
 1086
 1087%!  eq_code_words(-Words)//
 1088%
 1089%   Stuff that can be between single `=`.  This is limited to
 1090%
 1091%           - Start and end must be a word
 1092%           - In between may be the following punctuation chars:
 1093%             =|.-:/|=, notably dealing with file names and
 1094%             identifiers in various external languages.
 1095
 1096eq_code_words([Word]) -->
 1097    [ w(Word) ].
 1098eq_code_words([Word|T]) -->
 1099    [ w(Word) ], eq_code_internals(T, [End]), [w(End)].
 1100
 1101eq_code_internals(T, T) --> [].
 1102eq_code_internals([H|T], Tail) -->
 1103    eq_code_internal(H),
 1104    eq_code_internals(T, Tail).
 1105
 1106eq_code_internal(Word) -->
 1107    [w(Word)].
 1108eq_code_internal(Punct) -->
 1109    [Punct],
 1110    { eq_code_internal_punct(Punct) }.
 1111
 1112eq_code_internal_punct('.').
 1113eq_code_internal_punct('-').
 1114eq_code_internal_punct(':').
 1115eq_code_internal_punct('/').
 1116
 1117
 1118%!  code_face(+Text, +Term, +Vars, -Code) is det.
 1119%
 1120%   Deal with =|`... code ...`|=  sequences.   Text  is  the matched
 1121%   text, Term is the parsed Prolog term   and Code is the resulting
 1122%   intermediate code.
 1123
 1124code_face(Text, Var, _, Code) :-
 1125    var(Var),
 1126    !,
 1127    Code = var(Text).
 1128code_face(Text, _, _, code(Text)).
 1129
 1130
 1131%!  emphasis_seq(-Out, +ArgNames, +Options) is semidet.
 1132%
 1133%   Recognise emphasis sequences
 1134
 1135emphasis_seq(EmphTerm, ArgNames, Options) -->
 1136    emphasis_start(C),
 1137    next_level(Options, NOptions),
 1138    matches(limit(100, wiki_faces(Emph, ArgNames, NOptions)), Input, Last),
 1139    emphasis_end(C),
 1140    { emph_markdown(Last, Input),
 1141      emphasis_term(C, Emph, EmphTerm)
 1142    },
 1143    !.
 1144
 1145
 1146%!  emphasis_term(+Emphasis, +Tokens, -Term) is det.
 1147%!  emphasis_before(-Before)// is semidet.
 1148%!  emphasis_start(-Emphasis)// is semidet.
 1149%!  emphasis_end(+Emphasis)// is semidet.
 1150%
 1151%   Primitives for Doxygen emphasis handling.
 1152
 1153emphasis_term('_',   Term, i(Term)).
 1154emphasis_term('*',   Term, b(Term)).
 1155emphasis_term('__',  Term, strong(Term)).
 1156emphasis_term('**',  Term, strong(Term)).
 1157
 1158emph_markdown(_, [w(_)]) :- !.
 1159emph_markdown(Last, Tokens) :-
 1160    \+ emphasis_after_sep(Last),
 1161    E = error(_,_),
 1162    catch(b_getval(pldoc_object, Obj), E, Obj = '??'),
 1163    debug(markdown(emphasis), '~q: additionally emphasis: ~p',
 1164          [Obj, Tokens]).
 1165
 1166emphasis_before(Before) -->
 1167    [Before],
 1168    { emphasis_start_sep(Before) }.
 1169
 1170emphasis_start_sep('\n').
 1171emphasis_start_sep(' ').
 1172emphasis_start_sep('<').
 1173emphasis_start_sep('{').
 1174emphasis_start_sep('(').
 1175emphasis_start_sep('[').
 1176emphasis_start_sep(',').
 1177emphasis_start_sep(':').
 1178emphasis_start_sep(';').
 1179
 1180emphasis_start(Which), [w(Word)] -->
 1181    emphasis(Which),
 1182    [w(Word)].
 1183
 1184emphasis(**)   --> [*, *].
 1185emphasis(*)    --> [*].
 1186emphasis('__') --> ['_', '_'].
 1187emphasis('_')  --> ['_'].
 1188
 1189emphasis_end(Which), [After] -->
 1190    emphasis(Which),
 1191    [ After ],
 1192    !,
 1193    { emphasis_close_sep(After) -> true }.
 1194emphasis_end(Which) -->
 1195    emphasis(Which).
 1196
 1197% these characters should not be before a closing * or _.
 1198
 1199emphasis_after_sep('\n').
 1200emphasis_after_sep(' ').
 1201emphasis_after_sep('(').
 1202emphasis_after_sep('[').
 1203emphasis_after_sep('<').
 1204emphasis_after_sep('=').
 1205emphasis_after_sep('+').
 1206emphasis_after_sep('\\').
 1207emphasis_after_sep('@').
 1208
 1209emphasis_close_sep('\n').                       % white
 1210emphasis_close_sep(' ').                        % white
 1211emphasis_close_sep(',').                        % sentence punctuation
 1212emphasis_close_sep('.').
 1213emphasis_close_sep('!').
 1214emphasis_close_sep('?').
 1215emphasis_close_sep(':').
 1216emphasis_close_sep(';').
 1217emphasis_close_sep(']').                        % [**label**](link)
 1218emphasis_close_sep(')').                        % ... _italic_)
 1219emphasis_close_sep('}').                        % ... _italic_}
 1220emphasis_close_sep(Token) :-
 1221    structure_term(Token).
 1222
 1223
 1224%!  arg_list(-Atoms) is nondet.
 1225%
 1226%   Atoms  is  a  token-list  for  a    Prolog   argument  list.  An
 1227%   argument-list is a sequence of tokens '(' ... ')'.
 1228%
 1229%   @bug    the current implementation does not deal correctly with
 1230%           brackets that are embedded in quoted strings.
 1231
 1232arg_list(['('|T]) -->
 1233    ['('], arg_list_close(T, 1).
 1234
 1235arg_list_close(Tokens, Depth) -->
 1236    [')'],
 1237    !,
 1238    (   { Depth == 1 }
 1239    ->  { Tokens = [')'] }
 1240    ;   { Depth > 1 }
 1241    ->  { Tokens = [')'|More],
 1242          NewDepth is Depth - 1
 1243        },
 1244        arg_list_close(More, NewDepth)
 1245    ).
 1246arg_list_close(['('|T], Depth) -->
 1247    ['('], { NewDepth is Depth+1 },
 1248    arg_list_close(T, NewDepth).
 1249arg_list_close([H|T], Depth) -->
 1250    [w(H)],
 1251    !,
 1252    arg_list_close(T, Depth).
 1253arg_list_close([H|T], Depth) -->
 1254    [H],
 1255    arg_list_close(T, Depth).
 1256
 1257
 1258%!  term_face(+Text, +Term, +Vars, -Face, +Options) is semidet.
 1259%
 1260%   Process embedded Prolog-terms. Currently   processes  Alias(Arg)
 1261%   terms that refer to files.  Future   versions  will also provide
 1262%   pretty-printing of Prolog terms.
 1263
 1264term_face(_Text, Term, _Vars, \file(Name, FileOptions), Options) :-
 1265    ground(Term),
 1266    compound(Term),
 1267    compound_name_arity(Term, Alias, 1),
 1268    user:file_search_path(Alias, _),
 1269    existing_file(Term, FileOptions, [], Options),
 1270    !,
 1271    format(atom(Name), '~q', [Term]).
 1272term_face(Text, Term, Vars, Face, _Options) :-
 1273    code_face(Text, Term, Vars, Face).
 1274
 1275untag([], []).
 1276untag([w(W)|T0], [W|T]) :-
 1277    !,
 1278    untag(T0, T).
 1279untag([H|T0], [H|T]) :-
 1280    untag(T0, T).
 1281
 1282%!  image_label(-Label)//
 1283%
 1284%   Match File[;param=value[,param=value]*]
 1285
 1286image_label(\include(Name, image, Options)) -->
 1287    file_name(Base, Ext),
 1288    { autolink_extension(Ext, image),
 1289      file_name_extension(Base, Ext, Name),
 1290      resolve_file(Name, Options, RestOptions)
 1291    },
 1292    file_options(RestOptions).
 1293
 1294
 1295take_predref(\predref(Name/Arity)) -->
 1296    [ w(Name), '/' ], arity(Arity),
 1297    { functor_name(Name)
 1298    }.
 1299take_predref(\predref(Module:(Name/Arity))) -->
 1300    [ w(Module), ':', w(Name), '/' ], arity(Arity),
 1301    { functor_name(Name)
 1302    }.
 1303take_predref(\predref(Name/Arity)) -->
 1304    prolog_symbol_char(S0),
 1305    symbol_string(SRest), [ '/' ], arity(Arity),
 1306    !,
 1307    { atom_chars(Name, [S0|SRest])
 1308    }.
 1309take_predref(\predref(Name//Arity)) -->
 1310    [ w(Name), '/', '/' ], arity(Arity),
 1311    { functor_name(Name)
 1312    }.
 1313take_predref(\predref(Module:(Name//Arity))) -->
 1314    [ w(Module), ':', w(Name), '/', '/' ], arity(Arity),
 1315    { functor_name(Name)
 1316    }.
 1317
 1318%!  file_options(-Options) is det.
 1319%
 1320%   Extracts additional processing options for  files. The format is
 1321%   ;name="value",name2=value2,... Spaces are not allowed.
 1322
 1323file_options(Options) -->
 1324    [;], nv_pairs(Options),
 1325    !.
 1326file_options([]) -->
 1327    [].
 1328
 1329nv_pairs([H|T]) -->
 1330    nv_pair(H),
 1331    (   [',']
 1332    ->  nv_pairs(T)
 1333    ;   {T=[]}
 1334    ).
 1335
 1336nv_pair(Option) -->
 1337    [ w(Name), =,'"'], tokens(Tokens), ['"'],
 1338    !,
 1339    { untag(Tokens, Atoms),
 1340      atomic_list_concat(Atoms, Value0),
 1341      (   atom_number(Value0, Value)
 1342      ->  true
 1343      ;   Value = Value0
 1344      ),
 1345      Option =.. [Name,Value]
 1346    }.
 1347
 1348
 1349%!  wiki_link(-Link, +Options)// is semidet.
 1350%
 1351%   True if we can find a link to a file or URL. Links are described
 1352%   as one of:
 1353%
 1354%       $ filename :
 1355%       A filename defined using autolink_file/2 or
 1356%       autolink_extension/2
 1357%       $ <url-protocol>://<rest-url> :
 1358%       A fully qualified URL
 1359%       $ '<' URL '>' :
 1360%       Be more relaxed on the URL specification.
 1361
 1362:- multifile
 1363    user:url_path/2. 1364
 1365wiki_link(\file(Name, FileOptions), Options) -->
 1366    file_name(Base, Ext),
 1367    { file_name_extension(Base, Ext, Name),
 1368      (   autolink_file(Name, _)
 1369      ;   autolink_extension(Ext, _)
 1370      ),
 1371      !,
 1372      resolve_file(Name, FileOptions, Options)
 1373    }.
 1374wiki_link(\file(Name, FileOptions), Options) -->
 1375    [w(Name)],
 1376    { autolink_file(Name, _),
 1377      !,
 1378      resolve_file(Name, FileOptions, Options)
 1379    },
 1380    !.
 1381wiki_link(a(href(Ref), Label), Options) -->
 1382    [ w(Prot),:,/,/], { url_protocol(Prot) },
 1383    { option(end(End), Options, space)
 1384    },
 1385    tokens_no_whitespace(Rest), peek_end_url(End),
 1386    !,
 1387    { atomic_list_concat([Prot, :,/,/ | Rest], Ref),
 1388      option(label(Label), Options, Ref)
 1389    }.
 1390wiki_link(a(href(Ref), Label), _Options) -->
 1391    [<, w(Alias), :],
 1392    tokens_no_whitespace(Rest), [>],
 1393    { Term = (Alias:Rest),
 1394      prolog:url_expansion_hook(Term, Ref, Label), !
 1395    }.
 1396wiki_link(a(href(Ref), Label), Options) -->
 1397    [<, w(Alias), :],
 1398    { user:url_path(Alias, _)
 1399    },
 1400    tokens_no_whitespace(Rest), [>],
 1401    { atomic_list_concat(Rest, Local),
 1402      (   Local == ''
 1403      ->  Term =.. [Alias,'.']
 1404      ;   Term =.. [Alias,Local]
 1405      ),
 1406      E = error(_,_),
 1407      catch(expand_url_path(Term, Ref), E, fail),
 1408      option(label(Label), Options, Ref)
 1409    }.
 1410wiki_link(a(href(Ref), Label), Options) -->
 1411    [#, w(First)],
 1412    { option(end(End), Options) },
 1413    tokens_no_whitespace(Rest),
 1414    peek_end_url(End),
 1415    !,
 1416    { atomic_list_concat([#,First|Rest], Ref),
 1417      option(label(Label), Options, Ref)
 1418    }.
 1419wiki_link(a(href(Ref), Label), Options) -->
 1420    [<],
 1421    (   { option(relative(true), Options),
 1422          Parts = Rest
 1423        }
 1424    ->  tokens_no_whitespace(Rest)
 1425    ;   { Parts = [Prot, : | Rest]
 1426        },
 1427        [w(Prot), :], tokens_no_whitespace(Rest)
 1428    ),
 1429    [>],
 1430    !,
 1431    { atomic_list_concat(Parts, Ref),
 1432      option(label(Label), Options, Ref)
 1433    }.
 1434
 1435%!  prolog:url_expansion_hook(+Term, -HREF, -Label) is semidet.
 1436%
 1437%   This hook is called after   recognising  =|<Alias:Rest>|=, where
 1438%   Term is of the form Alias(Rest). If   it  succeeds, it must bind
 1439%   HREF to an atom or string representing the link target and Label
 1440%   to an html//1 expression for the label.
 1441
 1442%!  file_name(-Name:atom, -Ext:atom)// is semidet.
 1443%
 1444%   Matches a filename.  A filename is defined as a sequence
 1445%   <segment>{/<segment}.<ext>.
 1446
 1447file_name(FileBase, Extension) -->
 1448    segment(S1),
 1449    segments(List),
 1450    ['.'], file_extension(Extension),
 1451    !,
 1452    { atomic_list_concat([S1|List], '/', FileBase) }.
 1453file_name(FileBase, Extension) -->
 1454    [w(Alias), '('],
 1455    { once(user:file_search_path(Alias, _)) },
 1456    segment(S1),
 1457    segments(List),
 1458    [')'],
 1459    !,
 1460    { atomic_list_concat([S1|List], '/', Base),
 1461      Spec =.. [Alias,Base],
 1462      absolute_file_name(Spec, Path,
 1463                         [ access(read),
 1464                           extensions([pl]),
 1465                           file_type(prolog),
 1466                           file_errors(fail)
 1467                         ]),
 1468      file_name_extension(FileBase, Extension, Path)
 1469    }.
 1470
 1471
 1472segment(..) -->
 1473    ['.','.'],
 1474    !.
 1475segment(Word) -->
 1476    [w(Word)].
 1477segment(Dir) -->
 1478    [w(Word),'.',w(d)],
 1479    { atom_concat(Word, '.d', Dir) }.
 1480
 1481segments([H|T]) -->
 1482    ['/'],
 1483    !,
 1484    segment(H),
 1485    segments(T).
 1486segments([]) -->
 1487    [].
 1488
 1489file_extension(Ext) -->
 1490    [w(Ext)],
 1491    { autolink_extension(Ext, _)
 1492    }.
 1493
 1494
 1495%!  resolve_file(+Name, -FileOptions, ?RestOptions, +Options) is det.
 1496%
 1497%   Find the actual file based on the pldoc_file global variable. If
 1498%   present  and  the   file   is    resolvable,   add   an   option
 1499%   absolute_path(Path) that reflects the current   location  of the
 1500%   file.
 1501
 1502resolve_file(Name, FileOptions, Rest) :-
 1503    existing_file(Name, FileOptions, Rest, []),
 1504    !.
 1505resolve_file(_, Options, Options).
 1506
 1507
 1508existing_file(Name, FileOptions, Rest, Options) :-
 1509    \+ Options.get(link) == false,
 1510    E = error(_,_),
 1511    catch(existing_file_p(Name, FileOptions, Rest), E, fail).
 1512
 1513existing_file_p(Name, FileOptions, Rest) :-
 1514    (   nb_current(pldoc_file, RelativeTo),
 1515        RelativeTo \== []
 1516    ->  Extra = [relative_to(RelativeTo)|Extra1]
 1517    ;   Extra = Extra1
 1518    ),
 1519    (   compound(Name)
 1520    ->  Extra1 = [file_type(prolog)]
 1521    ;   Extra1 = []
 1522    ),
 1523    absolute_file_name(Name, Path,
 1524                       [ access(read),
 1525                         file_errors(fail)
 1526                       | Extra
 1527                       ]),
 1528    FileOptions = [ absolute_path(Path) | Rest ].
 1529
 1530%!  arity(-Arity:int)// is semidet.
 1531%
 1532%   True if the next token can be  interpreted as an arity. That is,
 1533%   refers to a non-negative integers of at most 20. Although Prolog
 1534%   allows for higher arities, we assume 20   is  a fair maximum for
 1535%   user-created predicates that are documented.
 1536
 1537arity(Arity) -->
 1538    [ w(Word) ],
 1539    { E = error(_,_),
 1540      catch(atom_number(Word, Arity), E, fail),
 1541      Arity >= 0, Arity < 20
 1542    }.
 1543
 1544%!  symbol_string(-String)// is nondet
 1545%
 1546%   Accept a sequence of Prolog symbol characters, starting with the
 1547%   shortest (empty) match.
 1548
 1549symbol_string([]) -->
 1550    [].
 1551symbol_string([H|T]) -->
 1552    [H],
 1553    { prolog_symbol_char(H) },
 1554    symbol_string(T).
 1555
 1556prolog_symbol_char(C) -->
 1557    [C],
 1558    { prolog_symbol_char(C) }.
 1559
 1560%!  prolog_symbol_char(?Char)
 1561%
 1562%   True if char is classified by Prolog as a symbol char.
 1563
 1564prolog_symbol_char(#).
 1565prolog_symbol_char($).
 1566prolog_symbol_char(&).
 1567prolog_symbol_char(*).
 1568prolog_symbol_char(+).
 1569prolog_symbol_char(-).
 1570prolog_symbol_char(.).
 1571prolog_symbol_char(/).
 1572prolog_symbol_char(:).
 1573prolog_symbol_char(<).
 1574prolog_symbol_char(=).
 1575prolog_symbol_char(>).
 1576prolog_symbol_char(?).
 1577prolog_symbol_char(@).
 1578prolog_symbol_char(\).
 1579prolog_symbol_char(^).
 1580prolog_symbol_char(~).
 1581
 1582
 1583functor_name(String) :-
 1584    sub_atom(String, 0, 1, _, Char),
 1585    char_type(Char, lower).
 1586
 1587url_protocol(http).
 1588url_protocol(https).
 1589url_protocol(ftp).
 1590url_protocol(mailto).
 1591
 1592peek_end_url(space) -->
 1593    peek(Punct, End),
 1594    { punct_token(Punct),
 1595      space_token(End)
 1596    },
 1597    !.
 1598peek_end_url(space) -->
 1599    peek(End),
 1600    { space_token(End) },
 1601    !.
 1602peek_end_url(space, [], []) :- !.
 1603peek_end_url(Token) -->
 1604    peek(Token),
 1605    !.
 1606
 1607punct_token('.').
 1608punct_token('!').
 1609punct_token('?').
 1610punct_token(',').
 1611punct_token(';').
 1612
 1613space_token(' ') :- !.
 1614space_token('\r') :- !.
 1615space_token('\n') :- !.
 1616space_token(T) :-
 1617    \+ atom(T),                     % high level format like p(...)
 1618    \+ T = w(_).
 1619
 1620%!  autolink_extension(?Ext, ?Type) is nondet.
 1621%
 1622%   True if Ext is a filename extensions that create automatic links
 1623%   in the documentation.
 1624
 1625autolink_extension(Ext, Type) :-
 1626    prolog:doc_autolink_extension(Ext, Type),
 1627    !.
 1628autolink_extension(Ext, prolog) :-
 1629    user:prolog_file_type(Ext,prolog),
 1630    !.
 1631autolink_extension(txt, wiki).
 1632autolink_extension(md,  wiki).
 1633autolink_extension(gif, image).
 1634autolink_extension(png, image).
 1635autolink_extension(jpg, image).
 1636autolink_extension(jpeg, image).
 1637autolink_extension(svg, image).
 1638
 1639%!  autolink_file(?File, -Type) is nondet.
 1640%
 1641%   Files to which we automatically create links, regardless of the
 1642%   extension.
 1643
 1644autolink_file('README', wiki).
 1645autolink_file('TODO', wiki).
 1646autolink_file('ChangeLog', wiki).
 1647
 1648%!  citations(-List)//
 1649%
 1650%   Parse @cite1[;@cite2]* into a list of citations.
 1651
 1652citations([H|T]) -->
 1653    citation(H),
 1654    (   [';']
 1655    ->  citations(T)
 1656    ;   {T=[]}
 1657    ).
 1658
 1659citation(Atom) -->
 1660    [@], wiki_words(Atoms),
 1661    { length(Atoms, Len),
 1662      Len > 10, !,
 1663      fail
 1664    ; true
 1665    },
 1666    end_citation,
 1667    !,
 1668    { atomic_list_concat(Atoms, Atom)
 1669    }.
 1670
 1671end_citation, [';'] --> [';'].
 1672end_citation, ['@'] --> ['@'].
 1673end_citation, [']'] --> [']'].
 1674
 1675
 1676                 /*******************************
 1677                 *           SECTIONS           *
 1678                 *******************************/
 1679
 1680%!  section_comment_header(+Lines, -Header, -RestLines) is semidet.
 1681%
 1682%   Processes   /**   <section>   comments.   Header   is   a   term
 1683%   \section(Type, Title), where  Title  is   an  atom  holding  the
 1684%   section title and Type is an atom holding the text between <>.
 1685%
 1686%   @param Lines    List of Indent-Codes.
 1687%   @param Header   DOM term of the format \section(Type, Title),
 1688%                   where Type is an atom from <type> and Title is
 1689%                   a string holding the type.
 1690
 1691section_comment_header([_-Line|Lines], Header, Lines) :-
 1692    phrase(section_line(Header), Line).
 1693
 1694section_line(\section(Type, Title)) -->
 1695    ws, "<", word(Codes), ">", normalise_white_space(TitleCodes),
 1696    { atom_codes(Type, Codes),
 1697      atom_codes(Title, TitleCodes)
 1698    }.
 1699
 1700                 /*******************************
 1701                 *           TOKENIZER          *
 1702                 *******************************/
 1703
 1704%!  tokenize_lines(+Lines:lines, -TokenLines) is det
 1705%
 1706%   Convert Indent-Codes into Indent-Tokens
 1707
 1708tokenize_lines(Lines, TokenLines) :-
 1709    tokenize_lines(Lines, -1, TokenLines).
 1710
 1711tokenize_lines([], _, []) :- !.
 1712tokenize_lines(Lines, Indent, [Pre|T]) :-
 1713    verbatim(Lines, Indent, Pre, RestLines),
 1714    !,
 1715    tokenize_lines(RestLines, Indent, T).
 1716tokenize_lines([I-H0|T0], Indent0, [I-H|T]) :-
 1717    phrase(line_tokens(H), H0),
 1718    (   H == []
 1719    ->  Indent = Indent0
 1720    ;   Indent = I
 1721    ),
 1722    tokenize_lines(T0, Indent, T).
 1723
 1724
 1725%!  line_tokens(-Tokens:list)// is det.
 1726%
 1727%   Create a list of tokens, where  is  token   is  either  a ' ' to
 1728%   denote spaces, a  term  w(Word)  denoting   a  word  or  an atom
 1729%   denoting a punctuation  character.   Underscores  (_)  appearing
 1730%   inside an alphanumerical string are considered part of the word.
 1731%   E.g., "hello_world_" tokenizes into [w(hello_world), '_'].
 1732
 1733line_tokens([H|T]) -->
 1734    line_token(H),
 1735    !,
 1736    line_tokens(T).
 1737line_tokens([]) -->
 1738    [].
 1739
 1740line_token(T) -->
 1741    [C],
 1742    (   { code_type(C, space) }
 1743    ->  ws,
 1744        { T = ' ' }
 1745    ;   { code_type(C, alnum) },
 1746        word(Rest),
 1747        { atom_codes(W, [C|Rest]),
 1748          T = w(W)
 1749        }
 1750    ;   { char_code(T, C) }
 1751    ).
 1752
 1753word([C0|T]) -->
 1754    [C0],  { code_type(C0, alnum) },
 1755    !,
 1756    word(T).
 1757word([0'_, C1|T]) -->
 1758    [0'_, C1],  { code_type(C1, alnum) },
 1759    !,
 1760    word(T).
 1761word([]) -->
 1762    [].
 1763
 1764alphas([C0|T]) -->
 1765    [C0],  { code_type(C0, alpha) },
 1766    !,
 1767    alphas(T).
 1768alphas([]) -->
 1769    [].
 1770
 1771%!  verbatim(+Lines, +EnvIndent, -Pre, -RestLines) is det.
 1772%
 1773%   Extract a verbatim environment.  The  returned   Pre  is  of the
 1774%   format pre(Attributes, String). The indentation   of the leading
 1775%   fence is substracted from the indentation of the verbatim lines.
 1776%   Two types of fences are supported:   the  traditional =|==|= and
 1777%   the Doxygen =|~~~|= (minimum  3   =|~|=  characters), optionally
 1778%   followed by =|{.ext}|= to indicate the language.
 1779%
 1780%   Verbatim environment is delimited as
 1781%
 1782%     ==
 1783%       ...,
 1784%       verbatim(Lines, Pre, Rest)
 1785%       ...,
 1786%     ==
 1787%
 1788%   In addition, a verbatim environment may  simply be indented. The
 1789%   restrictions are described in the documentation.
 1790
 1791verbatim(Lines, _,
 1792         Indent-pre([class(code), ext(Ext)],Pre),
 1793         RestLines) :-
 1794    skip_empty_lines(Lines, [Indent-FenceLine|CodeLines]),
 1795    verbatim_fence(FenceLine, Fence, Ext),
 1796    verbatim_body(CodeLines, Indent, [10|PreCodes], [],
 1797                  [Indent-Fence|RestLines]),
 1798    !,
 1799    atom_codes(Pre, PreCodes).
 1800verbatim([_-[],Indent-Line|Lines], EnvIndent,
 1801         Indent-pre(class(code),Pre),
 1802         RestLines) :-
 1803    EnvIndent >= 0,
 1804    Indent >= EnvIndent+4, Indent =< EnvIndent+8,
 1805    valid_verbatim_opening(Line),
 1806    indented_verbatim_body([Indent-Line|Lines], Indent,
 1807                           CodeLines, RestLines),
 1808    !,
 1809    lines_code_text(CodeLines, Indent, [10|PreCodes]),
 1810    atom_codes(Pre, PreCodes).
 1811
 1812verbatim_body(Lines, _, PreT, PreT, Lines).
 1813verbatim_body([I-L|Lines], Indent, [10|Pre], PreT, RestLines) :-
 1814    PreI is I - Indent,
 1815    phrase(pre_indent(PreI), Pre, PreT0),
 1816    verbatim_line(L, PreT0, PreT1),
 1817    verbatim_body(Lines, Indent, PreT1, PreT, RestLines).
 1818
 1819verbatim_fence(Line, Fence, '') :-
 1820    Line == [0'=,0'=],
 1821    !,
 1822    Fence = Line.
 1823verbatim_fence(Line, Fence, Ext) :-
 1824    tilde_fence(Line, Fence, 0, Ext).
 1825verbatim_fence(Line, Fence, Ext) :-
 1826    md_fence(Line, Fence, 0, Ext).
 1827
 1828tilde_fence([0'~|T0], [0'~|F0], C0, Ext) :-
 1829    !,
 1830    C1 is C0+1,
 1831    tilde_fence(T0, F0, C1, Ext).
 1832tilde_fence(List, [], C, Ext) :-
 1833    C >= 3,
 1834    (   List == []
 1835    ->  Ext = ''
 1836    ;   phrase(tilde_fence_ext(ExtCodes), List)
 1837    ->  atom_codes(Ext, ExtCodes)
 1838    ).
 1839
 1840%!  tilde_fence_ext(-Ext)// is semidet.
 1841%
 1842%   Detect ```{.prolog} (Doxygen) or ```{prolog} (GitHub)
 1843
 1844tilde_fence_ext(Ext) -->
 1845    "{.", !, alphas(Ext), "}".
 1846tilde_fence_ext(Ext) -->
 1847    "{", alphas(Ext), "}".
 1848
 1849md_fence([0'`|T0], [0'`|F0], C0, Ext) :-
 1850    !,
 1851    C1 is C0+1,
 1852    md_fence(T0, F0, C1, Ext).
 1853md_fence(List, [], C, Ext) :-
 1854    C >= 3,
 1855    (   List == []
 1856    ->  Ext = ''
 1857    ;   phrase(md_fence_ext(ExtCodes), List),
 1858        atom_codes(Ext, ExtCodes)
 1859    ).
 1860
 1861% Also support Doxygen's curly bracket notation.
 1862md_fence_ext(Ext) -->
 1863    tilde_fence_ext(Ext),
 1864    !.
 1865% In Markdown language names appear without brackets.
 1866md_fence_ext(Ext) -->
 1867    alphas(Ext).
 1868
 1869%!  indented_verbatim_body(+Lines, +Indent, -CodeLines, -RestLines)
 1870%
 1871%   Takes more verbatim lines. The input   ends  with the first line
 1872%   that is indented less than Indent. There cannot be more than one
 1873%   consequtive empty line in the verbatim body.
 1874
 1875indented_verbatim_body([I-L|T0], Indent, [I-L|T], RestLines) :-
 1876    L \== [], I >= Indent,
 1877    !,
 1878    indented_verbatim_body(T0, Indent, T, RestLines).
 1879indented_verbatim_body([I0-[],I-L|T0], Indent, [I0-[],I-L|T], RestLines) :-
 1880    I >= Indent,
 1881    valid_verbatim_opening(L),
 1882    indented_verbatim_body(T0, Indent, T, RestLines).
 1883indented_verbatim_body(Lines, _, [], Lines).
 1884
 1885%!  valid_verbatim_opening(+Line) is semidet.
 1886%
 1887%   Tests that line does not look like a list item or table.
 1888
 1889valid_verbatim_opening([0'||_]) :- !, fail.
 1890valid_verbatim_opening(Line) :-
 1891    Line \== [],
 1892    \+ ( phrase(line_tokens(Tokens), Line),
 1893         list_item_prefix(_Type, Tokens, _Rest)
 1894       ).
 1895
 1896%!  lines_code_text(+Lines, +Indent, -Codes) is det.
 1897%
 1898%   Extract the actual code content from a list of line structures.
 1899
 1900lines_code_text([], _, []).
 1901lines_code_text([_-[]|T0], Indent, [10|T]) :-
 1902    !,
 1903    lines_code_text(T0, Indent, T).
 1904lines_code_text([I-Line|T0], Indent, [10|T]) :-
 1905    PreI is I-Indent,
 1906    phrase(pre_indent(PreI), T, T1),
 1907    verbatim_line(Line, T1, T2),
 1908    lines_code_text(T0, Indent, T2).
 1909
 1910
 1911%!  pre_indent(+Indent)// is det.
 1912%
 1913%   Insert Indent leading spaces.  Note we cannot use tabs as these
 1914%   are not expanded by the HTML <pre> element.
 1915
 1916pre_indent(N) -->
 1917    { N > 0,
 1918      !,
 1919      N2 is N - 1
 1920    }, " ",
 1921    pre_indent(N2).
 1922pre_indent(_) -->
 1923    "".
 1924
 1925verbatim_line(Line, Pre, PreT) :-
 1926    append(Line, PreT, Pre).
 1927
 1928
 1929                 /*******************************
 1930                 *            SUMMARY           *
 1931                 *******************************/
 1932
 1933%!  summary_from_lines(+Lines:lines, -Summary:list(codes)) is det.
 1934%
 1935%   Produce a summary for Lines. Similar  to JavaDoc, the summary is
 1936%   defined as the first sentence of the documentation. In addition,
 1937%   a sentence is also ended by an  empty   line  or  the end of the
 1938%   comment.
 1939
 1940summary_from_lines(Lines, Sentence) :-
 1941    skip_empty_lines(Lines, Lines1),
 1942    summary2(Lines1, Sentence0),
 1943    end_sentence(Sentence0, Sentence).
 1944
 1945summary2(_, Sentence) :-
 1946    Sentence == [],
 1947    !.              % we finished our sentence
 1948summary2([], []) :- !.
 1949summary2([_-[]|_], []) :- !.            % empty line
 1950summary2([_-[0'@|_]|_], []) :- !.       % keyword line
 1951summary2([_-L0|Lines], Sentence) :-
 1952    phrase(sentence(Sentence, Tail), L0, _),
 1953    summary2(Lines, Tail).
 1954
 1955sentence([C,End], []) -->
 1956    [C,End],
 1957    { \+ code_type(C, period),
 1958      code_type(End, period)                % ., !, ?
 1959    },
 1960    space_or_eos,
 1961    !.
 1962sentence([0' |T0], T) -->
 1963    space,
 1964    !,
 1965    ws,
 1966    sentence(T0, T).
 1967sentence([H|T0], T) -->
 1968    [H],
 1969    sentence(T0, T).
 1970sentence([0' |T], T) -->                % '
 1971    eos.
 1972
 1973space_or_eos -->
 1974    [C],
 1975    !,
 1976    {code_type(C, space)}.
 1977space_or_eos -->
 1978    eos.
 1979
 1980%!  skip_empty_lines(+LinesIn, -LinesOut) is det.
 1981%
 1982%   Remove empty lines from the start of the input.  Note that
 1983%   this is used both to process character and token data.
 1984
 1985skip_empty_lines([], []).
 1986skip_empty_lines([_-[]|Lines0], Lines) :-
 1987    !,
 1988    skip_empty_lines(Lines0, Lines).
 1989skip_empty_lines(Lines, Lines).
 1990
 1991end_sentence([], []).
 1992end_sentence([0'\s], [0'.]) :- !.
 1993end_sentence([H|T0], [H|T]) :-
 1994    end_sentence(T0, T).
 1995
 1996
 1997                 /*******************************
 1998                 *        CREATE LINES          *
 1999                 *******************************/
 2000
 2001%!  indented_lines(+Text:list(codes), +Prefixes:list(codes),
 2002%!                 -Lines:list) is det.
 2003%
 2004%   Extract a list of lines  without   leading  blanks or characters
 2005%   from Prefix from Text. Each line   is a term Indent-Codes, where
 2006%   Indent specifies the line_position of the real text of the line.
 2007
 2008indented_lines(Comment, Prefixes, Lines) :-
 2009    must_be(codes, Comment),
 2010    phrase(split_lines(Prefixes, Lines), Comment),
 2011    !.
 2012
 2013split_lines(_, []) -->
 2014    end_of_comment.
 2015split_lines(Prefixes, [Indent-L1|Ls]) -->
 2016    take_prefix(Prefixes, 0, Indent0),
 2017    white_prefix(Indent0, Indent),
 2018    take_line(L1),
 2019    split_lines(Prefixes, Ls).
 2020
 2021
 2022%!  end_of_comment//
 2023%
 2024%   Succeeds if we hit the end of the comment.
 2025%
 2026%   @bug    %*/ will be seen as the end of the comment.
 2027
 2028end_of_comment -->
 2029    eos.
 2030end_of_comment -->
 2031    ws, stars, "*/".
 2032
 2033stars --> [].
 2034stars --> "*", !, stars.
 2035
 2036
 2037%!  take_prefix(+Prefixes:list(codes), +Indent0:int, -Indent:int)// is det.
 2038%
 2039%   Get the leading characters  from  the   input  and  compute  the
 2040%   line-position at the end of the leading characters.
 2041
 2042take_prefix(Prefixes, I0, I) -->
 2043    { member(Prefix, Prefixes),
 2044      string_codes(Prefix, PrefixCodes)
 2045    },
 2046    prefix(PrefixCodes),
 2047    !,
 2048    { string_update_linepos(PrefixCodes, I0, I) }.
 2049take_prefix(_, I, I) -->
 2050    [].
 2051
 2052prefix([]) --> [].
 2053prefix([H|T]) --> [H], prefix(T).
 2054
 2055white_prefix(I0, I) -->
 2056    [C],
 2057    {  code_type(C, white),
 2058       !,
 2059       update_linepos(C, I0, I1)
 2060    },
 2061    white_prefix(I1, I).
 2062white_prefix(I, I) -->
 2063    [].
 2064
 2065%!  string_update_linepos(+Codes, +Pos0, -Pos) is det.
 2066%
 2067%   Update line-position after adding Codes at Pos0.
 2068
 2069string_update_linepos([], I, I).
 2070string_update_linepos([H|T], I0, I) :-
 2071    update_linepos(H, I0, I1),
 2072    string_update_linepos(T, I1, I).
 2073
 2074%!  update_linepos(+Code, +Pos0, -Pos) is det.
 2075%
 2076%   Update line-position after adding Code.
 2077%
 2078%   @tbd    Currently assumes tab-width of 8.
 2079
 2080update_linepos(0'\t, I0, I) :-
 2081    !,
 2082    I is (I0\/7)+1.
 2083update_linepos(0'\b, I0, I) :-
 2084    !,
 2085    I is max(0, I0-1).
 2086update_linepos(0'\r, _, 0) :- !.
 2087update_linepos(0'\n, _, 0) :- !.
 2088update_linepos(_, I0, I) :-
 2089    I is I0 + 1.
 2090
 2091%!  take_line(-Line:codes)// is det.
 2092%
 2093%   Take  a  line  from  the  input.   Line  does  not  include  the
 2094%   terminating \r or \n character(s), nor trailing whitespace.
 2095
 2096take_line([]) -->
 2097    "\r\n",
 2098    !.                      % DOS file
 2099take_line([]) -->
 2100    "\n",
 2101    !.                        % Unix file
 2102take_line(Line) -->
 2103    [H], { code_type(H, white) },
 2104    !,
 2105    take_white(White, WT),
 2106    (   nl
 2107    ->  { Line = [] }
 2108    ;   { Line = [H|White] },
 2109        take_line(WT)
 2110    ).
 2111take_line([H|T]) -->
 2112    [H],
 2113    !,
 2114    take_line(T).
 2115take_line([]) -->                       % end of string
 2116    [].
 2117
 2118take_white([H|T0], T) -->
 2119    [H],  { code_type(H, white) },
 2120    !,
 2121    take_white(T0, T).
 2122take_white(T, T) -->
 2123    [].
 2124
 2125%!  normalise_indentation(+LinesIn, -LinesOut) is det.
 2126%
 2127%   Re-normalise the indentation, such that the  lef-most line is at
 2128%   zero.  Note that we skip empty lines in the computation.
 2129
 2130normalise_indentation(Lines0, Lines) :-
 2131    skip_empty_lines(Lines0, Lines1),
 2132    Lines1 = [I0-_|Lines2],
 2133    !,
 2134    smallest_indentation(Lines2, I0, Subtract),
 2135    (   Subtract == 0
 2136    ->  Lines = Lines0
 2137    ;   maplist(substract_indent(Subtract), Lines0, Lines)
 2138    ).
 2139normalise_indentation(Lines, Lines).
 2140
 2141smallest_indentation([], I, I).
 2142smallest_indentation([_-[]|T], I0, I) :-
 2143    !,
 2144    smallest_indentation(T, I0, I).
 2145smallest_indentation([X-_|T], I0, I) :-
 2146    I1 is min(I0, X),
 2147    smallest_indentation(T, I1, I).
 2148
 2149substract_indent(Subtract, I0-L, I-L) :-
 2150    I is max(0,I0-Subtract).
 2151
 2152
 2153                 /*******************************
 2154                 *             MISC             *
 2155                 *******************************/
 2156
 2157%!  strip_leading_par(+Dom0, -Dom) is det.
 2158%
 2159%   Remove the leading paragraph for  environments where a paragraph
 2160%   is not required.
 2161
 2162strip_leading_par([p(C)|T], L) :-
 2163    !,
 2164    append(C, T, L).
 2165strip_leading_par(L, L).
 2166
 2167
 2168                 /*******************************
 2169                 *           DCG BASICS         *
 2170                 *******************************/
 2171
 2172%!  ws// is det
 2173%
 2174%   Eagerly skip layout characters
 2175
 2176ws -->
 2177    [C], {code_type(C, space)},
 2178    !,
 2179    ws.
 2180ws -->
 2181    [].
 2182
 2183%       space// is det
 2184%
 2185%       True if then next code is layout.
 2186
 2187space -->
 2188    [C],
 2189    {code_type(C, space)}.
 2190
 2191%!  nl//
 2192%
 2193%   Get end-of-line
 2194
 2195nl -->
 2196    "\r\n",
 2197    !.
 2198nl -->
 2199    "\n".
 2200
 2201%!  peek(H)//
 2202%
 2203%   True if next token is H without eating it.
 2204
 2205peek(H, L, L) :-
 2206    L = [H|_].
 2207
 2208peek(H1, H2, L, L) :-
 2209    L = [H1, H2|_].
 2210
 2211%!  tokens(-Tokens:list)// is nondet.
 2212%!  tokens(+Max, -Tokens:list)// is nondet.
 2213%
 2214%   Defensively take tokens from the input.  Backtracking takes more
 2215%   tokens.  Do not include structure terms.
 2216
 2217tokens([]) --> [].
 2218tokens([H|T]) --> token(H), tokens(T).
 2219
 2220tokens(_, []) --> [].
 2221tokens(C, [H|T]) --> token(H), {succ(C1, C)}, tokens(C1, T).
 2222
 2223%!  tokens_no_whitespace(-Tokens:list(atom))// is nondet.
 2224%
 2225%   Defensively take tokens from the  input. Backtracking takes more
 2226%   tokens.  Tokens  cannot  include  whitespace.  Word  tokens  are
 2227%   returned as their represented words.
 2228
 2229tokens_no_whitespace([]) -->
 2230    [].
 2231tokens_no_whitespace([Word|T]) -->
 2232    [ w(Word) ],
 2233    !,
 2234    tokens_no_whitespace(T).
 2235tokens_no_whitespace([H|T]) -->
 2236    [H],
 2237    { \+ space_token(H) },
 2238    tokens_no_whitespace(T).
 2239
 2240token(Token) -->
 2241    [Token],
 2242    { token(Token) }.
 2243
 2244token(w(_)) :- !.
 2245token(Token) :- atom(Token).
 2246
 2247%!  limit(+Count, :Rule)//
 2248%
 2249%   As limit/2, but for grammar rules.
 2250
 2251:- meta_predicate limit(+,2,?,?). 2252
 2253limit(Count, Rule, Input, Rest) :-
 2254    Count > 0,
 2255    State = count(0),
 2256    call(Rule, Input, Rest),
 2257    arg(1, State, N0),
 2258    N is N0+1,
 2259    (   N =:= Count
 2260    ->  !
 2261    ;   nb_setarg(1, State, N)
 2262    ).
 2263
 2264
 2265                 /*******************************
 2266                 *           MESSAGES           *
 2267                 *******************************/
 2268
 2269:- multifile
 2270    prolog:message//1. 2271
 2272prolog:message(pldoc(deprecated_tag(Name, Tag))) -->
 2273    [ 'PlDoc: Deprecated tag @~w (use @~w)'-[Name, Tag]
 2274    ].
 2275prolog:message(pldoc(unknown_tag(Name))) -->
 2276    [ 'PlDoc: unknown tag @~w'-[Name]
 2277    ]