(setq *package* 'sxp)
;;============================================================================
;;The SXP package - Simple XML parser
;;============================================================================
;;AUTOLOAD: sxp:parse
;;AUTODOC: sxp:parse "parse an XML file into nested lists"
;;sxp:parse
;;---------
;;(sxp:parse buffer [:nil-error t] [:filename name] [:ignore-entities t]
;;                  [:forest t] [:entities plist] [:keep-whitespace t]
;;                  [:whitespace-keeping-tags taglist]
;;                  [:keep-comments t] [:keep-PIs t] [:keep-DTD t]
;;                  [:external-entities plist]
;;                  [:notation-parsers plist]
;;                  [:notation-default-parser function]
;;)
;;
;; Parses a string buffer (or a stream, which will be completly read) 
;; containing an XML document, and returns a compact parsed tree in memory 
;; in the form of nested vectors:
;;    #(element-type children-list attributes-plist) 
;;    or #(element-type children-list)
;;    or #(element-type)
;;    or string      (for character data)
;;    or #(element-type children-list attributes-plist application-data...)
;;
;; This representation was chosen as it should consume the least memory in 
;; most cases, and be easy to handle. Vectors are also safer than lists as they
;; evaluate to themselves. element-type is an atom of the tag of the element.
;; Applications are free to append private data at the end, so functions 
;; dealing with these stuctures should try to preserve those private infos.
;;
;; Parsing a file of name "filename" would thus be done by:
;;     (sxp:parse (open "filename") :filename "filename")
;;
;; It basically converts a string containing
;;     <elt alt1="val1" alt2="val2"> child1 child2 </elt>
;; into a returned recursive vector:
;;     #(elt (child1 child2) (:alt1 "val1" :alt2 "val2"))
;; Terminal Elements are strings, thus corresponding to character data
;; Element names (tags) and attribute names are atoms, to share the data
;;
;; In case of parse error, triggers an error, then throws the exception
;; sxp:Error, except if the keyword :nil-error was given, in which case the 
;; error is printed but the parsers just returns nil
;;
;; :filename is there just to specify the filename for reference in error 
;; messages
;;
;; If :forest is set to t, sxp:parse is able to parse a succession of xml tree,
;; and returns a list of all parsed trees (a "forest"). By default, only the 
;; last one is returned.
;;
;; :entities, with as argument a p-list or a Hashtable of entities names and
;; replacement values can be used to declare general entities to be pre-loaded
;; into the parser, e.g.:
;; (sxp:parse b :entities '("eacute" "é" "egrave" "è"))
;; These preloaded entities will override  any entity with the same 
;; name declared in the document DTD
;;
;; :external-entities with as argument a p-list or a Hashtable of entities 
;; names and file paths values can be used to declare external entities to 
;; be pre-loaded into the parser, e.g.:
;; (sxp:parse b :external-entities '("hosts" "/etc/hosts"))
;; Currently, only the local and file: protocols are implemented
;; These preloaded entities will override any entity with the same 
;; name declared in the document DTD
;; If a notation is present (NDATA notation), a handler is searched for
;; in the plist argument to :notation-parsers, and applied to the 4 arguments:
;;   the notation, a string such as "image/png", "audio/mp3"...
;;   the contents of the external file as a string
;;   the pathname of the file
;;   the current node to which the returned result will be appended
;; If no handler is given, and a :notation-default-parser is provided, it is
;; used instead, otherwise the default one is called, which just copies the
;; contents as chardata.
;;
;; if :ignore-entities is set to t, entities are left unparsed
;;
;; This parser is targeted at data representations, not full text and thus
;; has the following limits (that could be lifted at a later date):
;;  - only occidental characters: 8bit ISO-8859-1 encoding 
;;    (can be specified as UTF-8, but slower than native ISO-8859-1 encoding)
;;  - parses a complete string in memory, cannot handle gigabytes of data
;;  - does not handle entities other than the mandatory ones 
;;    (&lt;&gt;&amp;&apos;&quot;), and the numeric character code ones:
;;    &#64; &#x40;
;;    It does handle internal entities, though
;;  - does not validate the DTD (in current version skips the DTD)
;;
;; Text contents are converted to raw klone string objects.
;;
;; Comments, Processing Instructions, and DTDs are discarded by default
;; They can be parsed and incorporated in the parse tree with the optional
;; keywords:
;;  - :keep-comments t
;;    comments inserted in the parse tree as nodes of element names !
;;    with no attributes and with contents the raw comment itself as a string,
;;    without the preceding <!-- and closing --> but with spaces.
;;    Parsing <!-- a comment--> gives: #(! #(" a comment"))
;;  - :keep-PIs t
;;    PIs are kept in the parse tree just like comments, with element name ?
;;    and content excluding the <? and ?>
;;  - :keep-DTD t
;;    The DTD, if present, the DTD will be stored in the global var sxp:DTD
;;    as an unparsed string. Otherwise the contents of sxp:DTD is untouched to
;;    allow easy sub-parsing of XML fragments
;;
;; Although the DTD is not really parsed, it is scanned naively for entity 
;; declarations, so that general and external entities are understood.
;; However, internal entities (%...; form) are ignored, as non-local-file
;; external entities (http and ftp)
;;
;; Return values:
;; The parsed tree is returned, and the additional global variables are set:
;;  - sxp:encoding  sets to the string of the encoding of the parsed document
;;    Can only be "UTF-8" or "ISO-8859-1" in the current version, there is no 
;;    way to process 16-bit chars.
;;  - sxp:standalone set to (), "yes", or "no"
;; NOTE: you should always sepcify the "ISO-8859-1" in your data files, as
;; by default, xml are to be handled as UTF-8, which is slower.
;;
;; Pretty printing an XML file can thus be done by:
;; (sxp:print (sxp:parse (open "filename") :keep-comments t :keep-PIs t
;;  :keep-DTD t :ignore-entities t) :ignore-entities t)
;;
;; This parser, however, is quite fast (faster than most java parsers!), 
;; consumes little memory for the parsed tree, and should be very easy to use.
;;
;; As this parser is meant to parse data file, and not general text, 2 useful
;; optimizations are performed:
;;  - consecutive string children are concatenated, i.e.:
;;    <a>x&lt;y</a> gives (a ("x<y")), not (a ("x" "<" "y"))
;;  - if a node has only white space or sub-nodes, white spaces are removed
;;    <a> <b/> <c/> </a> gives #(a (#(b)#(c))), 
;;    not #(a (" " #(b) " " #(c) " ")),
;;    but <a> Hello, World! </a> gives #(a (" Hello, World! "))
;;    This will allow to pretty-print "data" XML files, but still yield the
;;    same internal representation
;;    if :keep-whitespace is set to t, this optimisation is disabled
;;    you can also disble it element by element by giving a taglist with the
;;    keyword :whitespace-keeping-tags. xhtml processors will want to provide
;;    :whitespace-keeping-tags '(pre) for instance
;;    sxp:keep-whitespace-tags is a plist of tags that keep whitespace
;;    (with value t), that you can modify dynamically by PI hooks if you want
;;
;; By default, processing instructions (PIs) are ignored, but can be processed
;; by defining a function sxp:parse-PI-hook taking as argument the whole 
;; string content inside the PI. On meeting <?palette "blue" ?>, sxp:parse
;; would thus call (sxp:parse-PI-hook "palette \"blue\" "
;;
;;e.g:
;; (sxp:parse "<book format=\"A5\"><title>SML reference manual</title></book>")
;; ==>  #(book (#(title ("SML reference manual"))) (:format "A5"))
;;
;; (sxp:parse "<supported-formats><jpeg/><gif/><png/></supported-formats>")
;; ==>  #(supported-formats (#(jpeg) #(gif) #(png)))
;;
;; (sxp:parse "<author>Colas <name>Nahaboo</name></author>")
;; ==>  #(author ("Colas " #(name "Nahaboo")))
;;
;; Tree structure handling:
;; ------------------------
;; Try not to access the structures in memory directly, but use the
;; functions of the xnl package: (XML Node Lists):
;; xnl:new xnl:name xnl:children xnl:attributes xnl:chardata? xnl:set-child
;; xnl:insert-child xnl:delete-child xnl:set-attribute xnl:delete-attribute
;; xnl:get-child xnl:get-attribute xnl:put-child

;;AUTOLOAD: xnl:new
;;AUTODOC: xnl:new create an XNL node name
;;AUTOLOAD: xnl:name
;;AUTODOC: xnl:name get/set node name
;;AUTOLOAD: xnl:children
;;AUTODOC: xnl:children get/set node children
;;AUTOLOAD: xnl:attributes
;;AUTODOC: xnl:attributes get/set node attributes p-list
;;AUTOLOAD: xnl:chardata?
;;AUTODOC: xnl:chardata? tests if node is a string
;;AUTOLOAD: xnl:set-child
;;AUTODOC: xnl:set-child sets a child of a node
;;AUTOLOAD: xnl:insert-child
;;AUTODOC: xnl:insert-child insert a child into a node
;;AUTOLOAD: xnl:delete-child
;;AUTODOC: xnl:delete-child delete a child from a node
;;AUTOLOAD: xnl:set-attribute
;;AUTODOC: xnl:set-attribute sets a node attribute
;;AUTOLOAD: xnl:delete-attribute
;;AUTODOC: xnl:delete-attribute delete a node attribute
;;AUTOLOAD: xnl:get-child
;;AUTODOC: xnl:get-child get a child of a node
;;AUTOLOAD: xnl:get-attribute
;;AUTODOC: xnl:get-attribute get an attribute value of a node
;;AUTOLOAD: xnl:put-child
;;AUTODOC: xnl:put-child synonym for xnl:set-child
;;
;; This is a simplified set of accessors to get or set fields of nodes
;; With only the node argument, they get the field, and with a value it sets it
;; the xnl: (XML Nested Lists interface)
;; (xnl:name node [name])       get/set node name (atom, not string)
;; (xnl:children node [children-list])       get/set node children (a list)
;; (xnl:attributes node [attributes-plist])  get/set node attributes p-list
;;                              attributes keys are keywords
;; (xnl:new [name [children-list [attributes-plist]]])  constructor
;; (xnl:chardata? node)         predicate, returns true if node is just 
;;                              chardata (a string)
;; (xnl:get-child node position) return child at position or () if not found
;; (xnl:set-child node position child) puts child at position (Integer)
;;                              position () or -1 ==> append
;;                              xnl:put-child is a synonym
;;                              will work even if children list is ()
;; (xnl:insert-child node position child) inserts child at position
;;                              position () or -1 ==> append
;;                              will work even if children list is ()
;; (xnl:delete-child node position)  deletes child at position
;; (xnl:get-attribute node key) returns value (string) of attribute (keyword)
;;                              or () if not found
;; (xnl:set-attribute node key value) sets attribute of name key (Keyword)
;;                              and value (String)
;;                              will work even if attributes list is ()
;; (xnl:delete-attribute node key) deletes attributes of name key, do nothing
;;                              if not found
;;
;; WARNING: this code will fail: (lappend (xnl:children node) child), as if
;; node has no children, (xnl:children node) will return (), that you cannot 
;; modify. Use xnl:set-child or xnl:set-attribute for this,
;; e.g: (xnl:set-child node child 0)

;;AUTOLOAD: sxp:load
;;AUTODOC: sxp:load "loads an XML file in memory as nested lists"
;; sxp:load
;;---------
;;(sxp:load filename)
;;
;; will just perform a (sxp:parse (open filename))

;;AUTOLOAD: sxp:print
;;AUTODOC: sxp:print "prints an XML file from nested lists"
;;sxp:print
;;---------
;;(sxp:print tree stream [:raw t] [ :encoding ("UTF-8"|"ISO-8859-1") ]
;;                       [:ignore-entities t] [:forest t]
;;
;; The opposite of sxp:parse: from a nested list representation in memory,
;; outputs an XML representation on the stream (default: standard output)
;;
;; (sxp:print #(book (#(title ("SML reference manual"))) (:format "A5")) ())
;; prints on stdout:
;; "<book format=\"A5\"><title>SML reference manual</title></book>"
;;
;; This representation is pretty-printed and indented, each level being 
;; indented by the redefinable string sxp:print-indent-string
;; (2 spaces by default)
;; Setting sxp:print-indent-string to () will disable any indentation
;; Only nodes without any character data (only subnodes) are indented to not 
;; introduce parasitic white space
;;
;; If :raw is t, dont print PI header nor terminating newline (useful
;; for printing XML subparts).
;;
;; Encoding can be the strings "UTF-8" or "ISO-8859-1" (default) to specify 
;; the encoding used in the output
;;
;; The lines will be wrapped, if possible, to fit in sxp:width (default 78)
;;
;; WARNING: if sxp:DTD is non empty, its contents will be used to print the 
;; DTD (see sxp:parse)
;;
;; if :ignore-entities is t, the & in the char datas and attribute values 
;; are not quoted, to be able to print trees read with sxp:parse with
;; :ignore-entities  t
;;
;; Setting :forest to t enables you to print a list of nodes, as returned by 
;; parsing with the :forest keyword

;;AUTOLOAD: sxp:eval
;;AUTODOC: sxp:eval "evaluates nested lists representing an XML syntax tree"
;;sxp:eval
;;--------
;;(sxp:eval tree [prefix])
;;(sxp:eval tree '(tag1 lambda1 tag2 lambda2...))
;;
;; Performs parsing of the tree via a simple mecanism: each node of the type:
;; (element-type children-list attributes-plist) is "evaluated" by executing
;; the call to:
;;    (prefix:element-type tag-name list-of-evaluated-children attributes...)
;; Implementing a parser is thus defining all the necessary prefix:element-type
;; functions with simple defuns
;; Prefix can be omitted and defaults to "sxe" (Simple XML Evaluator)
;; Alternatively, prefix can be a p-list of element names and evaluation 
;; functions, elements not in the p-list being passed along.
;;
;; If a function name  prefix:element-type is not defined (or defined to ()),
;; the element is passed along unmodified, unless a "catch-all" function is 
;; defined either as prefix: (first form), or as a lambda associated to the 
;; tag () (second form)
;; 
;; It works depth-first, evaluating firts the leaves, then walking up to root
;; the return value of your functions will be used as arguments to the other 
;; functions, so must be valid nodes. As it builds first a function call (list)
;; and then call it, you can make a top-first evaluation by using defunq 
;; instead of defun for your functions, and thus evaluating yourself the 
;; arguments at will
;;
;;e.g: 
;;    (sxp:eval #(book (#(title ("SML reference manual"))) (:format "A5"))
;;              "my")
;;    Will evaluate the expression:
;;    (my:book 'book (list (my:title 'title (list "SML reference manual")))
;;             :format "A5")
;;    That you will have to implement as:
;;    (defun my:book (tag children &key format)
;;	   ...
;;    )
;;    if both my:book and my:title are defined. If only my:book is defined, it
;;    will construct the list:
;;    (my:book 'book (list #(title ("SML reference manual"))) :format "A5")
;;    and then eval it.
;;    And, alternatively, you could say:
;;    (setq eval-funcs (list 'book (lambda (tag children &key format) ...)))
;;    (sxp:eval #(book (#(title ("SML reference manual"))) (:format "A5"))
;;              eval-funcs)

;;AUTOLOAD: sxp:donode
;;AUTODOC: sxp:donode "walk the nested lists representing an XML syntax tree"
;;sxp:donode
;;--------
;;(sxp:donode (variable tree [[chardata-form] depth-first?]) forms...)
;;
;; sxp:donode is a "do" function (a la do, dolist, dohash, dotimes...) to 
;; recursively execute the klone statements "forms" on each of the node of 
;; the tree, with node being the value of variable.
;; A global stack of nodes is available as sxp:donode-stack holding the list
;; of all nodes traversed from top (parents, starting from top) to reach the 
;; current node.
;; Only composite nodes (vectors) are treated, strings (chardata nodes) are 
;; handled by the optional form chardata-form
;; The return value of forms are ignored, all functionalities must be 
;; implemented by side effects.
;; The tree is traversed topfirst (root then children), unless the optional 
;; depth-first? argument is non-nil, in which case depth-first order is used.
;;
;; As is all "do" functions, it is dangerous to modify the list/tree being
;; walked on. Donode implements a set of conventions, so that it can perform
;; deletion/insertion of nodes in a cleanup pass after your forms are 
;; evaluated:
;;   * to delete the current node, replace it by (), e.g.:
;;       (replace-list node ()) 
;;   * to insert nodes, replace the node (by replace-list) by a node of name ()
;;     and with children the list of children to insert at this place,
;;     including itself if needed. donode will not recurse in the inserted 
;;     nodes
;;     e.g: to insert <foo/> before the current node:
;;       (replace-list node (xnl:new () (list (xnl:new 'foo) (copy node))))
;;       You need to copy the node not to create a circular list!
;;   * to replace current node by character data (a string "foo") replace the
;;     node by [() "foo"]
;; Note that you can call yourself the cleanup function on an xml tree by
;; (sxp:clean-tree tree)
;;
;; Exemple: a simple pretty-printer:
;; 
;; (sxp:donode (node tree 
;;     (PF "  = %1%r0\n" node (make-string (length sxp:donode-stack)))
;;   )
;;   (PF "==> %1<%0" (xnl:name node)
;;     (make-string (length sxp:donode-stack))
;;   )
;;   (dohash (key val (xnl:attributes node))
;;     (PF " %0=%r1" (subseq key 1) val)
;;   )
;;   (PF ">\n")
;; )

;;============================================================================
;;
;;HISTORY
;;=======
;;
;;by Colas Nahaboo <colas@sophia.inria.fr>
;;
;; TODO:
;;   - option to add to nodes either the raw text contents or the start/end
;;     positions of it in input stream
;;   - be able to handle 16-bits chars (much harder, as we cannot use regexps)
;;
;;v1.17 2000-Nov-03
;;    recognize the "standalone" declaration in xml decl
;;    sxp:print takes :forest keyword
;;v1.16 2000-Nov-02
;;    obeys the xml:space attribute, only on elements, not in DTD
;;v1.15 2000-Oct-11
;;    bugfix (introduced in 1.14): DTD could be ignored
;;    built-in entities definitions for well-known DTDs. 
;;        only one at the moment: html
;;    bugfix: encoding processing was case-sensitive
;;
;;v1.14 2000-Oct-09
;;    bugfix: entities provided as arguments were ignored if no DTD was present
;;
;;v1.13 2000-Sept-16
;;    :keep-whitespace and :whitespace-keeping-tags keywords
;;    sxp:keep-whitespace-tags internal plist of tags
;;    :keep-comments :keep-PIs :keep-DTD work
;;    sxp:print uses sxp:DTD
;;    sxp:print uses sxp:width 
;;    sxp:print syntax changed! uses keywords instead of optional params
;;    :ignore-entities keyword for sxp:parse sxp:print
;;    :external-entities keyword for sxp:parse, we also understand notations
;;    :notation-parsers, :notation-default-parser
;;    bugfix: outputting in utf-8 didnt work
;;
;;v1.12 1999-Sept-16
;;    UTF-8 encoding supported by converting first the buffer to be parsed
;;    into 8-bit ISO-8859-1 encoding, so it is slower than parsing ISO-8859-1
;;    documents.
;;    The chars must still be in the iso-latin-1 range, however. Attemps
;;    to read japanese will raise an error.
;;    WARNING: this requires the external functions in file utf8.kl
;;
;;v1.11 1999-July-3
;;    sxp:parse: new keyword :entities
;;
;;v1.10 1999-May-10
;;    Bug fix: trailing comments would make sxp:parse return ()
;;
;;v1.9 1999-Apr-30
;;    Simple entity parsing: only genral entities, of the form:
;;    <!ENTITY name "value">
;;    We do not parse yet external, parameter, and unparsed entities
;;v1.8 1999-Mar-26
;;    more explicit error reporting
;;    :forest keyword for sxp:parse
;;    WARNING: donode interface for modifying nodes changed:
;;                         before               now
;;        deleting node  [() children attribs]  ()
;;        new children   [children]             [() children]
;;        to string      not possible           [() string]
;;
;;v1.7 1999-Mar-19
;;    Bug fix: parsing an element with only whitespace contents will return
;;      and empty list of children, as was specified in the spec
;;
;;v1.6 1999-Feb-25
;;    sxp:donode
;;
;;v1.5 1999-Feb-22
;;    setqn defined if not there
;;    simpler mini-DOM in the xnl: package:
;;    xnl:name xnl:children xnl:attributes xnl:new xnl:chardata?

;;v1.4 1999-Feb-08
;;    sxp:E:EndTagDontMatch error now prints line of starting tag also
;;    bug fix: entity expansion was corrupting itself after a while
;;    new keyword :keep-comments to sxp:parse
;;    new keyword :filename to sxp:parse
;;    new "raw" optional arg to sxp:print
;;    format of error messages compatible with emacs parsing
;;
;;v1.3 1999-Feb-05
;;    implementation of nested lists slightly changed: elements are vectors, 
;;      no more lists, code handling them should be inchanged
;;    CDATA is now parsed ( <![CDATA[ ... ]]> )
;;    simple sxp:load, new convenience function
;;    entities are now expanded in attribute values
;;    all nested lists access functions documented (see sxp:parse doc)
;;    sxp:print indentation disablable
;;    sxp:eval implemented, with more functionalities:
;;       - no need to define all rules
;;       - alternate syntax with a p-list of functions rather than defuns
;;    sxp:parse-PI-hook to parse Processing Instructions
;;
;;v1.2 1998-Dec-18
;;    sxp:print now indents its output: try (sxp:print (sxp:load "file.xml"))
;;      to pretty-print XML files
;;    element names are now atoms instead of strings to conserve memory, 
;;      at the expense of some speed (we are fast enough)
;;
;;v1.1 1998-Dec-07
;;    Implementation of the nested lists in memory changed
;;    sxp:parse, sxp:print work (still primitive, but quite usable)
;;    sxp:eval not implemented
;;    definition of the sxp:node API
;;    a hack is done to silently ignore DOCTYPE
;;
;;v1.0 1998-Nov-30
;;    First spec: non validating, sxp:parse, sxp:print, sxp:eval

;;=============================================================================
;;                    Definitions
;;=============================================================================

;;=============================================================================
;;                    Errors
;;=============================================================================
;; define here functions that may be missing in some installations
(if (not (boundp 'setqn))
  (defunq setqn (&rest setqn:args)
    (dohash (setqn:var setqn:val setqn:args) (set setqn:var (eval setqn:val))))
)

(setqn					;Errors (fatal)
  sxp:E:BadDecl "Bad XML Declaration: %0"
  sxp:E:UnknownEntity "Unknown XML Entity: %0"
  sxp:E:MalformedEntity "Malformed Entity: %0"
  sxp:E:UnterminatedComment "Unterminated Comment"
  sxp:E:UnterminatedCDATA "Unterminated CDATA"
  sxp:E:UnterminatedPI "Unterminated Processing Instruction"
  sxp:E:UnterminatedAttribute "Unterminated Attribute %0"
  sxp:E:BadElementName "Bad element name: %r0"
  sxp:E:BadAttribute "Bad attribute"
  sxp:E:BadAttributeValue "no < allowed in attribute values"
  sxp:E:EndTagDontMatch "End tag \"/%1\" does not match start tag \"%0\" at line %2, col %3"
  sxp:E:BadEndTag "Bad end tag"
  sxp:E:EndTagMissing "End tag missing for: %0"
  sxp:E:DTDMisplaced "DTD cannot happen here"
  sxp:E:InternalError "Internal error in node structures: %0"
  sxp:E:ForestError "Resulting XML was not a single-rooted tree, but a forest!\n"
  sxp:E:ReadFile "Cannot read file: %0"
  sxp:E:BadXmlSpace "Bad xml:space attribute value: %0 (must be preserve or default"
)

(setqn					;Unimplemented features
  sxp:U:DTD "DTD parsing not implemented"
  sxp:U:Encoding "Unsupported Encoding: %0"
  sxp:U:NonLatin1Chars "Non Latin-1 (lower 256 ISO chars) character unsupported: %0 (may be you forgot the encoding=\"ISO-8859-1\" in the xml header?)"
  sxp:U:Protocol "Unsupported Protocol in external url: %0"
)

(setqn					;Warnings (non-fatal)
  sxp:W:BadVersion "Unknown XML version: %0"
)
;;============================================================================

(defun sxp:fatal-error (err &rest args)
  (sxp:error-header "ERROR")
  (apply print-format (+ (list *standard-error* err) args))
  (print-format *standard-error* "\n")
  (throw 'sxp:Error)
)

(defun sxp:warning (err &rest args)
  (sxp:error-header "WARNING")
  (apply print-format (+ (list *standard-error* err) args))
  (print-format *standard-error* "\n")
)

(defun sxp:parser-error (err &rest args)
  (sxp:error-header "UNIMPLEMENTED")
  (apply print-format (+ (list *standard-error* err) args))
  (print-format *standard-error* "\n")
  (throw 'sxp:Error)
)

(defun sxp:line-or-col-of-pos (p return-line? &aux
    (re (regcomp "^[^\n]*\n"))
    (c 0)
    (line 1)
    (col 1)
  )
  (catch 'Done
    (while (regexec re buffer c)
      (if (> (1 (getn re 0)) p) (throw 'Done
	  (setq col (+ 1 (- p c)))
      ))
      (setq c (1 (getn re 0)))
      (incf line)
    )
    (if (< c p)) (setq col (+ 1 (- p c)))
  )
  (setq sxp:line-or-col-of-pos-result (list line col))
  (if return-line? line col)
)

(defun sxp:line-of-pos (p) (sxp:line-or-col-of-pos p t))
(defun sxp:col-of-pos (p) (sxp:line-or-col-of-pos p ()))

(defun sxp:error-header (type)
  (if (and (boundp 'sxp:filename) sxp:filename)
    (print-format *standard-error* "\"%0\", " sxp:filename))
  (print-format *standard-error* "line %1: col %2: XML %0: " type 
    (sxp:line-or-col-of-pos pos t) (1 sxp:line-or-col-of-pos-result)
))
  
(defun sxp:fatal-error-node (err &rest args)
  (apply print-format (+ (list *standard-error* err) args))
  (print-format *standard-error* "\n")
  (throw 'sxp:Error)
)

(defun sxp:find-potential-name (buffer pos &aux
    (re-empty (regcomp "^[ \t\r\n]*[/]>"))
  )
  (if (regexec re-empty buffer pos)
    ""
    (subseq buffer pos (+ pos 20))
  )
)

;;=============================================================================
;;                    regexps
;;=============================================================================
;; lexical definitions: help in parsing tokens
(setq sxp:spaces "[ \t\n\r]*")
(setq sxp:space "[ \t\n\r]+")
(setq sxp::re-spaces (regcomp "^[ \t\n\r]*"))
(setq sxp::re-space  (regcomp "^[ \t\n\r]+"))
(setq sxp:quoteddata "(\"[^\"]*\"|'[^']*')")

(setq sxp:name "([A-Za-z][-A-Za-z0-9._]*)")
(setq sxp::re-entity (regcomp "^&([A-Za-z][-A-Za-z0-9._]*);"))
(setq sxp::re-entity-char-ref (regcomp "^&#([0-9]+);"))
(setq sxp::re-entity-hex-char-ref (regcomp "^&#x([0-9a-fA-F]+);"))

(setq sxp::re-chardata-e (regcomp "^[^<&]+"))
(setq sxp::re-chardata-apos-e (regcomp "^[^<&']*"))
(setq sxp::re-chardata-quot-e (regcomp "^[^<&\"]*"))

(setq sxp::re-chardata-ie (regcomp "^[^<]+"))
(setq sxp::re-chardata-apos-ie (regcomp "^[^<']*"))
(setq sxp::re-chardata-quot-ie (regcomp "^[^<\"]*"))

(setq sxp::re-comment-start (regcomp "^!--"))
(setq sxp::re-comment-end (regcomp "-->"))

(setq sxp::re-cdata-start (regcomp "^![[]CDATA[[]"))
(setq sxp::re-cdata-end (regcomp "[]][]]>"))

(setq sxp::re-PI-start (regcomp "^[?]"))
(setq sxp::re-PI-end (regcomp "[?]>"))

(setq sxp::re-name (regcomp "^[a-zA-Z_:][-a-zA-Z0-9._:\xb7]*"))
(setq sxp::re-endname (regcomp "^[/]([a-zA-Z_:][-a-zA-Z0-9._:\xb7]*)>"))

(setq sxp::re-element-end (regcomp "^[ \t\n\r]*(>|[/]>)"))

(setq sxp::re-attribute (regcomp (+ "^(" sxp::re-name 
      ")" sxp:spaces "=" sxp:spaces "([\"]([^<\"]*)[\"]|'([^<']*)')"
)))

;;=============================================================================
;;                    MAIN parser
;;=============================================================================
;;The Parser is a recursive descent parser, using the power of the fast regexp
;;engine as much as possible for the lex part. Each state XXX is parsed by a 
;;function of the name sxp:PS-XXX. (PS for Parse State)
;;For efficiency, we use the global (local to sxp:parse, actually) vars:
;; buffer	parse buffer (string containing the whole XML document)
;; pos		current offset in parse buffer
;; states       stack of parsing states for nested structures (stack at end)
;; elt          current element (the current end of the states list)
;; cur-chardata current chardata buffer being parsed for elt
;;
;; All sxp:PS-XXX should return freely modifiable contents (i.e., return 
;; copies of strings)

(defun sxp:parse (buffer &key keep-comments keep-PIs keep-DTD 
    nil-error 
    forest
    ((filename sxp:filename) ())
    ((entities general-entities) ())
    keep-whitespace
    whitespace-keeping-tags
    ignore-entities
    external-entities
    (notation-parsers sxp:notation-parsers)
    (notation-default-parser sxp:notation-default-parser)

    &aux
    res non-nil-res 
    (pos 0)
    (states (list))
    (states-pos (list))
    (elt (sxp:node-new))
    (cur-chardata (copy ""))
    (sxp:keep-whitespace-tags (vector))
    (in-header? t)
    (sxp:PS-toplevel sxp:PS-toplevel-header)
  )
  (if (not (typep buffer String))	;gobble up FDs if not strings
    (setq buffer (String buffer))
  )
  (dolist (tag whitespace-keeping-tags)
    (put sxp:keep-whitespace-tags (intern tag) t)
  )
  (if ignore-entities 
    (setqn sxp::entity-char 0
      sxp::re-chardata sxp::re-chardata-ie
      sxp::re-chardata-apos sxp::re-chardata-apos-ie
      sxp::re-chardata-quot sxp::re-chardata-quot-ie
    )
    (setqn sxp::entity-char #\&
      sxp::re-chardata sxp::re-chardata-e
      sxp::re-chardata-apos sxp::re-chardata-apos-e
      sxp::re-chardata-quot sxp::re-chardata-quot-e
    )
  )
  (setq sxp:encoding "UTF-8")		;default encoding
  (setq sxp:standalone ())
  (sxp:DTDInit general-entities external-entities)

  ;; We begin the parsing in state toplevel.
  (if nil-error
    (catch 'sxp:Error (setq res (sxp:parse-toplevel)))
    (setq res (sxp:parse-toplevel))
  )
  (if forest				;return
    (progn (sxp:element-clean) (1 elt)) ;list of top-level trees (forest)
    res					;or last tree (default)
  ) 
)

(defun sxp:parse-toplevel (&aux res non-nil-res xml_space)
  (catch 'sxp:EOF			;we exit on EOF
    ;; parse first line, process it if it is an XML decl
    (with (sxp:parse-PI-hook sxp:parse-XML-decl)
      (if (setq res (sxp:PS-toplevel))
	(setq non-nil-res res)
    ))
    (while t
      (setq res (sxp:PS-toplevel))
      (if res (setq non-nil-res res))
    )
  )
  non-nil-res
)

;; A convenience function:

(defun sxp:load (filename)
  (sxp:parse (open filename) :filename filename)
)

(defvar sxp:parse-PI-hook ())
(defvar sxp:DTD ())
(defvar sxp:outcol 0)

;;=============================================================================
;;                    Parser states
;;=============================================================================
;; Toplevel: main switch
;; TODO: do not allow entities and chardata at toplevel
(defun sxp:PS-toplevel-header (&aux (xml_space xml_space))
  (if (regexec sxp::re-space buffer pos) ;skip leading spaces
    (setq pos (1 (getn sxp::re-space 0)))
  )
  (if 
    (= (pos buffer) #\<) (progn
      (incf pos)
      (if 
	(= (pos buffer) #\?) (sxp:PS-PI)
	(= (pos buffer) #\!) (sxp:PS-DTD-or-comments)
	(progn (sxp:end-of-header) (sxp:PS-element))
    ))

    (= (pos buffer) sxp::entity-char) (progn 
      (sxp:end-of-header) 
      (sxp:PS-entity)
    )

    (not (pos buffer)) (throw 'sxp:EOF)

    (progn
      (sxp:end-of-header)
      (sxp:PS-chardata)
    )
))

(defun sxp:end-of-header ()
  (setq in-header? ())
  (sxp:DTDClose)
  (setq sxp:PS-toplevel sxp:PS-toplevel-body)
)

(defun sxp:PS-toplevel-body (&aux (xml_space xml_space))
  (if 
    (= (pos buffer) #\<) (progn
      (incf pos)
      (if 
	(= (pos buffer) #\?) (sxp:PS-PI)
	(= (pos buffer) #\!) (sxp:PS-DTD-or-comments)
	(sxp:PS-element)
    ))

    (= (pos buffer) sxp::entity-char) (sxp:PS-entity)

    (not (pos buffer)) (throw 'sxp:EOF)

    (sxp:PS-chardata)
))

;; Entity: minimal parsing for now
;; We must copy the result, as all strings put into the node tree may be 
;; NCONCed during parsing
(defun sxp:PS-entity (&aux name)
  (if (regexec sxp::re-entity buffer pos) (progn	;entity decoding
      (setq name (regsub sxp::re-entity 1))
      (setq pos (1 (getn sxp::re-entity 0)))
      (if					; first, Predefined entities
	(= name "lt") (copy "<")
	(= name "gt") (copy ">")
	(= name "amp") (copy "&")
	(= name "apos") (copy "'")
	(= name "quot") (copy "\"")
	;; general entities
	(setq value (getn sxp:EntDecls name)) 
	(copy (sxp:expand-entities value))
	;; external entities: returns a tree
	(setq value (getn sxp:EntDeclsExternal name)) 
	(sxp:load-url (0 value) (1 value))
	;; we should parse other DTD-declared entities here 
	(sxp:fatal-error sxp:E:UnknownEntity name)
      )
    )
    (regexec sxp::re-entity-char-ref buffer pos) (progn	;decimal charcode
      (setq pos (1 (getn sxp::re-entity-char-ref 0)))
      (String (list (Int (regsub sxp::re-entity-char-ref 1))))
    )
    (regexec sxp::re-entity-hex-char-ref buffer pos) (progn ;hex charcode
      (setq pos (1 (getn sxp::re-entity-hex-char-ref 0)))
      (String (list (xtoi (regsub sxp::re-entity-hex-char-ref 1))))
    )

    (sxp:fatal-error sxp:E:MalformedEntity)
))
      
;; Element: main workhorse: we push a new state on the stack, which becomes 
;; the current one, we scan the attributes and then the character data inside
;; for character datas, entities, sub-elements
;; pos must be set to the first char of the element name
;; for memory efficiency, attribute list is nil, not a unique empty list
(defun sxp:PS-element ()
  (if (regexec sxp::re-name buffer pos) 
    (with (parent-elt elt
	elt (sxp:node-new			;create new current element
	  (intern (regsub sxp::re-name 0))	;of name
      ))
      (lappend states elt)		;push elt to top of states stack
      (lappend states-pos pos)
      (if (eq () parent-elt) () (sxp:node-append-children parent-elt elt))
      (setq pos (1 (getn sxp::re-name 0)))
      (while (not (regexec sxp::re-element-end buffer pos))
	(if (regexec sxp::re-space buffer pos) ;skip spaces
	  (setq pos (1 (getn sxp::re-space 0)))
	)
	(sxp:PS-attribute)		;then parse each attribute decl
      )
      (if (xnl:get-attribute elt :xml:space)
	(if (= "preserve" (xnl:get-attribute elt :xml:space))
	  (setq xml_space "preserve")
	  (= "default" (xnl:get-attribute elt :xml:space))
	  (setq xml_space ())
	  t
	  (error 'sxp:E:BadXmlSpace (xnl:get-attribute elt :xml:space))
      ))
      (if xml_space (4 elt t))
      ;;end of start attribute
      (setq pos (1 (getn sxp::re-element-end 0)))
      (if (= #\> (getn buffer (0 (getn sxp::re-element-end 1))))
	;; end of a start-tag: parse children
	(sxp:PS-chardata)
	;; else end of element (no children), nothing else to do
	;; we could clean up here, but we have no children to clean
	(with (old-elt elt)
	  (delete states -1)
	  (delete states-pos -1)
	  (setq elt (getn states -1))
	  old-elt
	)
      )	
    )
    ;; not a name? confused
    (sxp:fatal-error sxp:E:BadElementName (sxp:find-potential-name buffer pos))
  )
)

;; Character data: we must parse it to detect embedded entities/elements
;; We append chardata/entities found to cur-chardata, and elements pushes the
;; cur-chardata in the list of children of the current state
;; We terminate when we read the closing tag to the current state (or EOF)
;; and clean the resulting element 
(defun sxp:PS-chardata (&aux res)
  (while (not res)			;till end of tag
    (while (regexec sxp::re-chardata buffer pos)
      ;; appends chars to current elt
      (sxp:node-append-children elt (regsub sxp::re-chardata 0))
      (setq pos (1 (getn sxp::re-chardata 0)))
    )
    (if 
      ;; opening bracket: end of current tag or start of a children
      (= #\< (pos buffer))
      (if
	(= #\/ ((incf pos) buffer))	; </ end tag
	(if (regexec sxp::re-endname buffer pos)
	  (with (name (regsub sxp::re-endname 1))
	    (if (eq (intern name) (sxp:node-get-name elt))
	      ()
	      (sxp:fatal-error sxp:E:EndTagDontMatch 
		(sxp:node-get-name elt) name 
		(sxp:line-of-pos (getn states-pos -1))
		(1 sxp:line-or-col-of-pos-result)
	    ))
	    (setq pos (1 (getn sxp::re-endname 0)))
	    (setq res elt)
	    (sxp:element-clean)
	    (delete states -1)
	    (delete states-pos -1)
	    (setq elt (getn states -1))
	  )
	  (sxp:fatal-error sxp:E:BadEndTag)
	)

	(= #\! (pos buffer))		; <! a comment or CDATA
	(sxp:PS-DTD-or-comments)

	(= #\? (pos buffer))		; <? a PI
	(sxp:PS-PI)

	;; start sub-element
	(sxp:PS-element)
      )

      ;; entity: append its expansion
      (= sxp::entity-char (pos buffer))		
      (sxp:node-append-children elt (sxp:PS-entity))

      ;; EOF in an element
      (if states
	(sxp:fatal-error sxp:E:EndTagMissing (sxp:node-get-name elt))
	;;last chardata of toplevel node)
	(throw 'sxp:EOF (setq res (getn (sxp:node-get-children elt) -1)))
      )
    )
  )
  res
)

;; Parse an attribute declaration, appends it to current state, and returns 
;; true. If no more attribs, return ()
;; pos must be at start of attrib decl
(defun sxp:PS-attribute ()
  (if (regexec sxp::re-attribute buffer pos)
    (with (attkey (intern (+ ":" (regsub sxp::re-attribute 1)))
	attvalue 
	(if (= #\" (getn buffer (0 (getn sxp::re-attribute 2))))
	  (progn (setq pos (0 (getn sxp::re-attribute 3)))
	    (sxp:PS-attvalue sxp::re-chardata-quot)
	  )
	  (progn (setq pos (0 (getn sxp::re-attribute 4)))
	    (sxp:PS-attvalue sxp::re-chardata-apos)
	  )
	)
      )
      (sxp:node-append-attributes elt attkey attvalue)
      (setq pos (1 (getn sxp::re-attribute 0)))
    )
    (sxp:fatal-error sxp:E:BadAttribute)
  )
)

;; <! DTD or comments are ignored,  CDATA parsed
(setq sxp:PS-DTD-or-comments-re-doctype-start (regcomp "^!DOCTYPE"))
(setq sxp:PS-DTD-or-comments-re-doctype-end (regcomp ">"))
(setq sxp:PS-DTD-or-comments-re-doctypes-start (regcomp "^!DOCTYPE[^>]*[[]"))
(setq sxp:PS-DTD-or-comments-re-doctypes-end (regcomp "[]]>"))
(setq sxp:PS-DTD-or-comments-re-doctypes-name (regcomp 
    "^!DOCTYPE[ \t\n\r]+([^ \t\n\r]+)"))

(defun sxp:PS-DTD-or-comments ()
  (if
    (regexec sxp::re-comment-start buffer pos)
    (if keep-comments
      (if (regexec sxp::re-comment-end buffer (+ 3 pos))
	(with (res (subseq buffer pos
	      (- (setq pos (1 (getn sxp::re-comment-end 0))) 1)
	  ))
	  (sxp:node-append-children elt (vector res () () t))
	)
	(sxp:fatal-error sxp:E:UnterminatedComment)
      )
      (if (regexec sxp::re-comment-end buffer (+ 3 pos))
	(setq pos (1 (getn sxp::re-comment-end 0)))
	(sxp:fatal-error sxp:E:UnterminatedComment)
      )
    )
    
    (regexec sxp::re-cdata-start buffer pos)
    (if (regexec sxp::re-cdata-end buffer (+ 8 pos)) (progn
	;; append chardata to current elements
	(sxp:node-append-children elt 
	  (subseq buffer (+ 8 pos) (0 (getn sxp::re-cdata-end 0)))
	)
	(setq pos (1 (getn sxp::re-cdata-end 0)))
      )
      (sxp:fatal-error sxp:E:UnterminatedCDATA)
    )
    
    (regexec sxp:PS-DTD-or-comments-re-doctypes-start buffer pos) (progn
      (if (not in-header?) (sxp:fatal-error sxp:E:DTDMisplaced))
      (if (regexec sxp:PS-DTD-or-comments-re-doctypes-end buffer (+ 3 pos))
	(progn
	  (if keep-DTD (setq sxp:DTD (subseq buffer
		(- (0 (getn sxp:PS-DTD-or-comments-re-doctypes-start 0)) 1)
		(1 (getn sxp:PS-DTD-or-comments-re-doctypes-end 0))
	  )))
	  (sxp:declare-built-in-entities)
	  ;; we do a limited parsing of DTD just to parse the entity decls
	  (sxp:PS-DTD-entities buffer ; gives in buffer the DTD zone by its:
	    (1 (getn sxp:PS-DTD-or-comments-re-doctypes-start 0))	;start
	    (0 (getn sxp:PS-DTD-or-comments-re-doctypes-end 0)) ;end
	  )
	  (setq pos (1 (getn sxp:PS-DTD-or-comments-re-doctypes-end 0)))
	)
	(sxp:fatal-error sxp:E:UnterminatedComment)
    ))
    
    (regexec sxp:PS-DTD-or-comments-re-doctype-start buffer pos) (progn
      (if (not in-header?) (sxp:fatal-error sxp:E:DTDMisplaced))
      (if (regexec sxp:PS-DTD-or-comments-re-doctype-end buffer (+ 3 pos))
	(progn
	  (if keep-DTD (setq sxp:DTD (subseq buffer 
		(- (0 (getn sxp:PS-DTD-or-comments-re-doctype-start 0)) 1)
		(1 (getn sxp:PS-DTD-or-comments-re-doctype-end 0))
	  )))
	  (sxp:declare-built-in-entities)
	  (setq pos (1 (getn sxp:PS-DTD-or-comments-re-doctype-end 0)))
	)
	(sxp:fatal-error sxp:E:UnterminatedComment)
    ))
    
    (sxp:parser-error sxp:U:DTD)
  )
  ()
)

(setq sxp:PS-PI-re (regcomp "^[?][ \t\n\r]*xml[ \t\n\r]"))
;; Processing Instructions are ignored
(defun sxp:PS-PI ()
  (if (regexec sxp::re-PI-end buffer (+ 1 pos)) (progn
      (if sxp:parse-PI-hook		;call user-defined hook if needed
	(sxp:parse-PI-hook (subseq buffer (+ 1 pos)
	    (- (1 (getn sxp::re-PI-end 0)) 2)
      )))
      (if (and keep-PIs (not (regexec sxp:PS-PI-re buffer pos)))
	(with (PI-contents (subseq buffer pos
	      (- (1 (getn sxp::re-PI-end 0)) 1)
	  ))
	  (sxp:node-append-children elt (vector PI-contents () () t))
      ))
      (setq pos (1 (getn sxp::re-PI-end 0)))
    )
    (sxp:fatal-error sxp:E:UnterminatedPI)
  )
  ()
)

;; Processing the <?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"yes\"?> decl
(defun sxp:parse-XML-decl (s &aux
    (re-xml-enc (regcomp 
	"^[ \t\n\r]*xml[ \t\n\r]*(version[ \t\n\r]*=[ \t\n\r]*['\"]([^'\"]+)['\"])?[ \t\n\r]*(encoding[ \t\n\r]*=[ \t\n\r]*['\"]([^'\"]+)['\"])?[ \t\n\r]*(standalone[ \t\n\r]*=[ \t\n\r]*['\"]([^'\"]+)['\"])?"
    ))
    (re-xml (regcomp "^[ \t\n\r]*xml[ \t\n\r]"))
  )
  (setq sxp:encoding "UTF-8")
  (setq sxp:standalone ())
  (if (regexec re-xml s) (progn
      (if (regexec re-xml-enc s) (progn
	  (if (/= "" (re-xml-enc 4))
	    (setq sxp:encoding (toupper (regsub re-xml-enc 4)))
	  )
	  (if (/= "" (re-xml-enc 6))
	    (setq sxp:standalone (tolower (regsub re-xml-enc 6)))
	  )
      ))
      (if (= "ISO-8859-1" sxp:encoding)
	()				;nothing to do, native format
	(= "UTF-8" sxp:encoding)
	(setq buffer (utf8-to-latin1 buffer 
	    (lambda (s) (sxp:fatal-error sxp:U:NonLatin1Chars s))
	)) ;convert buffer to latin-1
	t				;else error
	(sxp:fatal-error sxp:U:Encoding sxp:encoding)
))))

;; parse attribute values. Expanses &, and barf on <
;; re determines if we stop on " or '
(defun sxp:PS-attvalue (re &aux
    (not_done t)
    (res (copy ""))
  )
  (while not_done
    (regexec re buffer pos)
    (while (/= pos (setq pos (1 (getn re 0))))
      (nconc res (regsub re 0))
      (regexec re buffer pos)
    )
    (if					;examine stop-char
      (= #\" (pos buffer)) (progn (setq not_done ()) (incf pos))
      (= #\' (pos buffer)) (progn (setq not_done ()) (incf pos))
      (= sxp::entity-char (pos buffer)) (nconc res (sxp:PS-entity))
      (= #\< (pos buffer)) (sxp:fatal-error sxp:E:BadAttributeValue)
      ;; else EOF
      (sxp:fatal-error sxp:E:UnterminatedAttribute attkey)
    )
  )
  res
)

;; DTD parsing
;;============
;; Minimal for now. may be extended in the future
;; An entity is a pair: (value). Here are its access functions:
(defun sxp:EntDecl-declare (name value) 
  (if (getn sxp:EntDecls name) 
    ()					;XML retains only first decl
    (put sxp:EntDecls name value)
))
(defun sxp:EntDecl-declare-external (name value notation) 
  (if (getn sxp:EntDeclsExternal name) 
    ()					;XML retains only first decl
    (put sxp:EntDeclsExternal name (vector value notation)))
))

;; sets things up
(defun sxp:DTDInit (general-entities external-entities)
  (setqn
    sxp:EntDecl-value 0
    sxp:EntDecls (list)
    sxp:EntDeclsExternal (list)
    sxp:EntDeclsOverride general-entities
    sxp:EntDeclsExternalOverride external-entities
))

;; for later override parsed entities from DTD
(defun sxp:DTDClose ()
  (dohash (name value sxp:EntDeclsOverride)
    (put sxp:EntDecls (String name) value)
  )
  (dohash (name value sxp:EntDeclsExternalOverride)
    (if (typep value List)
      (put sxp:EntDeclsExternal (String name) value)
      (put sxp:EntDeclsExternal (String name) (vector value ()))
  ))
)

;; Entities
(setq sxp:PS-DTD-entities-re-declaration (regcomp (+ 
      "<!ENTITY" sxp:space sxp:name sxp:space "([^>]+)>"
)))
;; General Entities. will fail on parameter & external entities
(setq sxp:PS-DTD-entities-re-general (regcomp sxp:quoteddata))
;; External entities: System
(setq sxp:PS-DTD-entities-re-system (regcomp (+ 
      "SYSTEM" sxp:space sxp:quoteddata
)))
;; External entities: Public (we use the System field)
(setq sxp:PS-DTD-entities-re-public (regcomp (+ 
      "PUBLIC" sxp:space sxp:quoteddata sxp:space sxp:quoteddata
)))
;; Notation
(setq sxp:PS-DTD-entities-re-notation (regcomp (+
      sxp:space "NDATA" sxp:space sxp:name
)))

(defun sxp:PS-DTD-entities (buffer start end &aux
    name value
  )
  (while (and (regexec sxp:PS-DTD-entities-re-declaration buffer start)
      (<= (1 (getn sxp:PS-DTD-entities-re-declaration 0)) end)
    )
    (setq name (regsub sxp:PS-DTD-entities-re-declaration 1))
    (setq value (regsub sxp:PS-DTD-entities-re-declaration 2))
    (if
      (sxp:PS-DTD-entities-re-public value)
      (sxp:EntDecl-declare-external name 
	(sxp:unquoted-string (regsub sxp:PS-DTD-entities-re-public 2))
	(if (regexec sxp:PS-DTD-entities-re-notation value 
	    (1 (getn sxp:PS-DTD-entities-re-public 0))
	  )
	  (regsub sxp:PS-DTD-entities-re-notation 1)
      ))

      (sxp:PS-DTD-entities-re-system value)
      (sxp:EntDecl-declare-external name 
	(sxp:unquoted-string (regsub sxp:PS-DTD-entities-re-system 1))
	(if (regexec sxp:PS-DTD-entities-re-notation value 
	    (1 (getn sxp:PS-DTD-entities-re-system 0))
	  )
	  (regsub sxp:PS-DTD-entities-re-notation 1)
      ))
	
      (sxp:PS-DTD-entities-re-general value)
      (sxp:EntDecl-declare name (sxp:unquoted-string 
	  (regsub sxp:PS-DTD-entities-re-general 1)
      ))
    )
    (setq start (1 (getn sxp:PS-DTD-entities-re-declaration 0)))
  )
)

(defun sxp:unquoted-string (s)
  (subseq s 1 (- (length s) 1))
)

(setq sxp:expand-entities-re (regcomp (+ "&" sxp:name ";")))
(defun sxp:expand-entities (buffer &aux (pos 0) res)
  (while (regexec sxp:expand-entities-re buffer pos)
    (if (not res)
      (setq res (subseq buffer 0 (0 (getn sxp:expand-entities-re 0))))
    )
    (setq pos (0 (getn sxp:expand-entities-re 0)))
    (incf res (sxp:PS-entity))
  )
  (if res 
    (+ res (subseq buffer pos))
    buffer
  )
)

(setq sxp:load-url-re-unimplemented (regcomp 
    "^([hH][tT][tT][pP][sS]?|[fF][tT][pP])://"
))
(setq sxp:load-url-re-file (regcomp "^[fF][iI][lL][eE]:(.*)"))
(defun sxp:load-url (url notation &aux fd)
  (if
    (sxp:load-url-re-unimplemented url)
    (error sxp:U:Protocol url)
    
    (sxp:load-url-re-file url)
    (setq url (regsub sxp:load-url-re-file 1))
  )
  (if (setq fd (open url :error ()))
    (with (contents (String fd))
      (if notation
	;;unparsed entity: must ask application to encode it
	((get notation-parsers notation notation-default-parser)
	  notation contents url elt
	)
	(with (sxp:encoding ())
	  (sxp:parse contents :keep-comments keep-comments :keep-PIs keep-PIs
	    :filename url :entities general-entities 
	    :keep-whitespace keep-whitespace 
	    :whitespace-keeping-tags whitespace-keeping-tags
	    :external-entities external-entities
	    :notation-parsers notation-parsers
	    :notation-default-parser notation-default-parser
    ))))
    (error sxp:E:ReadFile url)
  )
)
  
(setq sxp:notation-parsers (list))
(defun sxp:notation-default-parser (notation contents url node)
  (sxp:expand-entities contents)
)

;;=============================================================================
;;                    Utils
;;=============================================================================
;; Cleaning a newly collected element
;; remove blanks if there are only sub-elements, no non-space data
(setq sxp:element-clean-re (regcomp "^[ \t\n\r]*$"))
(defun sxp:element-clean (&aux cdata? subelts?)
  (if (or xml_space
      (get sxp:keep-whitespace-tags (sxp:node-get-name elt) keep-whitespace)
    )
    ()
    (progn
      (dolist (c (sxp:node-get-children elt))
	(if (typep c String)
	  (if (regexec sxp:element-clean-re c) () (setq cdata? t))
	  (setq subelts? t)
      ))
      (if cdata? ()
	(if subelts?		;subnodes with whitespaces
	  (with (children (list))
	    (dolist (c (sxp:node-get-children elt))
	      (if (and (typep c String) (regexec sxp:element-clean-re c))
		()
		(lappend children c)
	      )
	    )
	    (sxp:node-set-children elt children)
	  )
	  ;; else only whitespace, removes children list to optimize
	  (if (sxp:node-get-attributes elt)
	    (sxp:node-set-children elt ())
	    (while (> (length elt) 1)
	      (delete elt 1)		;depends on implementation of node
	  ))
)))))

;; tells if a node contain character data, not only subnodes
(defun sxp:node-has-chardata? (node)
  (catch 'Yes
    (dolist (child (sxp:node-get-children node))
      (if (typep child String) (throw 'Yes t))
    )
    ()
  )
)

;;=============================================================================
;;                    Nested lists interface
;;=============================================================================
;; Interface to sxp:node, a poor man DOM
;; OBSOLETE: use the xnl interface below
;; Note: the following clean access functions are defined to access the 
;; returned nested lists:
;; (sxp:node-new atom)            ==> return an empty element node of name atom
;;
;; (sxp:node-get-name node)       ==> name of element (atom)
;; (sxp:node-set-name node atom)      sets the name
;;
;; (sxp:node-get-children node)   ==> lists of children of node 
;;                                    (strings or nodes), that can be 
;;                                    modified in place (this is not a copy)
;; (sxp:node-set-children node children-list) 
;;                                    sets the list of children in node
;; (sxp:node-append-children node child)
;;                                    appends a child to the children
;; (sxp:node-get-attributes node) ==> p-list of attributes of node
;; (sxp:node-set-attributes node attributes-p-list)
;;                                    sets the p-list of attributes of node
;; (sxp:node-put-attributes node attribute-key attribute-value)
;;                               sets key="value" in the node attributes
;;

(setq sxp:node-get-name 0)
(setq sxp:node-set-name 0)

(setq sxp:node-get-children 1)
(defun sxp:node-set-children (n children) (put n 1 children))
;; We concatenate jointive strings
(defun sxp:node-append-children (n c &aux (children (getn n 1)))
  (if 
    (eq () children) 
    (put n 1 (list c))

    (and (typep c String) (typep (getn children -1) String))
    (nconc (getn children -1) c)

    (lappend children c)
  ) 
  n
)

(setq sxp:node-get-attributes 2)
(defun sxp:node-set-attributes (n attributes) (put n 2 attributes))
(defun sxp:node-append-attributes (n key val &aux (attributes (getn n 2)))
  (if (eq () attributes)
    (put n 2 (list key val))
    (progn
      (lappend attributes key)
      (lappend attributes val)
    )
  )
  n
)

(setq sxp:node-get-no-end-tag 3)
(setq sxp:node-set-no-end-tag 3)

(setq sxp:node-new vector)

;;=============================================================================
;;                    A simpler version: xnl: for XML Nested Lists
;;=============================================================================
;; The recommended way. See doc in the header
(setqn
  xnl:name 0
  xnl:children 1
  xnl:attributes 2
  xnl:no-end-tag 3
  xnl:new vector
  xnl:chardata? (lambda (n) (typep n String))
)

(defun xnl:set-child-or-insert (n p c func &aux (children (getn n 1)))
  (if (typep p Int) () (setq p -1))
  (if 
    (eq () children) 
    (if (= -1 p) (put n 1 (list c))
      (put n 1 (func (list) p c))
    )

    (progn 
      (func children p c)
      (if (= -1 p) (setq p (- (length children) 1)))
      (if (and (typep c String) (typep (getn children (+ p 1)) String))
	(progn				;collapse string after
	  (put children p (+ c (getn children (+ p 1))))
	  (delete children (+ p 1))
      ))
      (if (and (typep c String) (> p 0) (typep (getn children (- p 1)) String))
	(progn				;collapse string
	  (put children (- p 1) (+ (getn children (- p 1)) c))
	  (delete children p)
      ))
    )
  ) 
  n
)
(defun xnl:get-child (n p) (p (1 n)))
(defun xnl:set-child (n p c) (xnl:set-child-or-insert n p c put))
(setq xnl:put-child xnl:set-child)
(defun xnl:insert-child (n p c) (xnl:set-child-or-insert n p c insert))

(defun xnl:delete-child (n p) (delete (xnl:children n) p))

(defun xnl:get-attribute (n key) (getn (2 n) key))

(defun xnl:set-attribute (n key val &aux (attributes (getn n 2)))
  (if (eq () attributes)
    (put n 2 (list key val))
    (put attributes key val)
  )
  n
)

(defun xnl:delete-attribute (n key &aux (attributes (getn n 2)))
  (delete (xnl:attributes n) key)
)


;;=============================================================================
;;                    sxp:print
;;=============================================================================
;; printing sxp:nodes is much simpler
;; We indent the output by the string sxp:print-indent-string (default "  ")
;; If () we dont indent
;; If raw is passed, dont print PI header nor terminating newline (useful
;; for printing XML subparts).
;; Only on nodes not having character data
;; Optional last non documented argument is to set the start level of 
;; indentation

(defvar sxp:print-indent-string "  ")
(defvar sxp:width 78)

(defun sxp:print (node stream &key
    raw 
    (encoding "ISO-8859-1")
    ignore-entities
    ;; internal use only options
    (level 1)				;base indent
    (final-encoding encoding)		;string to put in header
    standalone
    forest
    &aux
    (sxp:outcol 0)
  )
 (setq encoding (toupper encoding))
  (if ignore-entities
    (setqn sxp:re-print-chars sxp:re-print-chars-ie
      sxp:re-print-attval-chars sxp:re-print-attval-chars-ie
    )
    (setqn sxp:re-print-chars sxp:re-print-chars-e
      sxp:re-print-attval-chars sxp:re-print-attval-chars-e
  ))
  (if (not stream) (setq stream *standard-output*))
  (if 
    ;; UTF-8? we print in latin-1 on a buffer that we translate afterwards
    (= "UTF-8" encoding)
    (with (s (copy "")
	iso-stream (open s :type :string :direction :io))
      (sxp:print node iso-stream :raw raw :encoding "ISO-8859-1" :standalone
	standalone
	:level level :final-encoding encoding :ignore-entities ignore-entities)
      (print-format stream "%0"	(latin1-to-utf8 s))
    )
    ;; ISO-8859-1 is the native (fastest) format
    (= "ISO-8859-1" encoding)
    (with (sxp::print-indent (not sxp:print-indent-string))
      (if raw
	(if forest
	  (dolist (subnode node) (sxp:print-node subnode stream level))
	  (sxp:print-node node stream level)
	) 
	(progn
	  (if standalone
	    (sxp:print-format-stream 
	      "<?xml version=\"1.0\" encoding=\"%0\" standalone=\"%1\"?>\n" 
	      final-encoding standalone)
	    (sxp:print-format-stream 
	      "<?xml version=\"1.0\" encoding=\"%0\"?>\n" final-encoding)
	  )
	  (if sxp:DTD (sxp:print-format-stream "%0\n" sxp:DTD))
	  (if forest
	    (dolist (subnode node)
	      (sxp:print-node subnode stream level)
	      (sxp:write-chars-stream "\n")
	    )
	    (progn
	      (sxp:print-node node stream level)
	      (sxp:write-chars-stream "\n")
	  ))
	)
      )
    )
    ;; Unknown encoding, error
    (error 'sxp:U:Encoding encoding)
  )
  ()
)

(setq sxp:print-node:re-normaltag (regcomp "^[a-zA-Z_:]"))
;; uses the encoding global var
(defun sxp:print-node (node stream level &aux canwrap?)
  (if (typep node String)		;chardata node
    (sxp:print-chardata node stream)
    (xnl:name node)			;normal node
    (progn
      (sxp:print-format-stream "<%0" (sxp:node-get-name node))
      (dohash (key val (sxp:node-get-attributes node))
	(with (atts (copy "")
	    att (open atts :type :string :direction :io)
	  )
	  (print-format att " %0=\"" (subseq key 1))
	  (sxp:print-attval val att)
	  (print-format att "\"")
	  (if (and (> (+ sxp:outcol (length atts)) sxp:width) canwrap?)
	    (progn
	      (setq canwrap? ())
	      (sxp:write-chars-stream "\n")
	      (dotimes (sxp::dummy level)
		(sxp:write-chars-stream sxp:print-indent-string)
	      )
	      (sxp:write-chars-stream (subseq atts 1))
	    )
	    (progn
	      (setq canwrap? t)
	      (sxp:write-chars-stream atts)
	  ))
	  (setq atts (copy ""))
	  (setq att (open atts :type :string :direction :io))
      ))
      (if (sxp:node-get-children node) (progn
	  (sxp:write-chars-stream ">")
	  (if (or sxp::print-indent
	      (sxp:node-has-chardata? node) ; dont indent if there are chardata
	      (4 node)			;or node is marked as no-indent
	      
	    )
	    (dolist (child (sxp:node-get-children node))
	      (sxp:print-node child stream (+ level 1))
	    )
	    ;; only subnodes? indent
	    (progn
	      (sxp:write-chars-stream "\n")
	      (dolist (child (sxp:node-get-children node))
		(dotimes (sxp::dummy level)
		  (sxp:write-chars-stream sxp:print-indent-string)
		)
		(sxp:print-node child stream (+ level 1))
		(sxp:write-chars-stream "\n")
	      )
	      (dotimes (sxp::dummy (- level 1))
		(sxp:write-chars-stream sxp:print-indent-string)
	      )
	    )
	  )
	  (sxp:print-format-stream "</%0>" (sxp:node-get-name node))
	)
	(if (sxp:print-node:re-normaltag (sxp:node-get-name node))
	  (sxp:write-chars-stream "/>")
	  (sxp:write-chars-stream ">")	;PIs and comments
	)
      )
    )
    ;; else "forest" node: a group (list) of nodes
    (dolist (subnode (xnl:children node))
      (sxp:print-node subnode stream level)
    )
  )
)

(setq sxp:re-print-chars-e (regcomp "^[^<&]+"))
(setq sxp:re-print-chars-ie (regcomp "^[^<]+"))

(defun sxp:print-chardata (s stream &aux (pos 0))
  (catch 'Done (while t
      (if (regexec sxp:re-print-chars s pos)
	(progn
	  (sxp:print-format-stream "%0" (regsub sxp:re-print-chars 0))
	  (setq pos (1 (getn sxp:re-print-chars 0)))
	)

	(not (getn s pos)) (throw 'Done)

	(= #\< (getn s pos)) (progn (sxp:print-format-stream "&lt;") (incf pos))

	(= #\& (getn s pos)) (progn (sxp:print-format-stream "&amp;") (incf pos))
      )
    )
  )
)

(setq sxp:re-print-attval-chars-e (regcomp "^[^<&\"]+"))
(setq sxp:re-print-attval-chars-ie (regcomp "^[^<\"]+"))

(defun sxp:print-attval  (s stream &aux (pos 0))
  (catch 'Done (while t
      (if (regexec sxp:re-print-attval-chars s pos)
	(progn
	  (print-format stream "%0" (regsub sxp:re-print-attval-chars 0))
	  (setq pos (1 (getn sxp:re-print-attval-chars 0)))
	)

	(not (getn s pos)) (throw 'Done)

	(= #\< (getn s pos)) (progn (print-format stream "&lt;") (incf pos))

	(= #\" (getn s pos)) (progn (print-format stream "&quot;") (incf pos))

	(= #\& (getn s pos)) (progn (print-format stream "&amp;") (incf pos))
      )
    )
  )
)

;; wrapper to set the current column in sxp:outcol
(setq sxp:sxp:print-format-stream:re (regcomp ".*\n"))
(defun sxp:print-format-stream (fs &rest args &aux out)
  (setq out (apply print-format (+ (list String fs) args)))
  (sxp:write-chars-stream out)
)
(defun sxp:write-chars-stream (out)
  (if (sxp:sxp:print-format-stream:re out)
    (setq sxp:outcol (- (length out)
	(1 (getn sxp:sxp:print-format-stream:re 0))))
    (incf sxp:outcol (length out))
  )
  (write-chars out () stream)
)

;;=============================================================================
;;                    sxp:eval
;;=============================================================================
;; sxp:eval is a simple mecanism to process a XML tree in memory
;; we provide two equivalent way to describe the customisations: by defining
;; prefixed functions or giving a p-list of lambdas

(defun sxp:eval (node &optional (prefix "sxe"))
  (setq sxp::eval-prefix prefix)
  (eval (if (typep prefix List)		
      ;; p-list mode
      (with (sxp::eval-prefix (Hashtable sxp::eval-prefix) ; for efficiency
	  sxp::eval-node-get-handler sxp::eval-node-get-handler-plist
	  sxp:eval-default (getn sxp::eval-prefix ())
	)
	(sxp::eval-node node)
      )
      ;; prefixed functions mode
      (with (sxp::eval-node-get-handler sxp::eval-node-get-handler-prefix
	  sxp::eval-default-name (intern (+ sxp::eval-prefix ":"))
	  sxp:eval-default (if (boundp sxp::eval-default-name)
	    (eval sxp::eval-default-name) ()
	  )
	)
	(sxp::eval-node node)
      )
)))

(defun sxp::eval-node-get-handler-prefix (name &aux 
    (prefixed-name (intern (+ sxp::eval-prefix ":" name)))
  )
  (if (and (boundp prefixed-name) (eval prefixed-name)) 
    prefixed-name 
    sxp:eval-default
  )
)

(defun sxp::eval-node-get-handler-plist (name)
  (get sxp::eval-prefix name sxp:eval-default)
)
    
(defun sxp::eval-node (node)
  (if (typep node List) 
    ;; composite node
    (with (name (sxp::eval-node-get-handler (sxp:node-get-name node)))
      (if name
	(with (res (list name (list 'quote (sxp:node-get-name node))))
	  (if (sxp:node-get-children node)
	    (with (children-list (list 'list))
	      (dolist (child (sxp:node-get-children node))
		(lappend children-list (sxp::eval-node child))
	      )
	      (lappend res children-list)
	    )
	    (lappend res ())
	  )
	  (dohash (key val (sxp:node-get-attributes node))
	    (lappend res key)
	    (lappend res val)
	  )
	  res
	)
	node
      )
    )
    ;; character data are not parsed
    node
  )
)

;;=============================================================================
;;                    sxp:donode
;;=============================================================================
;; walk the node tree
;; (sxp:donode (node-var tree [form-for-chardatas [depth-first?]]) forms...)

(defmacro sxp:donode (varlist &rest forms &aux 
    tree 
    var 
    sxp:filename
  )
  (if (not (and
	(typep varlist List)
	(>= (length varlist) 2)
	(< (length varlist) 5)
	(or (not varlist) (typep (0 varlist) Symbol))
    ))
    (error 'Errors:BadDoSyntax)
  )
  (setq var (0 varlist))
  (setq tree (1 varlist))
  (setq chardata-forms (2 varlist))

  (if (not (or (typep tree String)
	(and (typep tree Vector)
	  (typep (0 tree) String)
    )))
    (error 'Errors:BadArgType tree 0 Vector)
  )
  (if (3 varlist) 
    ;; depth first
    `(with (sxp:donode-stack (vector)
	sxp:donode-func 
	(lambda (,var)
	  (if 
	    (typep ,var String) ,chardata-forms
	    ,var
	    (progn
	      (if (xnl:children ,var) ;; recurse
		(with (sxp:donode-stack (+ sxp:donode-stack (list ,var)))
		  (dolist (sxp:donode-child (copy (xnl:children ,var)))
		    (sxp:donode-func sxp:donode-child)
	      )))
	      (progn ,@forms)
	      (sxp:clean-node ,var (xnl:children (getn sxp:donode-stack -1)))
	)))
      )
      (sxp:donode-func ,tree)
    )
    ;; topdown order
    `(with (sxp:donode-stack (vector)
	sxp:donode-func 
	(lambda (,var)
	  (if 
	    (typep ,var String) ,chardata-forms
	    ,var
	    (progn
	      (progn ,@forms)
	      (if (and (xnl:name ,var);; node not deleted
		  (typep (xnl:name ,var) String) ;regular node
		  (xnl:children ,var) ;; recurse
		)
		(with (sxp:donode-stack (+ sxp:donode-stack (list ,var)))
		  (dolist (sxp:donode-child (copy (xnl:children ,var)))
		    (sxp:donode-func sxp:donode-child)
	      )))
	      (sxp:clean-node ,var (xnl:children (getn sxp:donode-stack -1)))
	)))
      )
      (sxp:donode-func ,tree)
    )
))

;;=============================================================================
;;                    sxp:clean-node
;;=============================================================================
;; routines to clean nodes that have been deleted/inserted/modified

(defun sxp:clean-tree (tree &aux
  )
  (if (typep tree String)
    ()					;chardata? do nothing
    (not (0 tree))			;replacement? error, a forest!
    (error sxp:E:ForestError)

    (sxp:clean-node-and-recurse tree)
  )
)

(defun sxp:clean-node-and-recurse (node)
  (sxp:clean-nodes (xnl:children node))
  (dolist (child (xnl:children node))
    (if (typep child String) ()
      (sxp:clean-node-and-recurse child)
  ))
)

(defun sxp:clean-nodes (children &aux 
    (i 0) 
    child 
  )
  (while (/= :nil (setq child (get children i :nil)))
    (if 
      (typep child String)		;skip chardata
      (incf i)

      (0 child)				;skip normal nodes
      (incf i)

      ;; (() ...) forms
      (not (1 child))			;delete empty nodes
      (delete children i)

      (typep (1 child) String)		;replace by string
      (progn
	(put children i (1 child))
	(incf i)
      )
					;replace by list of sons
      (and (typep (1 child) List) (= 2 (length child)))
      (with (curi i)
	(delete children i)
	(dolist (subchild (1 child))
	  (insert children curi subchild)
	  (incf curi)
      ))
      (error sxp:E:InternalError child)	;unknown form
  ))
  (sxp:collapse-strings children)
)

(defun sxp:clean-node (child children &aux
    dont-collapse
    (i (seekq children child))
  )
  (if 
    (typep child String)		;skip chardata
    (setq dont-collapse t)
    
    (0 child)				;skip normal nodes
    (setq dont-collapse t)
    
    ;; (() ...) forms
    (not (1 child))			;delete empty nodes
    (delete children i)
    
    (typep (1 child) String)		;replace by string
    (put children i (1 child))
					;replace by list of sons
    (and (typep (1 child) List) (= 2 (length child)))
    (with (curi i)
      (delete children i)
      (dolist (subchild (1 child))
	(insert children curi subchild)
	(incf curi)
    ))
    (error sxp:E:InternalError child)	;unknown form
  )
  (if dont-collapse ()
    (sxp:collapse-strings children)
  )
)

;; collapse strings in children of node
(defun sxp:collapse-strings (children &aux
    (pos 0)
  )
  (while (< pos (- (length children) 1))
    (if (and (typep (pos children) String)
	(typep (getn children (+ pos 1)) String)
      )
      (progn
	(put children pos 
	  (+ (pos children) (getn children (+ pos 1)))
	)
	(delete children (+ pos 1))
      )
      (incf pos)
    )
  )
)

;;=============================================================================
;;                    Built-in entities
;;=============================================================================
(defun sxp:declare-built-in-entities ()
  ;; built-in entities for famous DTDs
  (if (sxp:PS-DTD-or-comments-re-doctypes-name buffer pos)
    (with (dtd-name (tolower (sxp:PS-DTD-or-comments-re-doctypes-name 1)))
      (if (= dtd-name "html")
	(dohash (key val sxp:built-in-entities-html)
	  (sxp:EntDecl-declare key val)
    )))
))
  
(setq sxp:built-in-entities-html [
  "nbsp" " "   ;  no-break space
  "iexcl" "¡"   ;  inverted exclamation mark
  "cent" "¢"   ;  cent sign
  "pound" "£"   ;  pound sterling sign
  "curren" "¤"   ;  general currency sign
  "yen" "¥"   ;  yen sign
  "brvbar" "¦"   ;  broken (vertical) bar
  "sect" "§"   ;  section sign
  "uml" "¨"   ;  umlaut (dieresis)
  "copy" "©"   ;  copyright sign
  "ordf" "ª"   ;  ordinal indicator, feminine
  "laquo" "«"   ;  angle quotation mark, left
  "not" "¬"   ;  not sign
  "shy" "­"   ;  soft hyphen
  "reg" "®"   ;  registered sign
  "macr" "¯"   ;  macron
  "deg" "°"   ;  degree sign
  "plusmn" "±"   ;  plus-or-minus sign
  "sup2" "²"   ;  superscript two
  "sup3" "³"   ;  superscript three
  "acute" "´"   ;  acute accent
  "micro" "µ"   ;  micro sign
  "para" "¶"   ;  pilcrow (paragraph sign)
  "middot" "·"   ;  middle dot
  "cedil" "¸"   ;  cedilla
  "sup1" "¹"   ;  superscript one
  "ordm" "º"   ;  ordinal indicator, masculine
  "raquo" "»"   ;  angle quotation mark, right
  "frac14" "¼"   ;  fraction one-quarter
  "frac12" "½"   ;  fraction one-half
  "frac34" "¾"   ;  fraction three-quarters
  "iquest" "¿"   ;  inverted question mark
  "Agrave" "À"   ;  capital A, grave accent
  "Aacute" "Á"   ;  capital A, acute accent
  "Acirc" "Â"   ;  capital A, circumflex accent
  "Atilde" "Ã"   ;  capital A, tilde
  "Auml" "Ä"   ;  capital A, dieresis or umlaut mark
  "Aring" "Å"   ;  capital A, ring
  "AElig" "Æ"   ;  capital AE diphthong (ligature)
  "Ccedil" "Ç"   ;  capital C, cedilla
  "Egrave" "È"   ;  capital E, grave accent
  "Eacute" "É"   ;  capital E, acute accent
  "Ecirc" "Ê"   ;  capital E, circumflex accent
  "Euml" "Ë"   ;  capital E, dieresis or umlaut mark
  "Igrave" "Ì"   ;  capital I, grave accent
  "Iacute" "Í"   ;  capital I, acute accent
  "Icirc" "Î"   ;  capital I, circumflex accent
  "Iuml" "Ï"   ;  capital I, dieresis or umlaut mark
  "ETH" "Ð"   ;  capital Eth, Icelandic
  "Ntilde" "Ñ"   ;  capital N, tilde
  "Ograve" "Ò"   ;  capital O, grave accent
  "Oacute" "Ó"   ;  capital O, acute accent
  "Ocirc" "Ô"   ;  capital O, circumflex accent
  "Otilde" "Õ"   ;  capital O, tilde
  "Ouml" "Ö"   ;  capital O, dieresis or umlaut mark
  "times" "×"   ;  multiply sign
  "Oslash" "Ø"   ;  capital O, slash
  "Ugrave" "Ù"   ;  capital U, grave accent
  "Uacute" "Ú"   ;  capital U, acute accent
  "Ucirc" "Û"   ;  capital U, circumflex accent
  "Uuml" "Ü"   ;  capital U, dieresis or umlaut mark
  "Yacute" "Ý"   ;  capital Y, acute accent
  "THORN" "Þ"   ;  capital THORN, Icelandic
  "szlig" "ß"   ;  small sharp s, German (sz ligature)
  "agrave" "à"   ;  small a, grave accent
  "aacute" "á"   ;  small a, acute accent
  "acirc" "â"   ;  small a, circumflex accent
  "atilde" "ã"   ;  small a, tilde
  "auml" "ä"   ;  small a, dieresis or umlaut mark
  "aring" "å"   ;  small a, ring
  "aelig" "æ"   ;  small ae diphthong (ligature)
  "ccedil" "ç"   ;  small c, cedilla
  "egrave" "è"   ;  small e, grave accent
  "eacute" "é"   ;  small e, acute accent
  "ecirc" "ê"   ;  small e, circumflex accent
  "euml" "ë"   ;  small e, dieresis or umlaut mark
  "igrave" "ì"   ;  small i, grave accent
  "iacute" "í"   ;  small i, acute accent
  "icirc" "î"   ;  small i, circumflex accent
  "iuml" "ï"   ;  small i, dieresis or umlaut mark
  "eth" "ð"   ;  small eth, Icelandic
  "ntilde" "ñ"   ;  small n, tilde
  "ograve" "ò"   ;  small o, grave accent
  "oacute" "ó"   ;  small o, acute accent
  "ocirc" "ô"   ;  small o, circumflex accent
  "otilde" "õ"   ;  small o, tilde
  "ouml" "ö"   ;  small o, dieresis or umlaut mark
  "divide" "÷"   ;  divide sign
  "oslash" "ø"   ;  small o, slash
  "ugrave" "ù"   ;  small u, grave accent
  "uacute" "ú"   ;  small u, acute accent
  "ucirc" "û"   ;  small u, circumflex accent
  "uuml" "ü"   ;  small u, dieresis or umlaut mark
  "yacute" "ý"   ;  small y, acute accent
  "thorn" "þ"   ;  small thorn, Icelandic
  "yuml" "ÿ"   ;  small y, dieresis or umlaut mark
  ]
)
